mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-03 11:18:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix typo in documentation (#743)

Browse Source 
* Doc - Fix typo (And instance of -> An instance of)

* Add missing text_signature for WordLevel.from_file

Co-authored-by: Anthony Moi <m.anthony.moi@gmail.com>
This commit is contained in:
kingyiusuen
2021-08-13 07:08:23 -05:00
committed by GitHub
parent 71fb73e129
commit c1100dcbe3
2 changed files with 26 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
bindings/python/py_src/tokenizers/models/__init__.pyi
View File

@@ -294,6 +294,28 @@ class WordLevel(Model):
def __init__(self, vocab, unk_token):
pass
@staticmethod
def from_file(vocab, unk_token):
"""
Instantiate a WordLevel model from the given file
This method is roughly equivalent to doing::
vocab = WordLevel.read_file(vocab_filename)
wordlevel = WordLevel(vocab)
If you don't need to keep the :obj:`vocab` values lying around, this method is
more optimized than manually calling :meth:`~tokenizers.models.WordLevel.read_file` to
initialize a :class:`~tokenizers.models.WordLevel`
Args:
vocab (:obj:`str`):
The path to a :obj:`vocab.json` file
Returns:
:class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file
"""
pass
def id_to_token(self, id):
"""
Get the token associated to an ID
@@ -403,7 +425,7 @@ class WordPiece(Model):
The path to a :obj:`vocab.txt` file
Returns:
:class:`~tokenizers.models.WordPiece`: And instance of WordPiece loaded from file
:class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file
"""
pass
def id_to_token(self, id):

5
bindings/python/src/models.rs
View File

@@ -636,7 +636,7 @@ impl PyWordPiece {
/// The path to a :obj:`vocab.txt` file
///
/// Returns:
/// :class:`~tokenizers.models.WordPiece`: And instance of WordPiece loaded from file
/// :class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file
#[classmethod]
#[args(kwargs = "**")]
#[text_signature = "(vocab, **kwargs)"]
@@ -748,9 +748,10 @@ impl PyWordLevel {
/// The path to a :obj:`vocab.json` file
///
/// Returns:
/// :class:`~tokenizers.models.WordLevel`: And instance of WordLevel loaded from file
/// :class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file
#[classmethod]
#[args(unk_token = "None")]
#[text_signature = "(vocab, unk_token)"]
fn from_file(
_cls: &PyType,
py: Python,