mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-03 19:28:20 +00:00
Fix typo in documentation (#743)
* Doc - Fix typo (And instance of -> An instance of) * Add missing text_signature for WordLevel.from_file Co-authored-by: Anthony Moi <m.anthony.moi@gmail.com>
This commit is contained in:
@@ -294,6 +294,28 @@ class WordLevel(Model):
|
|||||||
|
|
||||||
def __init__(self, vocab, unk_token):
|
def __init__(self, vocab, unk_token):
|
||||||
pass
|
pass
|
||||||
|
@staticmethod
|
||||||
|
def from_file(vocab, unk_token):
|
||||||
|
"""
|
||||||
|
Instantiate a WordLevel model from the given file
|
||||||
|
|
||||||
|
This method is roughly equivalent to doing::
|
||||||
|
|
||||||
|
vocab = WordLevel.read_file(vocab_filename)
|
||||||
|
wordlevel = WordLevel(vocab)
|
||||||
|
|
||||||
|
If you don't need to keep the :obj:`vocab` values lying around, this method is
|
||||||
|
more optimized than manually calling :meth:`~tokenizers.models.WordLevel.read_file` to
|
||||||
|
initialize a :class:`~tokenizers.models.WordLevel`
|
||||||
|
|
||||||
|
Args:
|
||||||
|
vocab (:obj:`str`):
|
||||||
|
The path to a :obj:`vocab.json` file
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
:class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file
|
||||||
|
"""
|
||||||
|
pass
|
||||||
def id_to_token(self, id):
|
def id_to_token(self, id):
|
||||||
"""
|
"""
|
||||||
Get the token associated to an ID
|
Get the token associated to an ID
|
||||||
@@ -403,7 +425,7 @@ class WordPiece(Model):
|
|||||||
The path to a :obj:`vocab.txt` file
|
The path to a :obj:`vocab.txt` file
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
:class:`~tokenizers.models.WordPiece`: And instance of WordPiece loaded from file
|
:class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
def id_to_token(self, id):
|
def id_to_token(self, id):
|
||||||
|
|||||||
@@ -636,7 +636,7 @@ impl PyWordPiece {
|
|||||||
/// The path to a :obj:`vocab.txt` file
|
/// The path to a :obj:`vocab.txt` file
|
||||||
///
|
///
|
||||||
/// Returns:
|
/// Returns:
|
||||||
/// :class:`~tokenizers.models.WordPiece`: And instance of WordPiece loaded from file
|
/// :class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file
|
||||||
#[classmethod]
|
#[classmethod]
|
||||||
#[args(kwargs = "**")]
|
#[args(kwargs = "**")]
|
||||||
#[text_signature = "(vocab, **kwargs)"]
|
#[text_signature = "(vocab, **kwargs)"]
|
||||||
@@ -748,9 +748,10 @@ impl PyWordLevel {
|
|||||||
/// The path to a :obj:`vocab.json` file
|
/// The path to a :obj:`vocab.json` file
|
||||||
///
|
///
|
||||||
/// Returns:
|
/// Returns:
|
||||||
/// :class:`~tokenizers.models.WordLevel`: And instance of WordLevel loaded from file
|
/// :class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file
|
||||||
#[classmethod]
|
#[classmethod]
|
||||||
#[args(unk_token = "None")]
|
#[args(unk_token = "None")]
|
||||||
|
#[text_signature = "(vocab, unk_token)"]
|
||||||
fn from_file(
|
fn from_file(
|
||||||
_cls: &PyType,
|
_cls: &PyType,
|
||||||
py: Python,
|
py: Python,
|
||||||
|
|||||||
Reference in New Issue
Block a user