mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-12 05:18:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #99 from kdexd/get-vocab-size

Browse Source 
Expose get_vocab_size in tokenizer python API.
This commit is contained in:
MOI Anthony
2020-02-03 11:52:29 -05:00
committed by GitHub
parent 0094393610 b027c63c37
commit a48b337d7b
1 changed files with 12 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bindings/python/tokenizers/implementations/base_tokenizer.py
View File

@@ -13,6 +13,18 @@ class BaseTokenizer:
self._tokenizer.get_vocab_size(), self._tokenizer.get_vocab_size(),
', '.join(k + '=' + str(v) for k, v in self._parameters.items())) ', '.join(k + '=' + str(v) for k, v in self._parameters.items()))
def get_vocab_size(self, with_added_tokens: bool = True):
""" Return the size of vocabulary, with or without added tokens.
Args:
with_added_tokens: (`optional`) bool:
Whether to count in added special tokens or not
Returns:
Size of vocabulary
"""
return self._tokenizer.get_vocab_size(with_added_tokens=with_added_tokens)
def enable_padding(self, def enable_padding(self,
direction: Optional[str] = "right", direction: Optional[str] = "right",
pad_id: Optional[int] = 0, pad_id: Optional[int] = 0,