mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-31 04:29:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix typos

Browse Source
This commit is contained in:
thomwolf
2020-01-10 10:47:36 +01:00
parent d46ea842c2
commit 1a802cb484
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/encoding.rs
View File

@ -109,7 +109,7 @@ impl Encoding {
impl PyObjectProtocol for Encoding { impl PyObjectProtocol for Encoding {
fn __repr__(&self) -> PyResult<String> { fn __repr__(&self) -> PyResult<String> {
Ok(format!( Ok(format!(
"Encoding(num_tokens={}, attributs=[ids, type_ids, tokens, offsets, \ "Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \
attention_mask, special_tokens_mask, overflowing, original_str, normalized_str])", attention_mask, special_tokens_mask, overflowing, original_str, normalized_str])",
self.encoding.get_ids().len() self.encoding.get_ids().len()
)) ))

2
bindings/python/tokenizers/implementations/base_tokenizer.py
View File

@ -12,7 +12,7 @@ class BaseTokenizer:
def __repr__(self): def __repr__(self):
return "Tokenizer(vocabulary_size={}, {})".format( return "Tokenizer(vocabulary_size={}, {})".format(
self._tokenizer.get_vocab_size(), self._tokenizer.get_vocab_size(),
', '.join(k + ': ' + str(v) for k, v in self._parameters.items())) ', '.join(k + '=' + str(v) for k, v in self._parameters.items()))
def enable_padding(self, def enable_padding(self,
direction: Optional[str] = "right", direction: Optional[str] = "right",