mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-09 06:08:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ensure pad_to_multiple_of is correctly forwarded in base_tokenizer.py

Browse Source
This commit is contained in:
Morgan Funtowicz
2020-05-31 10:02:59 +02:00
parent 0934fe5803
commit fcb4e76d9b
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bindings/python/tokenizers/implementations/base_tokenizer.py
View File

@@ -84,6 +84,7 @@ class BaseTokenizer:
""" """
return self._tokenizer.enable_padding( return self._tokenizer.enable_padding(
direction=direction, direction=direction,
pad_to_multiple_of=pad_to_multiple_of,
pad_id=pad_id, pad_id=pad_id,
pad_type_id=pad_type_id, pad_type_id=pad_type_id,
pad_token=pad_token, pad_token=pad_token,