mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-18 06:19:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feature: Handle invalid truncate direction (#858)

Browse Source 
* refacto: TruncateDirection -> TruncationDirection

* feat(node): invalid direction will throw

* feat(python): invalid direction will throw

* Update bindings/node/lib/bindings/raw-encoding.test.ts

* Update bindings/python/tests/bindings/test_encoding.py

Co-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>
This commit is contained in:
Luc Georges
2021-12-27 14:31:57 +01:00
committed by GitHub
parent 38a85b2112
commit c4c9de23a5
6 changed files with 47 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bindings/python/tests/bindings/test_encoding.py
View File

@@ -106,3 +106,15 @@ class TestEncoding:
assert pair.char_to_word(2, 0) == 1
assert pair.char_to_word(2, 1) == None
assert pair.char_to_word(3, 1) == 1
def test_truncation(self, encodings):
single, _ = encodings
single.truncate(2, 1, "right")
assert single.tokens == ["[CLS]", "i"]
assert single.overflowing[0].tokens == ["i", "love"]
def test_invalid_truncate_direction(self, encodings):
single, _ = encodings
with pytest.raises(ValueError) as excinfo:
single.truncate(2, 1, "not_a_direction")
assert "Invalid truncation direction value : not_a_direction" == str(excinfo.value)