mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-18 06:19:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Improve pre-tokenizers docs

Browse Source
This commit is contained in:
Anthony MOI
2020-11-20 17:17:46 -05:00
committed by Anthony MOI
parent 5842b3db73
commit 933a2a9c99
3 changed files with 375 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/python/tests/bindings/test_pre_tokenizers.py
View File

@@ -58,7 +58,7 @@ class TestMetaspace:
def test_instantiate(self):
assert Metaspace() is not None
assert Metaspace(replacement="-") is not None
with pytest.raises(Exception, match="replacement must be a character"):
with pytest.raises(ValueError, match="expected a string of length 1"):
Metaspace(replacement="")
assert Metaspace(add_prefix_space=True) is not None
assert isinstance(Metaspace(), PreTokenizer)
@@ -69,7 +69,7 @@ class TestMetaspace:
class TestCharDelimiterSplit:
def test_instantiate(self):
assert CharDelimiterSplit("-") is not None
with pytest.raises(Exception, match="delimiter must be a single character"):
with pytest.raises(ValueError, match="expected a string of length 1"):
CharDelimiterSplit("")
assert isinstance(CharDelimiterSplit(" "), PreTokenizer)
assert isinstance(CharDelimiterSplit(" "), CharDelimiterSplit)