mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-18 06:19:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add content to Strip decoder to allow decoding mid tokens. (#1199)

Browse Source 
* Add `content` to Strip decoder to allow decoding mid tokens.

* Stub.

* Clippy.
This commit is contained in:
Nicolas Patry
2023-03-24 10:14:49 +01:00
committed by GitHub
parent 8a6a8dc9d5
commit 3aaf4946b3
7 changed files with 78 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bindings/python/tests/bindings/test_decoders.py
View File

@@ -111,13 +111,13 @@ class TestFuse:
class TestStrip:
def test_instantiate(self):
assert Strip(left=0, right=0) is not None
assert isinstance(Strip(left=0, right=0), Decoder)
assert isinstance(Strip(left=0, right=0), Strip)
assert isinstance(pickle.loads(pickle.dumps(Strip(left=0, right=0))), Strip)
assert isinstance(Strip(content="_", left=0, right=0), Decoder)
assert isinstance(Strip(content="_", left=0, right=0), Strip)
assert isinstance(pickle.loads(pickle.dumps(Strip(content="_", left=0, right=0))), Strip)
def test_decoding(self):
decoder = Strip(left=1, right=0)
assert decoder.decode(["My", " na", "me"]) == "ynae"
decoder = Strip(content="_", left=1, right=0)
assert decoder.decode(["_My", " na", "me", " _-", "__-"]) == "My name _-_-"
class TestMetaspace: