mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-06 04:38:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix split on special tokens & bump version

Browse Source
This commit is contained in:
Anthony MOI
2020-01-12 02:35:45 -05:00
parent 32d3955cd4
commit fc9e81d4ab
6 changed files with 32 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/setup.py
View File

@@ -3,7 +3,7 @@ from setuptools_rust import Binding, RustExtension
setup(
name="tokenizers",
version="0.1.0",
version="0.1.1",
description="Fast and Customizable Tokenizers",
long_description=open("README.md", "r", encoding="utf-8").read(),
long_description_content_type="text/markdown",