mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-07 05:08:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix split on special tokens & bump version

Browse Source
This commit is contained in:
Anthony MOI
2020-01-12 02:35:45 -05:00
parent 32d3955cd4
commit fc9e81d4ab
6 changed files with 32 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/python/Cargo.lock generated
View File

@@ -555,7 +555,7 @@ dependencies = [
[[package]]
name = "tokenizers"
version = "0.6.0"
version = "0.6.1"
dependencies = [
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"indicatif 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -571,10 +571,10 @@ dependencies = [
[[package]]
name = "tokenizers-python"
version = "0.1.0"
version = "0.1.1"
dependencies = [
"pyo3 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
"tokenizers 0.6.0",
"tokenizers 0.6.1",
]
[[package]]

2
bindings/python/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "tokenizers-python"
version = "0.1.0"
version = "0.1.1"
authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
edition = "2018"

2
bindings/python/setup.py
View File

@@ -3,7 +3,7 @@ from setuptools_rust import Binding, RustExtension
setup(
name="tokenizers",
version="0.1.0",
version="0.1.1",
description="Fast and Customizable Tokenizers",
long_description=open("README.md", "r", encoding="utf-8").read(),
long_description_content_type="text/markdown",

2
bindings/python/tokenizers/__init__.py
View File

@@ -1,4 +1,4 @@
__version__ = "0.1.0"
__version__ = "0.1.1"
from .tokenizers import Tokenizer, Encoding
from .tokenizers import decoders