mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing roberta type id (everything is zero). (#1072)

Browse Source 
* Fixing roberta type ids (everything is zero).

* We need to fix type_ids for all sequence even when not changing

anything else.

* Fixing tests hopefully better.
This commit is contained in:
Nicolas Patry
2022-09-26 18:00:41 +02:00
committed by GitHub
parent 6e5569a540
commit 5f6e978452
4 changed files with 15 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bindings/python/Cargo.toml
View File

@ -14,7 +14,7 @@ serde = { version = "1.0", features = [ "rc", "derive" ]}
serde_json = "1.0"
libc = "0.2"
env_logger = "0.7.1"
pyo3 = { version = "0.16.2", features = ["extension-module"] }
pyo3 = { version = "0.16.2" }
numpy = "0.16.2"
ndarray = "0.13"
onig = { version = "6.0", default-features = false }
@ -28,5 +28,6 @@ path = "../../tokenizers"
tempfile = "3.1"
[features]
default = ["pyo3/extension-module"]
test = ["pyo3/auto-initialize"]