mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

pyo3: update to 0.20 (#1386)

Browse Source 
Co-authored-by: Mike Lui <mikelui@meta.com>
This commit is contained in:
Michael Lui
2024-01-11 06:03:13 -10:00
committed by GitHub
parent 8939d4e26d
commit 888dd4bc65
4 changed files with 6 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/python/Cargo.toml
View File

@ -14,8 +14,8 @@ serde = { version = "1.0", features = [ "rc", "derive" ]}
serde_json = "1.0"
libc = "0.2"
env_logger = "0.10.0"
pyo3 = { version = "0.19" }
numpy = "0.19.0"
pyo3 = { version = "0.20" }
numpy = "0.20.0"
ndarray = "0.15"
onig = { version = "6.4", default-features = false }
itertools = "0.11"
@ -26,7 +26,7 @@ path = "../../tokenizers"
[dev-dependencies]
tempfile = "3.8"
pyo3 = { version = "0.19", features = ["auto-initialize"] }
pyo3 = { version = "0.20", features = ["auto-initialize"] }
[features]
defaut = ["pyo3/extension-module"]

6
bindings/python/src/processors.rs
View File

@ -266,15 +266,15 @@ impl FromPyObject<'_> for PySpecialToken {
Ok(Self(v.into()))
} else if let Ok(d) = ob.downcast::<PyDict>() {
let id = d
.get_item("id")
.get_item("id")?
.ok_or_else(|| exceptions::PyValueError::new_err("`id` must be specified"))?
.extract::<String>()?;
let ids = d
.get_item("ids")
.get_item("ids")?
.ok_or_else(|| exceptions::PyValueError::new_err("`ids` must be specified"))?
.extract::<Vec<u32>>()?;
let tokens = d
.get_item("tokens")
.get_item("tokens")?
.ok_or_else(|| exceptions::PyValueError::new_err("`tokens` must be specified"))?
.extract::<Vec<String>>()?;

1
bindings/python/src/tokenizer.rs
View File

@ -7,7 +7,6 @@ use pyo3::exceptions;
use pyo3::intern;
use pyo3::prelude::*;
use pyo3::types::*;
use pyo3::AsPyPointer;
use tk::models::bpe::BPE;
use tk::tokenizer::{
Model, PaddingDirection, PaddingParams, PaddingStrategy, PostProcessor, TokenizerImpl,

1
bindings/python/src/utils/iterators.rs
View File

@ -1,5 +1,4 @@
use pyo3::prelude::*;
use pyo3::AsPyPointer;
use std::collections::VecDeque;
/// An simple iterator that can be instantiated with a specified length.