mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enabling the option to use fancy_regex instead of onig.

Browse Source
This commit is contained in:
Nicolas Patry
2024-08-01 12:10:16 +02:00
parent 9e0c791f2b
commit 7415e28536
3 changed files with 34 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bindings/python/Cargo.toml
View File

@ -17,7 +17,6 @@ env_logger = "0.11"
pyo3 = { version = "0.21" }
numpy = "0.21"
ndarray = "0.15"
onig = { version = "6.4", default-features = false }
itertools = "0.12"
[dependencies.tokenizers]

8
bindings/python/src/utils/regex.rs
View File

@ -1,11 +1,11 @@
use onig::Regex;
use pyo3::exceptions;
use pyo3::prelude::*;
use tk::utils::SysRegex;
/// Instantiate a new Regex with the given pattern
#[pyclass(module = "tokenizers", name = "Regex")]
pub struct PyRegex {
pub inner: Regex,
pub inner: SysRegex,
pub pattern: String,
}
@ -15,8 +15,8 @@ impl PyRegex {
#[pyo3(text_signature = "(self, pattern)")]
fn new(s: &str) -> PyResult<Self> {
Ok(Self {
inner: Regex::new(s)
.map_err(|e| exceptions::PyException::new_err(e.description().to_owned()))?,
inner: SysRegex::new(s)
.map_err(|e| exceptions::PyException::new_err(e.to_string().to_owned()))?,
pattern: s.to_owned(),
})
}