mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Unsound call of set_var (#1664)

Browse Source 
* refactor: lift cloning to caller

* refactor: do not elide lifetimes as in Rust 2018

* fix: unsound use of env::set_var, was breaking stdlib change to make unsafe

It is generally not safe to set env variables. The correct way to set a config
value that needs to be overridden is to hold a copy internal to the library and
only read from the environment.
This commit is contained in:
sftse
2024-10-25 15:44:30 +02:00
committed by GitHub
parent a8738a95d1
commit 6ea758872d
6 changed files with 33 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/normalizers.rs
View File

@ -553,7 +553,7 @@ impl tk::tokenizer::Normalizer for CustomNormalizer {
Python::with_gil(|py| {
let normalized = PyNormalizedStringRefMut::new(normalized);
let py_normalized = self.inner.bind(py);
py_normalized.call_method("normalize", (normalized.get(),), None)?;
py_normalized.call_method("normalize", (normalized.get().clone(),), None)?;
Ok(())
})
}

2
bindings/python/src/pre_tokenizers.rs
View File

@ -634,7 +634,7 @@ impl tk::tokenizer::PreTokenizer for CustomPreTokenizer {
Python::with_gil(|py| {
let pretok = PyPreTokenizedStringRefMut::new(sentence);
let py_pretok = self.inner.bind(py);
py_pretok.call_method("pre_tokenize", (pretok.get(),), None)?;
py_pretok.call_method("pre_tokenize", (pretok.get().clone(),), None)?;
Ok(())
})
}

10
bindings/python/src/utils/mod.rs
View File

@ -18,11 +18,11 @@ pub trait DestroyPtr {
fn destroy(&mut self);
}
pub struct RefMutGuard<'r, T: DestroyPtr + Clone> {
pub struct RefMutGuard<'r, T: DestroyPtr> {
content: T,
r: PhantomData<&'r mut T>,
}
impl<T: DestroyPtr + Clone> RefMutGuard<'_, T> {
impl<T: DestroyPtr> RefMutGuard<'_, T> {
pub fn new(content: T) -> Self {
Self {
content,
@ -30,12 +30,12 @@ impl<T: DestroyPtr + Clone> RefMutGuard<'_, T> {
}
}
pub fn get(&self) -> T {
self.content.clone()
pub fn get(&self) -> &T {
&self.content
}
}
impl<T: DestroyPtr + Clone> Drop for RefMutGuard<'_, T> {
impl<T: DestroyPtr> Drop for RefMutGuard<'_, T> {
fn drop(&mut self) {
self.content.destroy()
}

2
bindings/python/src/utils/normalization.rs
View File

@ -396,7 +396,7 @@ impl DestroyPtr for PyNormalizedStringRefMut {
}
impl PyNormalizedStringRefMut {
pub fn new(normalized: &mut NormalizedString) -> RefMutGuard<Self> {
pub fn new(normalized: &mut NormalizedString) -> RefMutGuard<'_, Self> {
RefMutGuard::new(Self {
inner: RefMutContainer::new(normalized),
})

4
bindings/python/src/utils/pretokenization.rs
View File

@ -39,7 +39,7 @@ fn normalize(pretok: &mut PreTokenizedString, func: &Bound<'_, PyAny>) -> PyResu
} else {
ToPyResult(pretok.normalize(|normalized| {
let norm = PyNormalizedStringRefMut::new(normalized);
func.call((norm.get(),), None)?;
func.call((norm.get().clone(),), None)?;
Ok(())
}))
.into()
@ -272,7 +272,7 @@ impl DestroyPtr for PyPreTokenizedStringRefMut {
}
impl PyPreTokenizedStringRefMut {
pub fn new(pretok: &mut tk::PreTokenizedString) -> RefMutGuard<Self> {
pub fn new(pretok: &mut tk::PreTokenizedString) -> RefMutGuard<'_, Self> {
// SAFETY: This is safe because we return a RefMutGuard here.
// The compiler will make sure the &mut stays valid as necessary.
RefMutGuard::new(Self {