mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-05 12:18:20 +00:00
Re-enabling Custom Serialize
This commit is contained in:
committed by
Anthony MOI
parent
6887c0f04d
commit
1994dcad6e
@@ -382,9 +382,7 @@ impl Serialize for PyPreTokenizerWrapper {
|
||||
ser.end()
|
||||
}
|
||||
PyPreTokenizerWrapper::Wrapped(inner) => inner.serialize(serializer),
|
||||
PyPreTokenizerWrapper::Custom(_) => {
|
||||
unreachable!("Custom pretokenizers are currently disabled, how did you get here?")
|
||||
}
|
||||
PyPreTokenizerWrapper::Custom(inner) => inner.serialize(serializer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user