diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index b1c11730..07dae094 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -382,9 +382,7 @@ impl Serialize for PyPreTokenizerWrapper { ser.end() } PyPreTokenizerWrapper::Wrapped(inner) => inner.serialize(serializer), - PyPreTokenizerWrapper::Custom(_) => { - unreachable!("Custom pretokenizers are currently disabled, how did you get here?") - } + PyPreTokenizerWrapper::Custom(inner) => inner.serialize(serializer), } } }