mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-04 00:09:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

ByteLevel handles prefix spaces

Browse Source
This commit is contained in:
Anthony MOI
2019-12-17 18:35:40 -05:00
parent 6766585965
commit 4d14b08afe
4 changed files with 39 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/src/decoders.rs
View File

@ -32,7 +32,7 @@ impl ByteLevel {
#[staticmethod]
fn new() -> PyResult<Decoder> {
Ok(Decoder {
decoder: Container::Owned(Box::new(tk::decoders::byte_level::ByteLevel)),
decoder: Container::Owned(Box::new(tk::decoders::byte_level::ByteLevel::new(false))),
})
}
}