mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #333 from huggingface/fix-added-tokens

Browse Source 
Python - Fix Added token deserialization
This commit is contained in:
Anthony MOI
2020-07-06 14:52:37 -04:00
committed by GitHub
parent 2194970679 d33af1a3be
commit e874641cf9
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bindings/python/src/tokenizer.rs
View File

@ -107,6 +107,7 @@ impl AddedToken {
for (key, value) in state { for (key, value) in state {
let key: &str = key.extract()?; let key: &str = key.extract()?;
match key { match key {
"content" => self.content = value.extract()?,
"single_word" => self.single_word = Some(value.extract()?), "single_word" => self.single_word = Some(value.extract()?),
"lstrip" => self.lstrip = Some(value.extract()?), "lstrip" => self.lstrip = Some(value.extract()?),
"rstrip" => self.rstrip = Some(value.extract()?), "rstrip" => self.rstrip = Some(value.extract()?),