mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-09-02 23:39:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

WordPiece decoder with customizable prefix

Browse Source
This commit is contained in:
Anthony MOI
2020-01-06 20:20:42 -05:00
parent 742974f0c9
commit 4b9ae66419
3 changed files with 35 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bindings/python/src/decoders.rs
View File

@ -42,9 +42,17 @@ pub struct WordPiece {}
#[pymethods]
impl WordPiece {
#[staticmethod]
fn new() -> PyResult<Decoder> {
fn new(kwargs: Option<&PyDict>) -> PyResult<Decoder> {
let mut prefix = String::from("##");
if let Some(kwargs) = kwargs {
if let Some(p) = kwargs.get_item("prefix") {
prefix = p.extract()?;
}
}
Ok(Decoder {
decoder: Container::Owned(Box::new(tk::decoders::wordpiece::WordPiece)),
decoder: Container::Owned(Box::new(tk::decoders::wordpiece::WordPiece::new(prefix))),
})
}
}

9
bindings/python/tokenizers/decoders/__init__.pyi
View File

@ -24,6 +24,11 @@ class WordPiece:
"""
@staticmethod
def new() -> Decoder:
""" Instantiate a new WordPiece Decoder """
def new(prefix: str="##") -> Decoder:
""" Instantiate a new WordPiece Decoder
Args:
prefix: str:
The prefix to use for subwords that are not a beginning-of-word
"""
pass