mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - BPE construction

Browse Source
This commit is contained in:
Anthony MOI
2019-11-22 20:57:54 -05:00
parent f6a9b57b5b
commit 9b71c8f8de
1 changed files with 2 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/python/src/models.rs
View File

@ -30,11 +30,8 @@ pub struct BPE {}
#[pymethods] #[pymethods]
impl BPE { impl BPE {
#[staticmethod] #[staticmethod]
fn from_files() -> PyResult<Model> { fn from_files(vocab: &str, merges: &str) -> PyResult<Model> {
match tk::models::bpe::BPE::from_files( match tk::models::bpe::BPE::from_files(vocab, merges) {
"../../data/gpt2-vocab.json",
"../../data/gpt2-merges.txt",
) {
Err(e) => { Err(e) => {
println!("Error: {:?}", e); println!("Error: {:?}", e);
Err(exceptions::Exception::py_err( Err(exceptions::Exception::py_err(