mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-04 19:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Add bindings for some Model methods

Browse Source
This commit is contained in:
Anthony MOI
2020-09-21 11:28:12 -04:00
committed by Anthony MOI
parent 5276238b1b
commit 08a3128515
1 changed files with 17 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
bindings/python/src/models.rs
View File

@@ -2,6 +2,7 @@ use std::collections::HashMap;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::Arc; use std::sync::Arc;
use crate::token::PyToken;
use pyo3::exceptions; use pyo3::exceptions;
use pyo3::prelude::*; use pyo3::prelude::*;
use pyo3::types::*; use pyo3::types::*;
@@ -105,6 +106,22 @@ impl PyModel {
} }
} }
fn tokenize(&self, tokens: &str) -> PyResult<Vec<PyToken>> {
Ok(ToPyResult(self.model.tokenize(tokens))
.into_py()?
.into_iter()
.map(|t| t.into())
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
self.model.token_to_id(token)
}
fn id_to_token(&self, id: u32) -> Option<&str> {
self.model.id_to_token(id)
}
fn save(&self, folder: &str, name: Option<&str>) -> PyResult<Vec<String>> { fn save(&self, folder: &str, name: Option<&str>) -> PyResult<Vec<String>> {
let saved: PyResult<Vec<_>> = ToPyResult(self.model.save(Path::new(folder), name)).into(); let saved: PyResult<Vec<_>> = ToPyResult(self.model.save(Path::new(folder), name)).into();