mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Python - Fix ByteLevel instantiation from state (#621)

Browse Source
This commit is contained in:
Anthony MOI
2021-02-04 10:16:05 -05:00
committed by GitHub
parent 324cb8d380
commit 57200144ca
6 changed files with 30 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bindings/python/src/processors.rs
View File

@ -232,8 +232,11 @@ pub struct PyByteLevel {}
#[pymethods]
impl PyByteLevel {
#[new]
#[args(trim_offsets = "None")]
fn new(trim_offsets: Option<bool>) -> PyResult<(Self, PyPostProcessor)> {
#[args(trim_offsets = "None", _kwargs = "**")]
fn new(
trim_offsets: Option<bool>,
_kwargs: Option<&PyDict>,
) -> PyResult<(Self, PyPostProcessor)> {
let mut byte_level = ByteLevel::default();
if let Some(to) = trim_offsets {