mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-04 03:38:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

pyo3 v0.18 migration (#1173)

Browse Source 
* pyo v0.18 migration

* Fix formatting issues of black
This commit is contained in:
mert-kurttutan
2023-03-08 11:27:47 +01:00
committed by GitHub
parent 3138657565
commit 5c18ec5ff5
15 changed files with 138 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/python/examples/custom_components.py
View File

@@ -24,7 +24,7 @@ class JiebaPreTokenizer:
# Just an odd example...
splits = []
last = 0
for (i, char) in enumerate(str(normalized_string)):
for i, char in enumerate(str(normalized_string)):
if char.isnumeric() and int(char) % 2 == 1:
splits.append(normalized_string[last:i])
last = i

1
bindings/python/examples/train_with_datasets.py
View File

@@ -11,6 +11,7 @@ bpe_tokenizer.normalizer = normalizers.Lowercase()
# Initialize a dataset
dataset = datasets.load_dataset("wikitext", "wikitext-103-raw-v1")
# Build an iterator over this dataset
def batch_iterator():
batch_length = 1000