mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
09069717e9
* version = "0.15.3-dev-0” Improve performances of meta space, but also just fix it. (transformers) ➜ transformers git:(refactor-default-llama) ✗ python ../scripts/gemma-dummy.py Token indices sequence length is longer than the specified maximum sequence length for this model (14999 > 2048). Running this sequence through the model will result in indexing errors ['<REPR_END>', '▁inform', '<s>', '.', '▁Hey', '<unk>', '.', '▁', '▁', '▁', '▁', '▁', '▁', '▁.'] ['▁inform', '<s>', '.', '▁Hey', '<unk>', '.', '▁', '▁', '▁', '▁', '▁', '▁', '▁.'] [0.0006330013275146484, 0.0014591217041015625, 0.015890836715698242, 0.18584918975830078, 2.1726326942443848] (transformers) ➜ transformers git:(refactor-default-llama) ✗ python ../scripts/gemma-dummy.py Token indices sequence length is longer than the specified maximum sequence length for this model (10000 > 2048). Running this sequence through the model will result in indexing errors ['<REPR_END>', 'in', 'form', '<s>', '.', '▁Hey', '<unk>', '.', '▁▁▁▁▁▁', '▁.'] ['in', 'form', '<s>', '.', '▁Hey', '<unk>', '.', '▁▁▁▁▁▁', '▁.'] [0.0008409023284912109, 0.0008909702301025391, 0.00882411003112793, 0.10214710235595703, 1.187899112701416] * well what do we have * nit * be BC with non legacy * unrelated change for clippy * fix test * splitting is a must for word_ids * fmt and lint * Fixing everything (hopefully better). * Fixing node. * Including yarn.lock * Lint. * Stubs. * revert to use split * fix merge issues * fix tests * finish fixing tests * ruff --------- Co-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>
38 lines
948 B
Makefile
38 lines
948 B
Makefile
.PHONY: style check-style test
|
|
|
|
DATA_DIR = data
|
|
|
|
dir_guard=@mkdir -p $(@D)
|
|
check_dirs := examples py_src/tokenizers tests
|
|
|
|
# Format source code automatically
|
|
style:
|
|
python stub.py
|
|
ruff check $(check_dirs) --fix
|
|
ruff format $(check_dirs)
|
|
|
|
# Check the source code is formatted correctly
|
|
check-style:
|
|
python stub.py --check
|
|
ruff check examples py_src/tokenizers tests
|
|
ruff format --check examples py_src/tokenizers tests
|
|
|
|
TESTS_RESOURCES = $(DATA_DIR)/small.txt $(DATA_DIR)/roberta.json
|
|
|
|
# Launch the test suite
|
|
test: $(TESTS_RESOURCES)
|
|
pip install pytest requests setuptools_rust numpy pyarrow datasets
|
|
python -m pytest -s -v tests
|
|
cargo test --no-default-features
|
|
|
|
$(DATA_DIR)/big.txt :
|
|
$(dir_guard)
|
|
wget https://norvig.com/big.txt -O $@
|
|
|
|
$(DATA_DIR)/small.txt : $(DATA_DIR)/big.txt
|
|
head -100 $(DATA_DIR)/big.txt > $@
|
|
|
|
$(DATA_DIR)/roberta.json :
|
|
$(dir_guard)
|
|
wget https://huggingface.co/roberta-large/raw/main/tokenizer.json -O $@
|