mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve Makefile

Browse Source
This commit is contained in:
epwalsh
2020-01-08 10:06:42 -08:00
parent 383123e21f
commit 4e7fc93971
1 changed files with 13 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
tokenizers/Makefile
View File

@ -1,6 +1,5 @@
#
# Cargo helpers.
#
BENCHMARK_DIR = benches
BENCHMARK_RESOURCES = $(BENCHMARK_DIR)/gpt2-vocab.json $(BENCHMARK_DIR)/gpt2-merges.txt $(BENCHMARK_DIR)/big.txt
.PHONY : build
build :
@ -17,7 +16,7 @@ format :
.PHONY : lint
lint :
cargo fmt -- --check
cargo fmt -- ./benches/*.rs --check
cargo fmt -- $(BENCHMARK_DIR)/*.rs --check
cargo clippy --all-targets --all-features -- -D warnings
.PHONY : test
@ -28,23 +27,19 @@ test :
doc :
cargo doc
.PHONY : publish
publish :
cargo publish
.PHONY : all-checks
all-checks : lint test doc
.PHONY : bench
bench :
@echo 'Updating benchmark fixtures...'
@test -f ./benches/gpt2-vocab.json \
&& echo 'gpt2-vocab.json up-to-date' \
|| wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json -O ./benches/gpt2-vocab.json
@test -f ./benches/gpt2-merges.txt \
&& echo 'gpt2-merges.txt up-to-date' \
|| wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt -O ./benches/gpt2-merges.txt
@test -f ./benches/big.txt \
&& echo 'big.txt up-to-date' \
|| wget https://norvig.com/big.txt -O ./benches/big.txt
bench : $(BENCHMARK_RESOURCES)
cargo bench -- --verbose
.PHONY : publish
publish :
cargo publish
$(BENCHMARK_DIR)/gpt2-% :
wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-$* -O $@
$(BENCHMARK_DIR)/big.txt :
wget https://norvig.com/big.txt -O $@