diff --git a/bindings/python/README.md b/bindings/python/README.md index 914cf8e2..3755c52d 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -14,7 +14,10 @@ maturin develop --release # Then test: pip install transformers -python example.py --file -# or -python example.py +# Download vocab/merges from GPT-2 +wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json +wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt + +python examples/example.py --file --merges gpt2-merges.txt --vocab gpt2-vocab.json +python custom_pre_tokenizer.py --merges gpt2-merges.txt --vocab gpt2-vocab.json ```