From 512e85dfda75b868e83de7ddfe9073012c114398 Mon Sep 17 00:00:00 2001 From: Anthony MOI Date: Sun, 24 Nov 2019 00:55:13 -0500 Subject: [PATCH] Update python README --- bindings/python/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bindings/python/README.md b/bindings/python/README.md index 914cf8e2..3755c52d 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -14,7 +14,10 @@ maturin develop --release # Then test: pip install transformers -python example.py --file -# or -python example.py +# Download vocab/merges from GPT-2 +wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json +wget https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt + +python examples/example.py --file --merges gpt2-merges.txt --vocab gpt2-vocab.json +python custom_pre_tokenizer.py --merges gpt2-merges.txt --vocab gpt2-vocab.json ```