From 8e522a38d9721796f39d62fc67bfc8900a923de1 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 29 Aug 2023 13:15:26 +0200 Subject: [PATCH] Updating the docs with the new command. (#1333) --- bindings/python/README.md | 3 +- bindings/python/build-wheels.sh | 31 ------------------- bindings/python/tokenizers_manual_m1_build.sh | 9 ------ docs/source-doc-builder/installation.mdx | 15 +++------ docs/source/installation/python.inc | 9 ++---- 5 files changed, 7 insertions(+), 60 deletions(-) delete mode 100755 bindings/python/build-wheels.sh delete mode 100644 bindings/python/tokenizers_manual_m1_build.sh diff --git a/bindings/python/README.md b/bindings/python/README.md index 029691d6..1573a09f 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -64,8 +64,7 @@ python -m venv .env source .env/bin/activate # Install `tokenizers` in the current virtual env -pip install setuptools_rust -python setup.py install +pip install -e . ``` ### Load a pretrained tokenizer from the Hub diff --git a/bindings/python/build-wheels.sh b/bindings/python/build-wheels.sh deleted file mode 100755 index f032067a..00000000 --- a/bindings/python/build-wheels.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -set -ex - -if ! command -v cargo &> /dev/null -then - curl https://sh.rustup.rs -sSf | sh -s -- -y -fi - -export PATH="$HOME/.cargo/bin:$PATH" -# https://users.rust-lang.org/t/cargo-uses-too-much-memory-being-run-in-qemu/76531 -echo -e "[net]\ngit-fetch-with-cli = true" > "$HOME/.cargo/config" - -for PYBIN in /opt/python/cp{37,38,39,310,311}*/bin; do - export PYTHON_SYS_EXECUTABLE="$PYBIN/python" - - "${PYBIN}/pip" install -U setuptools-rust setuptools wheel - "${PYBIN}/python" setup.py bdist_wheel - rm -rf build/* -done - -for whl in ./dist/*.whl; do - auditwheel repair "$whl" -w dist/ -done - -# Keep only manylinux wheels -rm ./dist/*-linux_* - - -# Upload wheels -/opt/python/cp37-cp37m/bin/pip install -U awscli -/opt/python/cp37-cp37m/bin/python -m awscli s3 sync --exact-timestamps ./dist "s3://tokenizers-releases/python/$DIST_DIR" diff --git a/bindings/python/tokenizers_manual_m1_build.sh b/bindings/python/tokenizers_manual_m1_build.sh deleted file mode 100644 index df7d8113..00000000 --- a/bindings/python/tokenizers_manual_m1_build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash -for VARIABLE in "3.7.12" "3.8.12" "3.9.10" "3.10.2" -do - MACOSX_DEPLOYMENT_TARGET=10.11 SDKROOT="/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk" CFLAGS="-I/usr/include/openssl -I/usr/local/opt/readline/include -I/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include" CPPFLAGS="-I/usr/local/opt/zlib/include" LDFLAGS="-L/usr/lib -L/usr/local/opt/readline/lib" pyenv install $VARIABLE - - ~/.pyenv/versions/$VARIABLE/bin/pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall - - MACOSX_DEPLOYMENT_TARGET=10.11 ~/.pyenv/versions/$VARIABLE/bin/python setup.py bdist_wheel -done diff --git a/docs/source-doc-builder/installation.mdx b/docs/source-doc-builder/installation.mdx index 0d9dfbb1..a16f39dd 100644 --- a/docs/source-doc-builder/installation.mdx +++ b/docs/source-doc-builder/installation.mdx @@ -52,19 +52,12 @@ cd tokenizers/bindings/python ``` At this point you should have your [virtual environment]() already -activated. In order to compile 🤗 Tokenizers, you need to install the -Python package `setuptools_rust`: +activated. In order to compile 🤗 Tokenizers, you need to: ```bash -pip install setuptools_rust +pip install -e . ``` -Then you can have 🤗 Tokenizers compiled and installed in your virtual -environment with the following command: - -```bash -python setup.py install -``` ## Crates.io @@ -74,7 +67,7 @@ python setup.py install You just need to add it to your `Cargo.toml`: ```bash -tokenizers = "0.10" +cargo add tokenizers ``` @@ -86,4 +79,4 @@ You can simply install 🤗 Tokenizers with npm using: npm install tokenizers ``` - \ No newline at end of file + diff --git a/docs/source/installation/python.inc b/docs/source/installation/python.inc index 406a6a4d..6e7a8b08 100644 --- a/docs/source/installation/python.inc +++ b/docs/source/installation/python.inc @@ -37,11 +37,6 @@ Then we go into the python bindings folder:: cd tokenizers/bindings/python At this point you should have your `virtual environment`_ already activated. In order to -compile 🤗 Tokenizers, you need to install the Python package :obj:`setuptools_rust`:: +compile 🤗 Tokenizers, you need to:: - pip install setuptools_rust - -Then you can have 🤗 Tokenizers compiled and installed in your virtual environment with -the following command:: - - python setup.py install + pip install -e .