Preparing for 0.11.0 Re-release. (#856)

* Starting from master again. Upgrade libssl everywhere on quay Extra is ubuntu based (running the quay in a container). making only extra run + attempt to fix ssl update. Extra with newer openssl versions. `-y`. Use checkoint@v2 + remove `-` from environment name. Debugging back the conda release.. Attempt to use `base` env. 3.7 requires `activate-environement: true. MacOS and windows don't run on manylinux. Remove yum on windows/macOs. Miniconda doesn't like manylinux2014 anymore ? Attempting different approach for manylinux + conda. Use wget. Extra bracet. Executing $filename Activate the env. Activate the env on eevery step that requires it. Openssl-devel. Activating env for extracting version ? Retest all workflows. Manylinux2010 requires checkout@v1 Run on tag for extra and conda again. openssl-devel. * Putting back into deploy state. * Adding links in CHANGELOG. * Remove clippy from changelog.
2025-08-23 00:35:35 +00:00 · 2021-12-23 16:43:48 +01:00
parent 04368b1998
commit 943f4ef469
4 changed files with 112 additions and 18 deletions
--- a/.github/workflows/python-release-conda.yml
+++ b/.github/workflows/python-release-conda.yml
@ -11,28 +11,24 @@ env:
 jobs:
  build_and_package:
    runs-on: ${{ matrix.os }}
-    container: quay.io/pypa/manylinux2014_x86_64
    strategy:
-      fail-fast: false
      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [windows-latest, macos-latest]
        python: [3.7, 3.8, 3.9]
-    defaults:
-      run:
-        shell: bash -l {0}

    steps:
      - name: Checkout repository
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2

      - name: Install miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          auto-update-conda: true
-          auto-activate-base: false
-          activate-environment: "build-tokenizers"
          python-version: ${{ matrix.python }}
-          channels: conda-forge
+
+      - name: Conda info
+        shell: bash -l {0}
+        run: conda info

      - name: Install Rust
        uses: actions-rs/toolchain@v1
@ -40,19 +36,91 @@ jobs:
          toolchain: stable

      - name: Setup conda env
+        shell: bash -l {0}
        run: |
          conda install setuptools-rust
          conda install -c defaults anaconda-client conda-build

      - name: Extract version
+        shell: bash -l {0}
        working-directory: ./bindings/python
        run: echo "TOKENIZERS_VERSION=`python setup.py --version`" >> $GITHUB_ENV

      - name: Build conda packages
+        shell: bash -l {0}
        run: |
          conda info
          conda list
          conda-build .github/conda --python=${{ matrix.python }}

+          # - name: Upload to Anaconda
+          #   run: anaconda upload `conda-build .github/conda --output` --force
+          #
+  build_and_package_linux:
+    runs-on: ubuntu-18.04
+    container: quay.io/pypa/manylinux2014_x86_64
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [37, 38, 39]
+        include:
+          - python: 37
+            checksum: a1a7285dea0edc430b2bc7951d89bb30a2a1b32026d2a7b02aacaaa95cf69c7c 
+          - python: 38
+            checksum: 935d72deb16e42739d69644977290395561b7a6db059b316958d97939e9bdf3d 
+          - python: 39
+            checksum: 1ea2f885b4dbc3098662845560bc64271eb17085387a70c2ba3f29fff6f8d52f 
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install miniconda
+        run: |
+          yum install -y wget openssl-devel
+          export FILENAME=Miniconda3-py${{ matrix.python }}_4.10.3-Linux-x86_64.sh
+          wget https://repo.anaconda.com/miniconda/$FILENAME
+          sha256sum $FILENAME | awk '$1=="${{ matrix.checksum}}"{print"good to go"}'
+          bash $FILENAME -b -p $HOME/miniconda
+          source $HOME/miniconda/bin/activate
+
+      - name: Show glibc information
+        shell: bash -l {0}
+        run: ldd --version
+
+      - name: Conda info
+        shell: bash -l {0}
+        run: |
+          source $HOME/miniconda/bin/activate
+          conda info
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Setup conda env
+        shell: bash -l {0}
+        run: |
+          source $HOME/miniconda/bin/activate
+          conda install setuptools-rust
+          conda install -c defaults anaconda-client conda-build
+
+      - name: Extract version
+        shell: bash -l {0}
+        working-directory: ./bindings/python
+        run: |
+          source $HOME/miniconda/bin/activate
+          echo "TOKENIZERS_VERSION=`python setup.py --version`" >> $GITHUB_ENV
+
+      - name: Build conda packages
+        shell: bash -l {0}
+        run: |
+          source $HOME/miniconda/bin/activate
+          conda info
+          conda list
+          conda-build .github/conda --python=${{ matrix.python }}
+
      - name: Upload to Anaconda
        run: anaconda upload `conda-build .github/conda --output` --force
--- a/.github/workflows/python-release-extra.yml
+++ b/.github/workflows/python-release-extra.yml
@ -20,6 +20,9 @@ jobs:
    steps:
      - uses: actions/checkout@v1

+      - name: Upgrade libssl
+        run: sudo apt-get install -y libssl-dev
+
      - name: Pull images
        run: |
          docker pull multiarch/qemu-user-static
@ -34,13 +37,16 @@ jobs:
        run: |
          docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \
            --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_ppc64le \
-            /bin/bash -c "cd /io/bindings/python; sh build-wheels.sh"
+            /bin/bash -c "yum install -y openssl-devel && cd /io/bindings/python; sh build-wheels.sh"

  create_wheels_manylinux_2014_aarch64:
    runs-on: ubuntu-latest
    name: Create wheels for manylinux2014 - Aarch64
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+
+      - name: Upgrade libssl
+        run: sudo apt-get install -y libssl-dev

      - name: Pull images
        run: |
@ -56,13 +62,16 @@ jobs:
        run: |
          docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \
            --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_aarch64 \
-            /bin/bash -c "cd /io/bindings/python; sh build-wheels.sh"
+            /bin/bash -c "yum install -y openssl-devel && cd /io/bindings/python; sh build-wheels.sh"

  create_wheels_manylinux_2014_x390x:
    runs-on: ubuntu-latest
    name: Create wheels for manylinux2014 - S390X
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+
+      - name: Upgrade libssl
+        run: sudo apt-get install -y libssl-dev

      - name: Pull images
        run: |
@ -78,7 +87,7 @@ jobs:
        run: |
          docker run -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION -e DIST_DIR \
            --rm -v `pwd`/../..:/io quay.io/pypa/manylinux2014_s390x \
-            /bin/bash -c "cd /io/bindings/python; sh build-wheels.sh"
+            /bin/bash -c "yum install -y openssl-devel && cd /io/bindings/python; sh build-wheels.sh"

  upload_package:
    name: Upload package to PyPi
@ -89,7 +98,7 @@ jobs:
      - create_wheels_manylinux_2014_x390x

    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2

      - name: Install Python
        uses: actions/setup-python@v1
--- a/.github/workflows/python-release.yml
+++ b/.github/workflows/python-release.yml
@ -19,8 +19,12 @@ jobs:
    name: Create wheels for manylinux2010
    container: quay.io/pypa/manylinux2010_x86_64
    steps:
+      # v1 is required when using manylinux2010
      - uses: actions/checkout@v1

+      - name: Install dependencies
+        run: yum install -y openssl-devel
+
      - name: Build and audit wheels
        working-directory: ./bindings/python
        run: sh build-wheels.sh
@ -33,7 +37,7 @@ jobs:
        python: [3.7, 3.8, 3.9]
    steps:
      - name: Checkout repository
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2

      - name: Install Rust
        uses: actions-rs/toolchain@v1
@ -79,7 +83,7 @@ jobs:
        python: [3.7, 3.8, 3.9]
    steps:
      - name: Checkout repository
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2

      - name: Install Rust
        uses: actions-rs/toolchain@v1
--- a/bindings/python/CHANGELOG.md
+++ b/bindings/python/CHANGELOG.md
@ -6,10 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## [0.11.0]

+### Fixed
+
+- [#585] Conda version should now work on old CentOS
+- [#844] Fixing interaction between `is_pretokenized` and `trim_offsets`.
+- [#851] Doc links
+
 ### Added
 - [#657]: Add SplitDelimiterBehavior customization to Punctuation constructor
+- [#845]: Documentation for `Decoders`.

 ### Changed
+- [#850]: Added a feature gate to enable disabling `http` features
 - [#718]: Fix `WordLevel` tokenizer determinism during training
 - [#762]: Add a way to specify the unknown token in `SentencePieceUnigramTokenizer`
 - [#770]: Improved documentation for `UnigramTrainer`
@ -333,6 +341,11 @@ delimiter (Works like `.split(delimiter)`)
 - Fix a bug that was causing crashes in Python 3.5


+[#850]: https://github.com/huggingface/tokenizers/pull/850
+[#844]: https://github.com/huggingface/tokenizers/pull/844
+[#845]: https://github.com/huggingface/tokenizers/pull/845
+[#851]: https://github.com/huggingface/tokenizers/pull/851
+[#585]: https://github.com/huggingface/tokenizers/pull/585
 [#793]: https://github.com/huggingface/tokenizers/pull/793
 [#780]: https://github.com/huggingface/tokenizers/pull/780
 [#770]: https://github.com/huggingface/tokenizers/pull/770