mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
CD backports (#1318)
* CD backports follow huggingface/safetensors#317 * fix node bindings? `cargo check` doesnt work on my local configuration from `tokenizers/bindings/node/native` i don't think it will be a problem but i have difficulty telling * backport #315 * safetensors#317 back ports
This commit is contained in:
59
.github/workflows/python-release.yml
vendored
59
.github/workflows/python-release.yml
vendored
@ -73,13 +73,13 @@ jobs:
|
||||
|
||||
|
||||
create_wheels_others_64bit:
|
||||
name: Create wheels for other OSes
|
||||
name: Other OSes
|
||||
env:
|
||||
MACOSX_DEPLOYMENT_TARGET: 10.11
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [windows-latest, macos-10.15]
|
||||
os: [windows-latest, macos-latest]
|
||||
python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
@ -109,7 +109,49 @@ jobs:
|
||||
- name: Rename wheels
|
||||
shell: bash
|
||||
working-directory: ./bindings/python/dist
|
||||
run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} ; done
|
||||
run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done
|
||||
|
||||
- name: Upload wheels
|
||||
shell: bash
|
||||
run: |
|
||||
pip install awscli
|
||||
aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"
|
||||
|
||||
create_wheels_macos_13:
|
||||
name: MacOS 13
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [macos-13]
|
||||
python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Install Rust
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
override: true
|
||||
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
architecture: x64
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# On old versions of python there is an old version of setuptools already installed
|
||||
pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall
|
||||
- name: Build wheel
|
||||
working-directory: ./bindings/python
|
||||
run: python setup.py bdist_wheel
|
||||
|
||||
- name: Rename wheels
|
||||
shell: bash
|
||||
working-directory: ./bindings/python/dist
|
||||
run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done
|
||||
|
||||
- name: Upload wheels
|
||||
shell: bash
|
||||
@ -122,10 +164,11 @@ jobs:
|
||||
runs-on: macos-arm64
|
||||
strategy:
|
||||
matrix:
|
||||
python: ["3.8.16", "3.9.13", "3.10.6", "3.11.1"]
|
||||
python: [ "3.9.13", "3.10.6", "3.11.1"]
|
||||
target: ["12.0", "13.0"]
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Install Rust
|
||||
uses: actions-rs/toolchain@v1
|
||||
@ -145,17 +188,17 @@ jobs:
|
||||
which python
|
||||
pip install -U setuptools wheel setuptools-rust awscli
|
||||
cd ./bindings/python
|
||||
python setup.py bdist_wheel
|
||||
MACOSX_DEPLOYMENT_TARGET=${{ matrix.target }} python setup.py bdist_wheel
|
||||
cd ../../
|
||||
aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"
|
||||
|
||||
upload_package:
|
||||
name: Upload package to PyPi
|
||||
runs-on: ubuntu-latest
|
||||
needs: [create_wheels_manylinux, create_wheels_windows_32bit, create_wheels_others_64bit, create_wheels_macos_arm64]
|
||||
needs: [create_wheels_manylinux, create_wheels_windows_32bit, create_wheels_others_64bit, create_wheels_macos_arm64, create_wheels_macos_13]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@v1
|
||||
|
@ -701,7 +701,7 @@ declare_types! {
|
||||
let params_obj = neon_serde::to_value(&mut cx, &TruncationParams(options.clone()))?;
|
||||
let mut this = cx.this();
|
||||
let guard = cx.lock();
|
||||
this.borrow_mut(&guard)
|
||||
let _ = this.borrow_mut(&guard)
|
||||
.tokenizer.write().unwrap()
|
||||
.with_truncation(Some(options));
|
||||
|
||||
|
Reference in New Issue
Block a user