From ffaee13994e4c1e8bedb8985b567ba510010edeb Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 28 Feb 2022 10:20:49 +0100 Subject: [PATCH] Preparing for 0.11.6 release. --- bindings/node/package.json | 2 +- bindings/python/CHANGELOG.md | 7 +++++++ bindings/python/py_src/tokenizers/__init__.py | 2 +- bindings/python/setup.py | 2 +- tokenizers/CHANGELOG.md | 7 +++++++ tokenizers/Cargo.toml | 2 +- 6 files changed, 18 insertions(+), 4 deletions(-) diff --git a/bindings/node/package.json b/bindings/node/package.json index 49318bb2..ee328638 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "tokenizers", - "version": "0.8.2", + "version": "0.8.3", "description": "", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/bindings/python/CHANGELOG.md b/bindings/python/CHANGELOG.md index 6cd682ab..7e45f94d 100644 --- a/bindings/python/CHANGELOG.md +++ b/bindings/python/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.11.6] + +- [#919] Fixing single_word AddedToken. (regression from 0.11.2) +- [#916] Deserializing faster `added_tokens` by loading them in batch. + ## [0.11.5] - [#895] Build `python 3.10` wheels. @@ -355,6 +360,8 @@ delimiter (Works like `.split(delimiter)`) - Fix a bug that was causing crashes in Python 3.5 +[#919]: https://github.com/huggingface/tokenizers/pull/919 +[#916]: https://github.com/huggingface/tokenizers/pull/916 [#895]: https://github.com/huggingface/tokenizers/pull/895 [#884]: https://github.com/huggingface/tokenizers/pull/884 [#882]: https://github.com/huggingface/tokenizers/pull/882 diff --git a/bindings/python/py_src/tokenizers/__init__.py b/bindings/python/py_src/tokenizers/__init__.py index 27b1b313..81b9084f 100644 --- a/bindings/python/py_src/tokenizers/__init__.py +++ b/bindings/python/py_src/tokenizers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.11.0" +__version__ = "0.11.6" from typing import Tuple, Union, Tuple, List from enum import Enum diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 59e8d1cf..208ceb0d 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -7,7 +7,7 @@ extras["docs"] = ["sphinx", "sphinx_rtd_theme", "setuptools_rust"] setup( name="tokenizers", - version="0.11.5", + version="0.11.6", description="Fast and Customizable Tokenizers", long_description=open("README.md", "r", encoding="utf-8").read(), long_description_content_type="text/markdown", diff --git a/tokenizers/CHANGELOG.md b/tokenizers/CHANGELOG.md index f11ff890..19ea5430 100644 --- a/tokenizers/CHANGELOG.md +++ b/tokenizers/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.11.3] + +- [#919] Fixing single_word AddedToken. (regression from 0.11.2) +- [#916] Deserializing faster `added_tokens` by loading them in batch. + ## [0.11.2] - [#884] Fixing bad deserialization following inclusion of a default for Punctuation @@ -135,6 +140,8 @@ advised, but that's not the question) split up in multiple bytes - [#174]: The `LongestFirst` truncation strategy had a bug +[#919]: https://github.com/huggingface/tokenizers/pull/919 +[#916]: https://github.com/huggingface/tokenizers/pull/916 [#884]: https://github.com/huggingface/tokenizers/pull/884 [#882]: https://github.com/huggingface/tokenizers/pull/882 [#868]: https://github.com/huggingface/tokenizers/pull/868 diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml index 1ca9ba0b..7953f368 100644 --- a/tokenizers/Cargo.toml +++ b/tokenizers/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Anthony MOI "] edition = "2018" name = "tokenizers" -version = "0.11.2" +version = "0.11.3" homepage = "https://github.com/huggingface/tokenizers" repository = "https://github.com/huggingface/tokenizers" documentation = "https://docs.rs/tokenizers/"