Python - Pretty json saving defaults to true (#793)

* Python - Pretty json saving defaults to true

* Update changelog
This commit is contained in:
Anthony MOI
2021-09-02 14:43:54 +02:00
committed by GitHub
parent 23cf8c69ae
commit b8b584d4e5
4 changed files with 11 additions and 7 deletions

View File

@ -4,11 +4,14 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
## [Unreleased]
### Added
- [#657]: Add SplitDelimiterBehavior customization to Punctuation constructor
### Changed
- [#793]: Saving a pretty JSON file by default when saving a tokenizer
## [0.10.3]
### Fixed
@ -326,6 +329,7 @@ delimiter (Works like `.split(delimiter)`)
- Fix a bug that was causing crashes in Python 3.5
[#793]: https://github.com/huggingface/tokenizers/pull/793
[#714]: https://github.com/huggingface/tokenizers/pull/714
[#707]: https://github.com/huggingface/tokenizers/pull/707
[#693]: https://github.com/huggingface/tokenizers/pull/693

View File

@ -1006,7 +1006,7 @@ class Tokenizer:
The `optional` :class:`~tokenizers.pre_tokenizers.PreTokenizer` in use by the Tokenizer
"""
pass
def save(self, pretty=False):
def save(self, pretty=True):
"""
Save the :class:`~tokenizers.Tokenizer` to the file at the given path.
@ -1014,7 +1014,7 @@ class Tokenizer:
path (:obj:`str`):
A path to a file in which to save the serialized tokenizer.
pretty (:obj:`bool`, defaults to :obj:`False`):
pretty (:obj:`bool`, defaults to :obj:`True`):
Whether the JSON file should be pretty formatted.
"""
pass

View File

@ -326,7 +326,7 @@ class BaseTokenizer:
"""
return self._tokenizer.model.save(directory, prefix=prefix)
def save(self, path: str, pretty: bool = False):
def save(self, path: str, pretty: bool = True):
"""Save the current Tokenizer at the given path
Args:

View File

@ -601,10 +601,10 @@ impl PyTokenizer {
/// path (:obj:`str`):
/// A path to a file in which to save the serialized tokenizer.
///
/// pretty (:obj:`bool`, defaults to :obj:`False`):
/// pretty (:obj:`bool`, defaults to :obj:`True`):
/// Whether the JSON file should be pretty formatted.
#[args(pretty = false)]
#[text_signature = "(self, pretty=False)"]
#[args(pretty = true)]
#[text_signature = "(self, pretty=True)"]
fn save(&self, path: &str, pretty: bool) -> PyResult<()> {
ToPyResult(self.tokenizer.save(path, pretty)).into()
}