mirror of
https://github.com/mii443/tokenizers.git
synced 2025-08-22 16:25:30 +00:00
Python - Pretty json saving defaults to true (#793)
* Python - Pretty json saving defaults to true * Update changelog
This commit is contained in:
@ -4,11 +4,14 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- [#657]: Add SplitDelimiterBehavior customization to Punctuation constructor
|
||||
|
||||
### Changed
|
||||
- [#793]: Saving a pretty JSON file by default when saving a tokenizer
|
||||
|
||||
## [0.10.3]
|
||||
|
||||
### Fixed
|
||||
@ -326,6 +329,7 @@ delimiter (Works like `.split(delimiter)`)
|
||||
- Fix a bug that was causing crashes in Python 3.5
|
||||
|
||||
|
||||
[#793]: https://github.com/huggingface/tokenizers/pull/793
|
||||
[#714]: https://github.com/huggingface/tokenizers/pull/714
|
||||
[#707]: https://github.com/huggingface/tokenizers/pull/707
|
||||
[#693]: https://github.com/huggingface/tokenizers/pull/693
|
||||
|
@ -1006,7 +1006,7 @@ class Tokenizer:
|
||||
The `optional` :class:`~tokenizers.pre_tokenizers.PreTokenizer` in use by the Tokenizer
|
||||
"""
|
||||
pass
|
||||
def save(self, pretty=False):
|
||||
def save(self, pretty=True):
|
||||
"""
|
||||
Save the :class:`~tokenizers.Tokenizer` to the file at the given path.
|
||||
|
||||
@ -1014,7 +1014,7 @@ class Tokenizer:
|
||||
path (:obj:`str`):
|
||||
A path to a file in which to save the serialized tokenizer.
|
||||
|
||||
pretty (:obj:`bool`, defaults to :obj:`False`):
|
||||
pretty (:obj:`bool`, defaults to :obj:`True`):
|
||||
Whether the JSON file should be pretty formatted.
|
||||
"""
|
||||
pass
|
||||
|
@ -326,7 +326,7 @@ class BaseTokenizer:
|
||||
"""
|
||||
return self._tokenizer.model.save(directory, prefix=prefix)
|
||||
|
||||
def save(self, path: str, pretty: bool = False):
|
||||
def save(self, path: str, pretty: bool = True):
|
||||
"""Save the current Tokenizer at the given path
|
||||
|
||||
Args:
|
||||
|
@ -601,10 +601,10 @@ impl PyTokenizer {
|
||||
/// path (:obj:`str`):
|
||||
/// A path to a file in which to save the serialized tokenizer.
|
||||
///
|
||||
/// pretty (:obj:`bool`, defaults to :obj:`False`):
|
||||
/// pretty (:obj:`bool`, defaults to :obj:`True`):
|
||||
/// Whether the JSON file should be pretty formatted.
|
||||
#[args(pretty = false)]
|
||||
#[text_signature = "(self, pretty=False)"]
|
||||
#[args(pretty = true)]
|
||||
#[text_signature = "(self, pretty=True)"]
|
||||
fn save(&self, path: &str, pretty: bool) -> PyResult<()> {
|
||||
ToPyResult(self.tokenizer.save(path, pretty)).into()
|
||||
}
|
||||
|
Reference in New Issue
Block a user