mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix a clerical error in the comment (#1356)

Browse Source
This commit is contained in:
天地
2023-10-11 03:31:44 +08:00
committed by GitHub
parent 4322056e6e
commit 0d8c57da48
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/models/bpe/model.rs
View File

@ -117,7 +117,7 @@ impl BpeBuilder {
self self
} }
/// Set the `fuse_unk` option. /// Set the `byte_fallback` option.
#[must_use] #[must_use]
pub fn byte_fallback(mut self, byte_fallback: bool) -> Self { pub fn byte_fallback(mut self, byte_fallback: bool) -> Self {
self.config.byte_fallback = byte_fallback; self.config.byte_fallback = byte_fallback;