mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Wrap rustdoc html entity in code block (#1130)

Browse Source
This commit is contained in:
Roy Hvaara
2022-12-23 23:30:45 +01:00
committed by GitHub
parent 03ce27d2fa
commit 3e7476de86
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tokenizers/src/decoders/ctc.rs
View File

@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize};
pub struct CTC {
/// The pad token used by CTC to delimit a new token.
pub pad_token: String,
/// The word delimiter token. It will be replaced by a <space>
/// The word delimiter token. It will be replaced by a `<space>`.
pub word_delimiter_token: String,
/// Whether to cleanup some tokenization artifacts.
/// Mainly spaces before punctuation, and some abbreviated english forms.