mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix error display

Browse Source
This commit is contained in:
epwalsh
2019-12-12 07:02:46 -08:00
committed by MOI Anthony
parent 3bdb849bb3
commit 13df36ca55
1 changed files with 11 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tokenizers/src/models/wordpiece/mod.rs
View File

@ -15,7 +15,7 @@ impl std::error::Error for Error {}
impl fmt::Display for Error { impl fmt::Display for Error {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
match self { match self {
MissingUnkToken => write!( Error::MissingUnkToken => write!(
fmt, fmt,
"WordPiece error: Missing [UNK] token from the vocabulary" "WordPiece error: Missing [UNK] token from the vocabulary"
), ),
@ -158,3 +158,13 @@ impl Model for WordPiece {
self.vocab_r.get(&id).map(|token| token.clone()) self.vocab_r.get(&id).map(|token| token.clone())
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_error_display() {
assert!(format!("{}", Error::MissingUnkToken).contains("Missing [UNK] token"));
}
}