Fixes and cleanup, suggestions by @n1t0.

This commit is contained in:
Anthony MOI
2020-08-04 09:56:36 +02:00
committed by Anthony MOI
parent f6adcf0e7c
commit 363adedb4c
14 changed files with 51 additions and 155 deletions

View File

@@ -301,7 +301,7 @@ mod test {
match py_dec.decoder {
PyDecoderWrapper::Wrapped(msp) => match msp.as_ref() {
DecoderWrapper::Metaspace(_) => {}
_ => panic!("Expected Whitespace"),
_ => panic!("Expected Metaspace"),
},
_ => panic!("Expected wrapped, not custom."),
}

View File

@@ -1,4 +1,3 @@
use std::collections::hash_map::RandomState;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -10,11 +9,11 @@ use serde::{Deserialize, Serialize};
use tk::models::bpe::BPE;
use tk::models::wordlevel::WordLevel;
use tk::models::wordpiece::WordPiece;
use tk::models::ModelWrapper;
use tk::{Model, Token};
use tokenizers as tk;
use super::error::ToPyResult;
use tk::models::ModelWrapper;
/// A Model represents some tokenization algorithm like BPE or Word
/// This class cannot be constructed directly. Please use one of the concrete models.
@@ -55,7 +54,7 @@ impl Model for PyModel {
self.model.id_to_token(id)
}
fn get_vocab(&self) -> &HashMap<String, u32, RandomState> {
fn get_vocab(&self) -> &HashMap<String, u32> {
self.model.get_vocab()
}