mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-03 03:08:21 +00:00
@@ -42,6 +42,7 @@ use tokenizers::tokenizer::{Result, Tokenizer};
|
||||
fn main() -> Result<()> {
|
||||
# #[cfg(feature = "http")]
|
||||
# {
|
||||
// needs http feature enabled
|
||||
let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?;
|
||||
|
||||
let encoding = tokenizer.encode("Hey there!", false)?;
|
||||
|
||||
Reference in New Issue
Block a user