mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add feature flag hint to README.md, fixes #1633 (#1709)

Browse Source
This commit is contained in:
sftse
2024-12-30 17:01:53 +01:00
committed by GitHub
parent 24d29f498d
commit 555d44c47a
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tokenizers/README.md
View File

@ -42,10 +42,11 @@ use tokenizers::tokenizer::{Result, Tokenizer};
fn main() -> Result<()> { fn main() -> Result<()> {
# #[cfg(feature = "http")] # #[cfg(feature = "http")]
# { # {
let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?; // needs http feature enabled
let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?;
let encoding = tokenizer.encode("Hey there!", false)?; let encoding = tokenizer.encode("Hey there!", false)?;
println!("{:?}", encoding.get_tokens()); println!("{:?}", encoding.get_tokens());
# } # }
Ok(()) Ok(())
} }