mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 08:15:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing the README. (#1714)

Browse Source
This commit is contained in:
Nicolas Patry
2025-01-08 12:31:17 +01:00
committed by GitHub
parent 3a6504d274
commit 166edd87c8
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tokenizers/src/lib.rs
View File

@ -30,10 +30,11 @@
//! fn main() -> Result<()> {
//! # #[cfg(feature = "http")]
//! # {
//! let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?;
//! // needs http feature enabled
//! let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?;
//!
//! let encoding = tokenizer.encode("Hey there!", false)?;
//! println!("{:?}", encoding.get_tokens());
//! let encoding = tokenizer.encode("Hey there!", false)?;
//! println!("{:?}", encoding.get_tokens());
//! # }
//! Ok(())
//! }