mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing node doc location and rust README.

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-29 11:47:10 +02:00
committed by Anthony MOI
parent 44e8f4be8f
commit 128197b59a
2 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
docs/source/index.rst
View File

@ -50,8 +50,8 @@ Load an existing tokenizer:
.. group-tab:: Node
.. literalinclude:: ../../bindings/node/examples/load.test.ts
:language: typescript
.. literalinclude:: ../../bindings/node/examples/load.test.js
:language: javascript
:emphasize-lines: 11
@ -74,6 +74,6 @@ Small guide of :ref:`how to create a Tokenizer options<tokenizer_blocks>`.
.. group-tab:: Node
.. literalinclude:: ../../bindings/node/examples/train.test.ts
:language: typescript
.. literalinclude:: ../../bindings/node/examples/train.test.js
:language: javascript

14
tokenizers/README.md
View File

@ -87,21 +87,21 @@ fn main() -> Result<()> {
let tokenizer = TokenizerBuilder::new()
.with_model(BPE::default())
.with_normalizer(Some(Sequence::new(vec![
NormalizerWrapper::StripNormalizer(Strip::new(true, true)),
NormalizerWrapper::NFC(NFC),
Strip::new(true, true).into(),
NFC.into(),
])))
.with_pre_tokenizer(Some(PreTokenizerWrapper::ByteLevel(ByteLevel::default())))
.with_post_processor(Some(PostProcessorWrapper::ByteLevel(ByteLevel::default())))
.with_decoder(Some(DecoderWrapper::ByteLevel(ByteLevel::default())))
.with_pre_tokenizer(Some(ByteLevel::default()))
.with_post_processor(Some(ByteLevel::default()))
.with_decoder(Some(ByteLevel::default()))
.build()?;
let pretty = false;
tokenizer
.train(
&trainer,
vec!["path/to/vocab.txt".to_string()],
)?
.get_model()
.save(Path::new("result-folder"), Some("some-prefix"))?;
.save("tokenizer.json", pretty)?;
Ok(())
}