mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 00:35:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added ability to inspect a 'Sequence' pre-tokenizer. (#1341)

Browse Source 
* Added ability to inspect a 'Sequence' pre-tokenizer.

* Added ability to inspect a 'Sequence' pre-tokenizer.

* Added ability to inspect a 'Sequence' pre-tokenizer.

* Linting error.

* Fix.

* Revert rename,
This commit is contained in:
Anthony Platanios
2023-09-20 23:10:16 -07:00
committed by GitHub
parent 2c565e42c7
commit 18bd5e8f9d
1 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tokenizers/src/pre_tokenizers/sequence.rs
View File

@ -13,6 +13,14 @@ impl Sequence {
pub fn new(pretokenizers: Vec<PreTokenizerWrapper>) -> Self { pub fn new(pretokenizers: Vec<PreTokenizerWrapper>) -> Self {
Self { pretokenizers } Self { pretokenizers }
} }
pub fn get_pre_tokenizers(&self) -> &[PreTokenizerWrapper] {
&self.pretokenizers
}
pub fn get_pre_tokenizers_mut(&mut self) -> &mut [PreTokenizerWrapper] {
&mut self.pretokenizers
}
} }
impl PreTokenizer for Sequence { impl PreTokenizer for Sequence {