mirror of
https://github.com/mii443/tokenizers.git
synced 2025-12-07 13:18:31 +00:00
Rust | Python | Node - Also add char_to_word
This commit is contained in:
@@ -131,6 +131,18 @@ class Encoding:
|
||||
The index of the token that contains this char
|
||||
"""
|
||||
pass
|
||||
def char_to_word(self, pos: int) -> Optional[int]:
|
||||
"""
|
||||
Get the word that contains the given char.
|
||||
|
||||
Args:
|
||||
pos: int:
|
||||
The position of a char in the input string
|
||||
|
||||
Returns:
|
||||
The index of the word that contains this char
|
||||
"""
|
||||
pass
|
||||
def pad(
|
||||
self,
|
||||
length: int,
|
||||
|
||||
Reference in New Issue
Block a user