mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Preparing release. (#1355)

Browse Source 
* Preparing release.

* Fix new clippy
This commit is contained in:
Nicolas Patry
2023-10-06 12:56:36 +02:00
committed by GitHub
parent aed491df8c
commit 4322056e6e
7 changed files with 27 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/node/Cargo.toml
View File

@ -2,7 +2,7 @@
authors = ["Nicolas Patry <nicolas@huggingface.co>"]
edition = "2021"
name = "node"
version = "0.14.1-dev.0"
version = "0.14.2-dev.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

7
bindings/node/index.d.ts vendored
View File

@ -175,7 +175,7 @@ export class Encoding {
getSequenceIds(): Array<number | undefined | null>
tokenToSequence(token: number): number | null
}
export class Model { }
export class Model {}
export type Bpe = BPE
export class BPE {
static empty(): Model
@ -204,7 +204,7 @@ export class Normalizer {
export class PreTokenizer {
preTokenizeString(sequence: string): [string, [number, number]][]
}
export class Processor { }
export class Processor {}
export class AddedToken {
constructor(token: string, isSpecial: boolean, options?: AddedTokenOptions | undefined | null)
getContent(): string
@ -229,7 +229,6 @@ export class Tokenizer {
decodeBatch(ids: Array<Array<number>>, skipSpecialTokens: boolean): Promise<string[]>
static fromString(s: string): Tokenizer
static fromFile(file: string): Tokenizer
// static fromPretrained(file: string, parameters?: JsFromPretrainedParameters | undefined | null): Tokenizer
addSpecialTokens(tokens: Array<string>): void
setTruncation(maxLength: number, options?: TruncationOptions | undefined | null): void
disableTruncation(): void
@ -251,4 +250,4 @@ export class Tokenizer {
addSpecialTokens?: boolean | undefined | null,
): Encoding
}
export class Trainer { }
export class Trainer {}