mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-23 16:49:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

words -> word_ids & sequences -> sequence_ids

Browse Source
This commit is contained in:
Anthony MOI
2020-11-09 15:14:14 -05:00
committed by Anthony MOI
parent 57d162b269
commit d3d9f2c76b
11 changed files with 70 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/node/lib/implementations/encoding.ts
View File

@ -149,7 +149,7 @@ export class Encoding {
return this._wordIndexes;
}
return (this._wordIndexes = this._rawEncoding.getWords());
return (this._wordIndexes = this._rawEncoding.getWordIds());
}
get sequenceIndexes(): (number | undefined)[] {
@ -157,7 +157,7 @@ export class Encoding {
return this._sequenceIndexes;
}
return (this._sequenceIndexes = this._rawEncoding.getSequences());
return (this._sequenceIndexes = this._rawEncoding.getSequenceIds());
}
/**