mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-12-03 19:28:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade neon version + tests in JS instead of TS.

Browse Source
This commit is contained in:
Nicolas Patry
2020-09-28 15:11:47 +02:00
committed by Anthony MOI
parent 81bb4f6da3
commit 6f8892e3ae
4 changed files with 1478 additions and 2771 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/node/examples/load.test.ts → bindings/node/examples/load.test.js
View File

@@ -1,7 +1,5 @@
/* eslint-disable @typescript-eslint/no-explicit-any */ /*eslint-disable no-undef*/
/* eslint-disable @typescript-eslint/no-empty-function */ const tokenizers = require("tokenizers");
const tokenizers = await import("tokenizers");
describe("loadExample", () => { describe("loadExample", () => {
it("", () => { it("", () => {

15
bindings/node/examples/train.test.ts → bindings/node/examples/train.test.js
View File

@@ -1,5 +1,4 @@
/* eslint-disable @typescript-eslint/no-explicit-any */ /*eslint-disable no-undef*/
/* eslint-disable @typescript-eslint/no-empty-function */
const { const {
Tokenizer, Tokenizer,
@@ -10,11 +9,11 @@ const {
decoders, decoders,
trainers, trainers,
AddedToken, AddedToken,
} = await import("tokenizers"); } = require("tokenizers");
describe("trainExample", () => { describe("trainExample", () => {
it("", () => { it("", () => {
const vocab_size = 100; const vocabSize = 100;
const tokenizer = new Tokenizer(models.BPE.empty()); const tokenizer = new Tokenizer(models.BPE.empty());
tokenizer.normalizer = normalizers.sequenceNormalizer([ tokenizer.normalizer = normalizers.sequenceNormalizer([
@@ -26,16 +25,16 @@ describe("trainExample", () => {
tokenizer.decoder = decoders.byteLevelDecoder(); tokenizer.decoder = decoders.byteLevelDecoder();
const trainer = trainers.bpeTrainer({ const trainer = trainers.bpeTrainer({
vocab_size, vocabSize,
min_frequency: 0, minFrequency: 0,
special_tokens: [ specialTokens: [
new AddedToken("<s>", true), new AddedToken("<s>", true),
new AddedToken("<pad>", true), new AddedToken("<pad>", true),
new AddedToken("</s>", true), new AddedToken("</s>", true),
new AddedToken("<unk>", true), new AddedToken("<unk>", true),
new AddedToken("<mask>", true), new AddedToken("<mask>", true),
], ],
show_progress: true, showProgress: false,
}); });
tokenizer.train(trainer, ["data/small.txt"]); tokenizer.train(trainer, ["data/small.txt"]);

4132
bindings/node/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

24
bindings/node/package.json
View File

@@ -15,24 +15,24 @@
"author": "Anthony MOI <m.anthony.moi@gmail.com>", "author": "Anthony MOI <m.anthony.moi@gmail.com>",
"license": "Apache-2.0", "license": "Apache-2.0",
"dependencies": { "dependencies": {
"@types/node": "^13.1.6", "@types/node": "^13.13.21",
"node-pre-gyp": "^0.14.0" "node-pre-gyp": "^0.14.0"
}, },
"devDependencies": { "devDependencies": {
"@types/jest": "^26.0.7", "@types/jest": "^26.0.14",
"@typescript-eslint/eslint-plugin": "^3.7.0", "@typescript-eslint/eslint-plugin": "^3.10.1",
"@typescript-eslint/parser": "^3.7.0", "@typescript-eslint/parser": "^3.10.1",
"eslint": "^7.5.0", "eslint": "^7.10.0",
"eslint-config-prettier": "^6.11.0", "eslint-config-prettier": "^6.12.0",
"eslint-plugin-jest": "^23.18.0", "eslint-plugin-jest": "^23.20.0",
"eslint-plugin-jsdoc": "^30.0.3", "eslint-plugin-jsdoc": "^30.6.1",
"eslint-plugin-prettier": "^3.1.4", "eslint-plugin-prettier": "^3.1.4",
"eslint-plugin-simple-import-sort": "^5.0.3", "eslint-plugin-simple-import-sort": "^5.0.3",
"jest": "^26.1.0", "jest": "^26.4.2",
"neon-cli": "^0.3.3", "neon-cli": "^0.4.2",
"prettier": "^2.0.5", "prettier": "^2.1.2",
"shelljs": "^0.8.3", "shelljs": "^0.8.3",
"ts-jest": "^26.1.3", "ts-jest": "^26.4.0",
"typescript": "^3.9.7" "typescript": "^3.9.7"
}, },
"engines": { "engines": {