mii/tokenizers
1
0
Fork 0
mirror of https://github.com/mii443/tokenizers.git synced 2025-08-22 16:25:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

add more failing tests w/ last stable rust version

Browse Source
This commit is contained in:
Pierric Cistac
2020-01-21 14:08:54 -05:00
parent 40f022ccb2
commit d056fe4104
2 changed files with 35 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
bindings/node/lib/bindings/tokenizer.test.ts
View File

@ -16,23 +16,43 @@ import { Tokenizer } from "./tokenizer";
// const TokenizerMock = mocked(Tokenizer);
let tokenizer: Tokenizer;
beforeEach(() => {
// Clear all instances and calls to constructor and all methods:
// TokenizerMock.mockClear();
const model = BPE.empty();
tokenizer = new Tokenizer(model);
tokenizer.addTokens(["my", "name", "is", "john", "pair"]);
});
describe("Tokenizer", () => {
describe("addTokens", () => {
it("accepts a list of string as new tokens when initial model is empty", () => {
const model = BPE.empty();
const tokenizer = new Tokenizer(model);
const nbAdd = tokenizer.addTokens(["my", "name", "is", "john", "pair"]);
expect(nbAdd).toBe(5);
});
});
describe("encode", () => {
let tokenizer: Tokenizer;
beforeEach(() => {
// Clear all instances and calls to constructor and all methods:
// TokenizerMock.mockClear();
const model = BPE.empty();
tokenizer = new Tokenizer(model);
tokenizer.addTokens(["my", "name", "is", "john", "pair"]);
});
it("is a function w/ parameters", async () => {
expect(typeof tokenizer.encode).toBe("function");
});
it("accepts a pair of strings as parameters", async () => {
const encode = promisify(tokenizer.encode.bind(tokenizer));
await encode("my name is john", null);
await encode("my name is john", "pair");
const encoding = await encode("my name is john", "pair");
expect(encoding).toBeDefined();
});
it("accepts a string with a null pair", async () => {
const encode = promisify(tokenizer.encode.bind(tokenizer));
const encoding = await encode("my name is john", null);
expect(encoding).toBeDefined();
});
it("returns an Encoding", async () => {

6
bindings/node/native/Cargo.lock generated
View File

@ -256,7 +256,7 @@ dependencies = [
[[package]]
name = "num_cpus"
version = "1.11.1"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -329,7 +329,7 @@ dependencies = [
"crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -522,7 +522,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum neon-build 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ae406bf1065c4399e69d328a3bd8d4f088f2a205dc3881bf68c0ac775bfef337"
"checksum neon-runtime 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8465ac4ed3f340dead85e053b75a5f639f48ac6343b3523eff90a751758eead"
"checksum neon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ae4cf3871ca5a395077e68144c1754e94e9e1e3329e7f8399d999ca573ed89a"
"checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
"checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6"
"checksum number_prefix 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
"checksum rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"