Node - Fix encoding pad

This commit is contained in:
Pierric Cistac
2020-06-15 17:59:54 -04:00
parent c3ec7b1544
commit 252b84a100
2 changed files with 29 additions and 1 deletions

View File

@@ -1,5 +1,6 @@
import { promisify } from "util";
import { PaddingDirection } from "./enums";
import { Model, WordPiece, WordPieceOptions } from "./models";
import { whitespacePreTokenizer } from "./pre-tokenizers";
import { RawEncoding } from "./raw-encoding";
@@ -137,4 +138,31 @@ describe("RawEncoding", () => {
expect(index).toBeUndefined();
});
});
describe("pad", () => {
it("works correctly with only one parameter", () => {
encoding.pad(10);
expect(encoding.getTokens()).toHaveLength(10);
});
it("accepts `undefined` as second parameter", () => {
encoding.pad(10, undefined);
expect(encoding.getTokens()).toHaveLength(10);
});
it("accepts options as second parameter", () => {
encoding.pad(10, {
direction: PaddingDirection.Left,
padToken: "[PA]",
padTypeId: 10,
padId: 400
});
const tokens = encoding.getTokens();
expect(tokens).toHaveLength(10);
expect(tokens[0]).toBe("[PA]");
expect(encoding.getTypeIds()[0]).toBe(10);
expect(encoding.getIds()[0]).toBe(400);
});
});
});

View File

@@ -257,7 +257,7 @@ declare_types! {
// padToken?: string = "[PAD]"
// }
let length = cx.extract::<usize>(0)?;
let params = cx.extract_opt::<PaddingParams>(0)?
let params = cx.extract_opt::<PaddingParams>(1)?
.map_or_else(tk::PaddingParams::default, |p| p.0);
let mut this = cx.this();