From 4518b0f7f2031ff78fd604d37d07f3cb1cd1b73b Mon Sep 17 00:00:00 2001 From: Chris Ha Date: Thu, 11 May 2023 00:04:34 +0900 Subject: [PATCH] fix unigram.rs test_sample() (#1244) https://github.com/huggingface/tokenizers/blob/87230bb59b2a70b6c361943d1394f42218b0040b/tokenizers/tests/unigram.rs#LL71C1-L71C53 When running cargo test --release, the above line causes an error. referring to https://github.com/huggingface/tokenizers/blob/87230bb59b2a70b6c361943d1394f42218b0040b/tokenizers/src/models/unigram/lattice.rs#L138 It seems that lattice::from should only take 3 arguments. If i had to guess, it should be Lattice::from("ABC", 0, 2); This change makes cargo test --release pass without error. --- tokenizers/tests/unigram.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tokenizers/tests/unigram.rs b/tokenizers/tests/unigram.rs index b0d90679..bccd0bca 100644 --- a/tokenizers/tests/unigram.rs +++ b/tokenizers/tests/unigram.rs @@ -68,7 +68,7 @@ fn test_train_unigram_from_file() { #[cfg(not(debug_assertions))] #[test] fn test_sample() { - let mut lattice = Lattice::from("ABC", 0, 1, 2); + let mut lattice = Lattice::from("ABC", 0, 2); lattice.insert(0, 1, 1.0, 3); // A lattice.insert(1, 1, 1.2, 4); // B lattice.insert(2, 1, 1.5, 5); // C