Python - Test Models

bindings/python/src/models.rs

@@ -288,4 +288,11 @@ impl WordLevel {
             }),
         }
     }
+
+    #[staticmethod]
+    fn empty() -> Model {
+        Model {
+            model: Container::Owned(Box::new(tk::models::wordlevel::WordLevel::default())),
+        }
+    }
 }

bindings/python/tests/bindings/test_models.py

@@ -0,0 +1,23 @@
+from ..utils import data_dir, roberta_files, bert_files
+
+from tokenizers.models import Model, BPE, WordPiece, WordLevel
+
+
+class TestBPE:
+    def test_instantiate(self, roberta_files):
+        assert isinstance(BPE.empty(), Model)
+        assert isinstance(BPE.from_files(roberta_files["vocab"], roberta_files["merges"]), Model)
+
+
+class TestWordPiece:
+    def test_instantiate(self, bert_files):
+        assert isinstance(WordPiece.empty(), Model)
+        assert isinstance(WordPiece.from_files(bert_files["vocab"]), Model)
+
+
+class TestWordLevel:
+    def test_instantiate(self, roberta_files):
+        assert isinstance(WordLevel.empty(), Model)
+        # The WordLevel model expects a vocab.json using the same format as roberta
+        # so we can just try to load with this file
+        assert isinstance(WordLevel.from_files(roberta_files["vocab"]), Model)

bindings/python/tests/utils.py

@@ -35,3 +35,12 @@ def roberta_files(data_dir):
             "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt"
         ),
     }
+
+
+@pytest.fixture(scope="session")
+def bert_files(data_dir):
+    return {
+        "vocab": download(
+            "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt"
+        ),
+    }

bindings/python/tokenizers/models/__init__.pyi

@@ -158,3 +158,7 @@ class WordLevel(Model):
                 The unknown token to be used by the model.
         """
         pass
+    @staticmethod
+    def empty() -> Model:
+        """ Instantiate an empty WordLevel Model. """
+        pass