remove enforcement of non special when adding tokens (#1521)

* remove enforcement of non special when adding tokens

* mut no longer needed

* add a small test

* nit

* style

* audit

* ignore cargo audit's own vulnerability

* update

* revert

* remove CVE

.github/workflows/rust.yml

@@ -81,6 +81,9 @@ jobs:
           command: test
           args: --verbose --manifest-path ./tokenizers/Cargo.toml --doc
 
+      - name: Install cargo-audit
+        run: cargo install cargo-audit
+
       - name: Run Audit
         uses: actions-rs/cargo@v1
         with: