Add WordPiece decoder for Bert

tokenizers/src/decoders/mod.rs

@@ -1,2 +1,3 @@
 // Re-export this as a decoder
 pub use super::pre_tokenizers::byte_level;
+pub mod wordpiece;

tokenizers/src/decoders/wordpiece.rs

@@ -0,0 +1,9 @@
+use crate::tokenizer::Decoder;
+
+pub struct WordPiece;
+
+impl Decoder for WordPiece {
+    fn decode(&self, tokens: Vec<String>) -> String {
+        tokens.join(" ").replace(" ##", "")
+    }
+}