Python - Black auto formatting

This commit is contained in:
Anthony MOI
2020-02-18 10:45:36 -05:00
parent 4706151c32
commit 81be207819
16 changed files with 179 additions and 211 deletions

View File

@ -7,28 +7,33 @@ parser.add_argument("--vocab", default=None, type=str, required=True, help="The
parser.add_argument("--merges", default=None, type=str, required=True, help="The merges.txt file")
args = parser.parse_args()
class GoodCustom:
"""GoodCustom
This class represents a good custom PreTokenizer that will be called
by `tokenizers` when needed
"""
def pre_tokenize(self, sentence):
return sentence.split(" ")
def decode(self, tokens):
return ", ".join(tokens)
class BadCustom:
"""Bad Pretok
This class represents a bad custom PreTokenizer that will trigger an exception
when called by `tokenizers`
"""
def pre_tokenize(self, sentence):
return None
def decode(self, tokens):
return None
def tokenize(sentence):
output = tokenizer.encode(sentence).tokens
print(f"`{sentence}` tokenized to {output}")
@ -66,4 +71,3 @@ try:
encoding = tokenizer.encode("Hey friend!")
except:
print("Bad tokenizer didn't work")