added test and debug tokenizer configuration serialization

This commit is contained in:
thomwolf committed 2019-08-30 17:09:36 +02:00
1 parent 88111de07c
commit 69da972ace
9 files changed
+32 -18

No files matched your search

@@ -41,8 +41,8 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester):
with open(self.vocab_file, "w", encoding='utf-8') as vocab_writer:
vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
def get_tokenizer(self):
return BertTokenizer.from_pretrained(self.tmpdirname)
def get_tokenizer(self, **kwargs):
return BertTokenizer.from_pretrained(self.tmpdirname, **kwargs)
def get_input_output_texts(self):
input_text = u"UNwant\u00E9d,running"