preparing for first release
This commit is contained in:
1 parent
8513741b57
commit
1de35b624b
21 files changed
+3045
-958
No files matched your search
@@ -34,7 +34,7 @@ class TokenizationTest(unittest.TestCase):
|
||||
|
||||
vocab_file = vocab_writer.name
|
||||
|
||||
tokenizer = tokenization.FullTokenizer(vocab_file)
|
||||
tokenizer = tokenization.BertTokenizer(vocab_file)
|
||||
os.remove(vocab_file)
|
||||
|
||||
tokens = tokenizer.tokenize(u"UNwant\u00E9d,running")
|
||||
|
||||
Reference in new issue
Block a user