added tokenizers serialization tests

This commit is contained in:
thomwolf committed 2019-04-15 12:03:56 +02:00
1 parent 3e65f255dc
commit 870b734bfd
7 files changed
+51 -32

No files matched your search

@@ -148,6 +148,7 @@ class TransfoXLTokenizer(object):
index = 0
vocab_file = os.path.join(vocab_path, VOCAB_NAME)
torch.save(self.__dict__, vocab_file)
return vocab_file
def build_vocab(self):
if self.vocab_file: