fix tokenizers serialization

This commit is contained in:
thomwolf committed 2019-08-30 17:36:11 +02:00
1 parent cd65c41a83
commit 7044ed6b05
4 files changed
+7 -6

No files matched your search

@@ -95,7 +95,8 @@ class TransfoXLTokenizer(PreTrainedTokenizer):
# in a library like ours, at all.
vocab_dict = torch.load(pretrained_vocab_file)
for key, value in vocab_dict.items():
self.__dict__[key] = value
if key not in self.__dict__:
self.__dict__[key] = value
if vocab_file is not None:
self.build_vocab()