update tokenizer - update squad example for xlnet

This commit is contained in:
thomwolf committed 2019-07-15 17:30:42 +02:00
1 parent 3b469cb422
commit 15d8b1266c
20 files changed
+191 -131

No files matched your search

@@ -229,9 +229,9 @@ class TransfoXLTokenizer(PreTrainedTokenizer):
else:
raise ValueError('Token not in vocabulary and no <unk> token in vocabulary for replacement')
def _convert_ids_to_string(self, tokens_ids):
"""Converts a sequence of ids in a string."""
out_string = ' '.join(tokens_ids).strip()
def convert_tokens_to_string(self, tokens):
""" Converts a sequence of tokens (string) in a single string. """
out_string = ' '.join(tokens).strip()
return out_string
def convert_to_tensor(self, symbols):