Added binary masking tests

This commit is contained in:
LysandreJik committed 2019-09-19 10:55:06 +02:00
1 parent e391d4735e
commit c3df2136e1
2 files changed
+14

No files matched your search

@@ -690,6 +690,8 @@ class PreTrainedTokenizer(object):
if add_special_tokens:
return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens, output_mask)
else:
if output_mask:
logger.warning("Can't output mask if no special tokens are involved. Please call the method with add_special_tokens set to True.")
return first_sentence_tokens, second_sentence_tokens
def add_special_tokens_single_sentence(self, token_ids):