c-my/transformers
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sentence -> Sequence. Removed output_mask from the special token addition methods.

Browse Source
This commit is contained in:
LysandreJik committed 2019-09-19 10:55:06 +02:00
1 parent 8cba057260
commit bf503158c5
13 files changed
+49 -76
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pytorch_transformers/tests/tokenization_xlnet_test.py
+2 -2
View File
@@ -95,8 +95,8 @@ class XLNetTokenizationTest(CommonTestCases.CommonTokenizerTester):
text = tokenizer.encode("sequence builders")
text_2 = tokenizer.encode("multi-sequence build")
encoded_sentence = tokenizer.add_special_tokens_single_sentence(text)
encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2)
encoded_sentence = tokenizer.add_special_tokens_single_sequence(text)
encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2)
assert encoded_sentence == text + [4, 3]
assert encoded_pair == text + [4] + text_2 + [4, 3]