c-my/transformers
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sentence -> Sequence. Removed output_mask from the special token addition methods.

Browse Source
This commit is contained in:
LysandreJik committed 2019-09-19 10:55:06 +02:00
1 parent 8cba057260
commit bf503158c5
13 files changed
+49 -76
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pytorch_transformers/tests/tokenization_roberta_test.py
+2 -2
View File
@@ -87,8 +87,8 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester):
encoded_text_from_decode = tokenizer.encode("sequence builders", add_special_tokens=True)
encoded_pair_from_decode = tokenizer.encode("sequence builders", "multi-sequence build", add_special_tokens=True)
encoded_sentence = tokenizer.add_special_tokens_single_sentence(text)
encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2)
encoded_sentence = tokenizer.add_special_tokens_single_sequence(text)
encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2)
assert encoded_sentence == encoded_text_from_decode
assert encoded_pair == encoded_pair_from_decode