c-my/transformers
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated the GLUE data processor. Corrections to RoBERTa and XLNet.

Browse Source
This commit is contained in:
LysandreJik committed 2019-09-19 10:55:06 +02:00
1 parent c3df2136e1
commit bac332fec0
3 files changed
+4 -56
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pytorch_transformers/tokenization_roberta.py
+1 -1
View File
@@ -98,7 +98,7 @@ class RobertaTokenizer(GPT2Tokenizer):
if output_mask:
return (
cls + token_ids_0 + sep + sep + token_ids_1 + sep,
[0] * len(cls + token_ids_0 + sep) + [1] * len(sep + token_ids_1 + sep)
[0] * len(cls + token_ids_0 + sep + sep) + [1] * len(token_ids_1 + sep)
)
else:
return cls + token_ids_0 + sep + sep + token_ids_1 + sep