c-my/transformers
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Appends space before special token

Browse Source
This commit is contained in:
LysandreJik committed 2019-09-01 21:07:00 -04:00
1 parent fede4ef45d
commit 31d3373bc9
1 file changed
+1 -1
Download Patch File Download Diff File Expand all files Collapse all files
pytorch_transformers/tokenization_utils.py
+1 -1
View File
@@ -760,7 +760,7 @@ class PreTrainedTokenizer(object):
if current_sub_text:
sub_texts.append(self.convert_tokens_to_string(current_sub_text))
current_sub_text = []
sub_texts.append(token)
sub_texts.append(" " + token)
else:
current_sub_text.append(token)
if current_sub_text: