update function to add special tokens

Since I started my PR the `add_special_token_single_sequence` function
has been deprecated for another; I replaced it with the new function.
This commit is contained in:
Rémi Louf authored and Julien Chaumond committed 2019-12-09 20:37:55 -05:00
1 parent 5c877fe94a
commit 07f4cd73f6
1 file changed
+2 -2
+2 -2
View File
@@ -139,11 +139,11 @@ def encode_for_summarization(story_lines, summary_lines, tokenizer):
sentences. sentences.
""" """
story_lines_token_ids = [ story_lines_token_ids = [
tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line)) tokenizer.build_inputs_with_special_tokens(tokenizer.encode(line))
for line in story_lines for line in story_lines
] ]
summary_lines_token_ids = [ summary_lines_token_ids = [
tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line)) tokenizer.build_inputs_with_special_tokens(tokenizer.encode(line))
for line in summary_lines for line in summary_lines
] ]