max_len_single_sentence & max_len_sentences_pair as attributes so they can be modified

This commit is contained in:
thomwolf committed 2019-08-23 22:07:26 +02:00
1 parent ab7bd5ef98
commit 3bcbebd440
8 files changed
+26 -40

No files matched your search

@@ -87,6 +87,9 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer):
def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
super(OpenAIGPTTokenizer, self).__init__(unk_token=unk_token, **kwargs)
self.max_len_single_sentence = self.max_len # no default special tokens - you can update this value if you add special tokens
self.max_len_sentences_pair = self.max_len # no default special tokens - you can update this value if you add special tokens
try:
import ftfy
from spacy.lang.en import English