remove set max length

2020-04-22 07:55:19 +02:00
parent bad887ee78
commit 7d23815012
1 changed files with 0 additions and 6 deletions
@@ -96,12 +96,6 @@ class ReformerTokenizer(PreTrainedTokenizer):
            additional_special_tokens=additional_special_tokens,
            **kwargs,
        )
-        self.max_len_single_sentence = (
-            self.max_len
-        )  # no default special tokens - you can update this value if you add special tokens
-        self.max_len_sentences_pair = (
-            self.max_len
-        )  # no default special tokens - you can update this value if you add special tokens

        try:
            import sentencepiece as spm