Uncomment useful code.

Camembert variants

- The issue is that this information is not contained in the
`tokenizer` config file.
- It used to be harcoded already (with 512 value too).
- It is unclear right now how to "properly" fix it.

src/transformers/tokenization_camembert.py

@@ -113,6 +113,9 @@ class CamembertTokenizer(PreTrainedTokenizer):
         pad_token="<pad>",
         mask_token="<mask>",
         additional_special_tokens=["<s>NOTUSED", "</s>NOTUSED"],
+        # XXX: More discussion is necessary to remove this hard coded value
+        # more info here: https://github.com/huggingface/transformers/issues/8117
+        model_max_length=512,
         **kwargs
     ):
         super().__init__(
@@ -124,6 +127,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
+            model_max_length=model_max_length,
             **kwargs,
         )
         self.sp_model = spm.SentencePieceProcessor()

src/transformers/tokenization_camembert_fast.py

@@ -14,7 +14,6 @@
 # limitations under the License
 """ Fast tokenization classes for Camembert model."""
 
-
 import os
 from shutil import copyfile
 from typing import List, Optional, Tuple
@@ -125,6 +124,9 @@ class CamembertTokenizerFast(PreTrainedTokenizerFast):
         pad_token="<pad>",
         mask_token="<mask>",
         additional_special_tokens=["<s>NOTUSED", "</s>NOTUSED"],
+        # XXX: More discussion is necessary to remove this hard coded value
+        # more info here: https://github.com/huggingface/transformers/issues/8117
+        model_max_length=512,
         **kwargs
     ):
         super().__init__(
@@ -138,6 +140,7 @@ class CamembertTokenizerFast(PreTrainedTokenizerFast):
             pad_token=pad_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
+            model_max_length=model_max_length,
             **kwargs,
         )
 

src/transformers/tokenization_roberta.py

@@ -145,6 +145,9 @@ class RobertaTokenizer(GPT2Tokenizer):
         pad_token="<pad>",
         mask_token="<mask>",
         add_prefix_space=False,
+        # XXX: More discussion is necessary to remove this hard coded value
+        # more info here: https://github.com/huggingface/transformers/issues/8117
+        model_max_length=512,
         **kwargs
     ):
         bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
@@ -169,6 +172,7 @@ class RobertaTokenizer(GPT2Tokenizer):
             pad_token=pad_token,
             mask_token=mask_token,
             add_prefix_space=add_prefix_space,
+            model_max_length=model_max_length,
             **kwargs,
         )
 

src/transformers/tokenization_roberta_fast.py

@@ -154,6 +154,9 @@ class RobertaTokenizerFast(GPT2TokenizerFast):
         pad_token="<pad>",
         mask_token="<mask>",
         add_prefix_space=False,
+        # XXX: More discussion is necessary to remove this hard coded value
+        # more info here: https://github.com/huggingface/transformers/issues/8117
+        model_max_length=512,
         **kwargs
     ):
         super().__init__(
@@ -169,6 +172,7 @@ class RobertaTokenizerFast(GPT2TokenizerFast):
             pad_token=pad_token,
             mask_token=mask_token,
             add_prefix_space=add_prefix_space,
+            model_max_length=model_max_length,
             **kwargs,
         )
 

tests/test_tokenization_camembert.py

@@ -50,6 +50,9 @@ class CamembertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         tokenizer = self.get_tokenizer()
         rust_tokenizer = self.get_rust_tokenizer()
 
+        self.assertEqual(tokenizer.model_max_length, 512)
+        self.assertEqual(rust_tokenizer.model_max_length, 512)
+
         sequence = "I was born in 92000, and this is falsé."
 
         tokens = tokenizer.tokenize(sequence)

tests/test_tokenization_roberta.py

@@ -102,6 +102,21 @@ class RobertaTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
             [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2],
         )
 
+    def test_model_max_length_by_default(self):
+        tokenizer = self.get_tokenizer()
+        self.assertEqual(tokenizer.model_max_length, 512)
+
+        rust_tokenizer = self.tokenizer_class.from_pretrained("roberta-base", use_fast=True)
+        self.assertEqual(rust_tokenizer.model_max_length, 512)
+
+    @slow
+    def test_model_max_length(self):
+        tokenizer = self.tokenizer_class.from_pretrained("roberta-base")
+        self.assertEqual(tokenizer.model_max_length, 512)
+
+        rust_tokenizer = self.tokenizer_class.from_pretrained("roberta-base", use_fast=True)
+        self.assertEqual(rust_tokenizer.model_max_length, 512)
+
     @slow
     def test_sequence_builders(self):
         tokenizer = self.tokenizer_class.from_pretrained("roberta-base")