mbart - marian - mobilebert - openai

src/transformers/__init__.py

@@ -55,6 +55,7 @@ from .data import (
 # Files and general utilities
 from .file_utils import (
     CONFIG_NAME,
+    TOKENIZER_CONFIG_NAME,
     MODEL_CARD_NAME,
     PYTORCH_PRETRAINED_BERT_CACHE,
     PYTORCH_TRANSFORMERS_CACHE,
@@ -102,27 +103,30 @@ from .models.auto import (
     AutoConfig,
     AutoTokenizer,
 )
-from .models.bart import BartConfig, BartTokenizer
+from .models.bart import BartConfig, BartTokenizer, BART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from .models.bert import (
     BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
+    BERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
     BasicTokenizer,
     BertConfig,
     BertTokenizer,
     WordpieceTokenizer,
 )
 from .models.bert_generation import BertGenerationConfig
-from .models.bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer
+from .models.bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer, BERT_JAPANESE_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.bertweet import BertweetTokenizer
+from .models.bertweet import BertweetTokenizer, BERT_TWEET_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from .models.blenderbot import (
     BLENDERBOT_PRETRAINED_CONFIG_ARCHIVE_MAP,
+    BLENDERBOT_LARGE_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+    BLENDERBOT_SMALL_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
     BlenderbotConfig,
     BlenderbotSmallTokenizer,
     BlenderbotTokenizer,
 )
 from .models.camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
-from .models.ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig, CTRLTokenizer
+from .models.ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig, CTRLTokenizer, CTRL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig, DebertaTokenizer
+from .models.deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig, DebertaTokenizer, DEBERTA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig, DistilBertTokenizer
+from .models.distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig, DistilBertTokenizer, DISTILBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from .models.dpr import (
     DPR_PRETRAINED_CONFIG_ARCHIVE_MAP,
     DPRConfig,
@@ -130,23 +134,26 @@ from .models.dpr import (
     DPRQuestionEncoderTokenizer,
     DPRReaderOutput,
     DPRReaderTokenizer,
+        DPR_CONTEXT_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+        DPR_QUESTION_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+        DPR_READER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 )
-from .models.electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraTokenizer
+from .models.electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraTokenizer, ELECTRA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from .models.encoder_decoder import EncoderDecoderConfig
-from .models.flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig, FlaubertTokenizer
+from .models.flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig, FlaubertTokenizer, FLAUBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.fsmt import FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP, FSMTConfig, FSMTTokenizer
+from .models.fsmt import FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP, FSMTConfig, FSMTTokenizer, FSMT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig, FunnelTokenizer
+from .models.funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig, FunnelTokenizer, FUNNEL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config, GPT2Tokenizer
+from .models.gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config, GPT2Tokenizer, GPT2_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.herbert import HerbertTokenizer
+from .models.herbert import HerbertTokenizer, HERBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.layoutlm import LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMConfig, LayoutLMTokenizer
+from .models.layoutlm import LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMConfig, LayoutLMTokenizer, LAYOUTLM_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig, LongformerTokenizer
+from .models.longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig, LongformerTokenizer, LONGFORMER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-from .models.lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig, LxmertTokenizer
+from .models.lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig, LxmertTokenizer, LXMERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from .models.marian import MarianConfig
 from .models.mbart import MBartConfig
 from .models.mmbt import MMBTConfig
-from .models.mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig, MobileBertTokenizer
+from .models.mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig, MobileBertTokenizer, MOBILEBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from .models.mt5 import MT5Config
-from .models.openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, OpenAIGPTTokenizer
+from .models.openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, OpenAIGPTTokenizer, OPENAIGPT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from .models.pegasus import PegasusConfig
 from .models.phobert import PhobertTokenizer
 from .models.prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig, ProphetNetTokenizer
@@ -204,11 +211,11 @@ from .tokenization_utils_base import (
 
 
 if is_sentencepiece_available():
-    from .models.albert import AlbertTokenizer
+    from .models.albert import AlbertTokenizer, ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.bert_generation import BertGenerationTokenizer
+    from .models.bert_generation import BertGenerationTokenizer, BERT_GENERATION_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.camembert import CamembertTokenizer
+    from .models.camembert import CamembertTokenizer, CAMEMBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.marian import MarianTokenizer
+    from .models.marian import MarianTokenizer, MARIAN_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.mbart import MBartTokenizer
+    from .models.mbart import MBartTokenizer, MBART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     from .models.pegasus import PegasusTokenizer
     from .models.reformer import ReformerTokenizer
     from .models.t5 import T5Tokenizer
@@ -219,22 +226,25 @@ else:
     from .utils.dummy_sentencepiece_objects import *
 
 if is_tokenizers_available():
-    from .models.albert import AlbertTokenizerFast
+    from .models.albert import AlbertTokenizerFast, ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.bart import BartTokenizerFast
+    from .models.bart import BartTokenizerFast, BART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.bert import BertTokenizerFast
+    from .models.bert import BertTokenizerFast, BERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.camembert import CamembertTokenizerFast
+    from .models.camembert import CamembertTokenizerFast, CAMEMBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.distilbert import DistilBertTokenizerFast
+    from .models.distilbert import DistilBertTokenizerFast, DISTILBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.dpr import DPRContextEncoderTokenizerFast, DPRQuestionEncoderTokenizerFast, DPRReaderTokenizerFast
+    from .models.dpr import (DPRContextEncoderTokenizerFast, DPRQuestionEncoderTokenizerFast, DPRReaderTokenizerFast,        DPR_CONTEXT_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
-    from .models.electra import ElectraTokenizerFast
+        DPR_QUESTION_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
-    from .models.funnel import FunnelTokenizerFast
+        DPR_READER_PRETRAINED_TOKENIZER_ARCHIVE_LIST)
-    from .models.gpt2 import GPT2TokenizerFast
+
-    from .models.herbert import HerbertTokenizerFast
+    from .models.electra import ElectraTokenizerFast, ELECTRA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.layoutlm import LayoutLMTokenizerFast
+    from .models.funnel import FunnelTokenizerFast, FUNNEL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.longformer import LongformerTokenizerFast
+    from .models.gpt2 import GPT2TokenizerFast, GPT2_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.lxmert import LxmertTokenizerFast
+    from .models.herbert import HerbertTokenizerFast, HERBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.mbart import MBartTokenizerFast
+    from .models.layoutlm import LayoutLMTokenizerFast, LAYOUTLM_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.mobilebert import MobileBertTokenizerFast
+    from .models.longformer import LongformerTokenizerFast, LONGFORMER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
-    from .models.openai import OpenAIGPTTokenizerFast
+    from .models.lxmert import LxmertTokenizerFast, LXMERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+    from .models.mbart import MBartTokenizerFast, MBART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+    from .models.mobilebert import MobileBertTokenizerFast, MOBILEBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+    from .models.openai import OpenAIGPTTokenizerFast, OPENAIGPT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     from .models.pegasus import PegasusTokenizerFast
     from .models.reformer import ReformerTokenizerFast
     from .models.retribert import RetriBertTokenizerFast

src/transformers/convert_slow_tokenizers_checkpoints_to_fast.py

@@ -46,7 +46,7 @@ def convert_slow_checkpoint_to_fast(tokenizer_name, checkpoint_name, dump_path,
 
         add_prefix = True
         if checkpoint_name is None:
-            checkpoint_names = list(tokenizer_class.max_model_input_sizes.keys())
+            raise ValueError("Please supply a checkpoint name")
         else:
             checkpoint_names = [checkpoint_name]
 
@@ -78,16 +78,16 @@ def convert_slow_checkpoint_to_fast(tokenizer_name, checkpoint_name, dump_path,
                 "=> {} with prefix {}, add_prefix {}".format(dump_path_full, checkpoint_prefix_name, add_prefix)
             )
 
-            if checkpoint in list(tokenizer.pretrained_vocab_files_map.values())[0]:
+            # if checkpoint in list(tokenizer.pretrained_vocab_files_map.values())[0]:
-                file_path = list(tokenizer.pretrained_vocab_files_map.values())[0][checkpoint]
+            #     file_path = list(tokenizer.pretrained_vocab_files_map.values())[0][checkpoint]
-                next_char = file_path.split(checkpoint)[-1][0]
+            #     next_char = file_path.split(checkpoint)[-1][0]
-                if next_char == "/":
+            #     if next_char == "/":
-                    dump_path_full = os.path.join(dump_path_full, checkpoint_prefix_name)
+            #         dump_path_full = os.path.join(dump_path_full, checkpoint_prefix_name)
-                    checkpoint_prefix_name = None
+            #         checkpoint_prefix_name = None
 
-                logger.info(
+            #     logger.info(
-                    "=> {} with prefix {}, add_prefix {}".format(dump_path_full, checkpoint_prefix_name, add_prefix)
+            #         "=> {} with prefix {}, add_prefix {}".format(dump_path_full, checkpoint_prefix_name, add_prefix)
-                )
+            #     )
 
             file_names = tokenizer.save_pretrained(
                 dump_path_full, legacy_format=False, filename_prefix=checkpoint_prefix_name

src/transformers/file_utils.py

@@ -234,6 +234,7 @@ WEIGHTS_NAME = "pytorch_model.bin"
 TF2_WEIGHTS_NAME = "tf_model.h5"
 TF_WEIGHTS_NAME = "model.ckpt"
 CONFIG_NAME = "config.json"
+TOKENIZER_CONFIG_NAME = "tokenizer_config.json"
 MODEL_CARD_NAME = "modelcard.json"
 
 SENTENCEPIECE_UNDERLINE = "▁"

src/transformers/models/albert/__init__.py

@@ -7,10 +7,10 @@ from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertCo
 
 
 if is_sentencepiece_available():
-    from .tokenization_albert import AlbertTokenizer
+    from .tokenization_albert import AlbertTokenizer, ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_tokenizers_available():
-    from .tokenization_albert_fast import AlbertTokenizerFast
+    from .tokenization_albert_fast import AlbertTokenizerFast, ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_albert import (

src/transformers/models/albert/tokenization_albert.py

@@ -27,31 +27,20 @@ from ...utils import logging
 
 
 logger = logging.get_logger(__name__)
+
 VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "albert-base-v1",
-        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
+    "albert-large-v1",
-        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
+    "albert-xlarge-v1",
-        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
+    "albert-xxlarge-v1",
-        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
+    "albert-base-v2",
-        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
+    "albert-large-v2",
-        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
+    "albert-xlarge-v2",
-        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
+    "albert-xxlarge-v2",
-        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
+    # See all ALBERT models at https://huggingface.co/models?filter=albert
-    }
+]
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "albert-base-v1": 512,
-    "albert-large-v1": 512,
-    "albert-xlarge-v1": 512,
-    "albert-xxlarge-v1": 512,
-    "albert-base-v2": 512,
-    "albert-large-v2": 512,
-    "albert-xlarge-v2": 512,
-    "albert-xxlarge-v2": 512,
-}
 
 SPIECE_UNDERLINE = "▁"
 
@@ -108,8 +97,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(
         self,

src/transformers/models/albert/tokenization_albert_fast.py

@@ -32,39 +32,17 @@ else:
 logger = logging.get_logger(__name__)
 VOCAB_FILES_NAMES = {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "albert-base-v1",
-        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
+    "albert-large-v1",
-        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
+    "albert-xlarge-v1",
-        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
+    "albert-xxlarge-v1",
-        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
+    "albert-base-v2",
-        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
+    "albert-large-v2",
-        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
+    "albert-xlarge-v2",
-        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
+    "albert-xxlarge-v2",
-        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
+    # See all ALBERT models at https://huggingface.co/models?filter=albert
-    },
+]
-    "tokenizer_file": {
-        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json",
-        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json",
-        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json",
-        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json",
-        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json",
-        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json",
-        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json",
-        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json",
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "albert-base-v1": 512,
-    "albert-large-v1": 512,
-    "albert-xlarge-v1": 512,
-    "albert-xxlarge-v1": 512,
-    "albert-base-v2": 512,
-    "albert-large-v2": 512,
-    "albert-xlarge-v2": 512,
-    "albert-xxlarge-v2": 512,
-}
 
 SPIECE_UNDERLINE = "▁"
 
@@ -113,8 +91,7 @@ class AlbertTokenizerFast(PreTrainedTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     slow_tokenizer_class = AlbertTokenizer
 
     def __init__(

src/transformers/models/auto/tokenization_auto.py

@@ -14,11 +14,14 @@
 # limitations under the License.
 """ Auto Tokenizer class. """
 
+import json
+import os
 
 from collections import OrderedDict
 
 from ...configuration_utils import PretrainedConfig
-from ...file_utils import is_sentencepiece_available, is_tokenizers_available
+from ...file_utils import is_sentencepiece_available, is_tokenizers_available, cached_path, hf_bucket_url, is_remote_url
+from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE
 from ...utils import logging
 from ..bart.tokenization_bart import BartTokenizer
 from ..bert.tokenization_bert import BertTokenizer
@@ -93,6 +96,7 @@ from .configuration_auto import (
 if is_sentencepiece_available():
     from ..albert.tokenization_albert import AlbertTokenizer
     from ..bert_generation.tokenization_bert_generation import BertGenerationTokenizer
+    from ..blenderbot.tokenization_blenderbot import BlenderbotTokenizer
     from ..camembert.tokenization_camembert import CamembertTokenizer
     from ..marian.tokenization_marian import MarianTokenizer
     from ..mbart.tokenization_mbart import MBartTokenizer
@@ -105,6 +109,7 @@ if is_sentencepiece_available():
 else:
     AlbertTokenizer = None
     BertGenerationTokenizer = None
+    BlenderbotTokenizer = None
     CamembertTokenizer = None
     MarianTokenizer = None
     MBartTokenizer = None
@@ -217,8 +222,11 @@ NO_CONFIG_TOKENIZER = [
     HerbertTokenizer,
     HerbertTokenizerFast,
     PhobertTokenizer,
+    BlenderbotTokenizer,
 ]
 
+class NoTokenizerClassAttribute(Exception):
+    pass
 
 SLOW_TOKENIZER_MAPPING = {
     k: (v[0] if v[0] is not None else v[1])
@@ -322,18 +330,70 @@ class AutoTokenizer:
 
         """
         config = kwargs.pop("config", None)
-        if not isinstance(config, PretrainedConfig):
+        tokenizer_class_name = config.tokenizer_class if isinstance(config, PretrainedConfig) else None
-            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+        if tokenizer_class_name is None:
+            # Try first to load a tokenizer_config.json file
+            # This logic is identical to the from_pretrained logic in configuration_utils.py
+            # We duplicate it because we don't have (yet?) a full class for tokenizer config, they are just simple dicts
+            cache_dir = kwargs.get("cache_dir", None)
+            force_download = kwargs.get("force_download", False)
+            resume_download = kwargs.get("resume_download", False)
+            proxies = kwargs.get("proxies", None)
+            local_files_only = kwargs.get("local_files_only", False)
+            revision = kwargs.get("revision", None)
+
+            if os.path.isdir(pretrained_model_name_or_path):
+                tokenizer_config_file = os.path.join(pretrained_model_name_or_path, TOKENIZER_CONFIG_FILE)
+            elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
+                tokenizer_config_file = pretrained_model_name_or_path
+            else:
+                tokenizer_config_file = hf_bucket_url(
+                    pretrained_model_name_or_path, filename=TOKENIZER_CONFIG_FILE, revision=revision, mirror=None
+                )
+
+            try:
+                # Load from URL or cache if already cached
+                resolved_tokenizer_config_file = cached_path(
+                    tokenizer_config_file,
+                    cache_dir=cache_dir,
+                    force_download=force_download,
+                    proxies=proxies,
+                    resume_download=resume_download,
+                    local_files_only=local_files_only,
+                )
+                # Load config dict
+                with open(resolved_tokenizer_config_file, "r", encoding="utf-8") as reader:
+                    text = reader.read()
+                config = json.loads(text)
+
+                tokenizer_class_name = config.get("tokenizer_class_name", None)
+                if tokenizer_class_name is None:
+                    raise NoTokenizerClassAttribute
+
+            # If we cannot find the tokenizer_config file (EnvironmentError)
+            # or cannot decode it (json.JSONDecodeError)
+            # or it doesn't have a "tokenizer_class" attribute (NoTokenizerClassAttribute)
+            # We try to load the configuration for the model
+            except (EnvironmentError, json.JSONDecodeError, NoTokenizerClassAttribute):
+                # Here we are ok to raise the errors because we found no way to identify the tokenizer class
+                model_config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+                tokenizer_class_name = model_config.tokenizer_class
+
+                # We only overwrite the potentially provided config if it's None
+                # Since it can be used to detect class (see below the type(config) test)
+                if config is None:
+                    config = model_config
 
         use_fast = kwargs.pop("use_fast", True)
 
-        if config.tokenizer_class is not None:
+        if tokenizer_class_name is not None:
             tokenizer_class = None
-            if use_fast and not config.tokenizer_class.endswith("Fast"):
+            if use_fast and not tokenizer_class_name.endswith("Fast"):
-                tokenizer_class_candidate = f"{config.tokenizer_class}Fast"
+                tokenizer_class_candidate = f"{tokenizer_class_name}Fast"
                 tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
             if tokenizer_class is None:
-                tokenizer_class_candidate = config.tokenizer_class
+                tokenizer_class_candidate = tokenizer_class_name
                 tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
 
             if tokenizer_class is None:
@@ -342,6 +402,9 @@ class AutoTokenizer:
                 )
             return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
 
+        # Could find a tokenizer_class in the tokenizer or model config.
+        # Let's try other (backward compatible) options
+
         # if model is an encoder decoder, the encoder tokenizer class is used by default
         if isinstance(config, EncoderDecoderConfig):
             if type(config.decoder) is not type(config.encoder):  # noqa: E721

src/transformers/models/bart/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_bart import BartConfig
-from .tokenization_bart import BartTokenizer
+from .tokenization_bart import BartTokenizer, BART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_bart_fast import BartTokenizerFast
+    from .tokenization_bart_fast import BartTokenizerFast, BART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_bart import (

src/transformers/models/bart/tokenization_bart.py

@@ -25,17 +25,19 @@ from ..roberta.tokenization_roberta import RobertaTokenizer
 logger = logging.get_logger(__name__)
 
 
-# vocab and merges same as roberta
+VOCAB_FILES_NAMES = {
-vocab_url = "https://huggingface.co/roberta-large/resolve/main/vocab.json"
+    "vocab_file": "vocab.json",
-merges_url = "https://huggingface.co/roberta-large/resolve/main/merges.txt"
+    "merges_file": "merges.txt",
-_all_bart_models = [
+}
+
+BART_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
     "facebook/bart-base",
     "facebook/bart-large",
     "facebook/bart-large-mnli",
     "facebook/bart-large-cnn",
     "facebook/bart-large-xsum",
     "yjernite/bart_eli5",
-    # This is not exhaustive: see https://huggingface.co/models?filter=bart
+    # See all BART models at https://huggingface.co/models?filter=bart
 ]
 
 
@@ -49,12 +51,8 @@ class BartTokenizer(RobertaTokenizer):
     Refer to superclass :class:`~transformers.RobertaTokenizer` for usage examples and documentation concerning the
     initialization parameters and other methods.
     """
-    # merges and vocab same as Roberta
+    vocab_files_names = VOCAB_FILES_NAMES
-    max_model_input_sizes = {m: 1024 for m in _all_bart_models}
+    max_model_input_sizes = 1024
-    pretrained_vocab_files_map = {
-        "vocab_file": {m: vocab_url for m in _all_bart_models},
-        "merges_file": {m: merges_url for m in _all_bart_models},
-    }
 
     @add_start_docstrings(PREPARE_SEQ2SEQ_BATCH_DOCSTRING)
     def prepare_seq2seq_batch(

src/transformers/models/bart/tokenization_bart_fast.py

@@ -26,29 +26,26 @@ from .tokenization_bart import BartTokenizer
 logger = logging.get_logger(__name__)
 
 
-# vocab and merges same as roberta
+VOCAB_FILES_NAMES = {
-vocab_url = "https://huggingface.co/roberta-large/resolve/main/vocab.json"
+    "vocab_file": "vocab.json",
-merges_url = "https://huggingface.co/roberta-large/resolve/main/merges.txt"
+    "merges_file": "merges.txt",
-tokenizer_url = "https://huggingface.co/roberta-large/resolve/main/tokenizer.json"
+    "tokenizer_file": "tokenizer.json"
-_all_bart_models = [
+}
+
+BART_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
     "facebook/bart-base",
     "facebook/bart-large",
     "facebook/bart-large-mnli",
     "facebook/bart-large-cnn",
     "facebook/bart-large-xsum",
     "yjernite/bart_eli5",
-    # This is not exhaustive: see https://huggingface.co/models?filter=bart
+    # See all BART models at https://huggingface.co/models?filter=bart
 ]
 
 
 class BartTokenizerFast(RobertaTokenizerFast):
-    # merges and vocab same as Roberta
+    vocab_files_names = VOCAB_FILES_NAMES
-    max_model_input_sizes = {m: 1024 for m in _all_bart_models}
+    max_model_input_sizes = 1024
-    pretrained_vocab_files_map = {
-        "vocab_file": {m: vocab_url for m in _all_bart_models},
-        "merges_file": {m: merges_url for m in _all_bart_models},
-        "tokenizer_file": {m: tokenizer_url for m in _all_bart_models},
-    }
     slow_tokenizer_class = BartTokenizer
 
     @add_start_docstrings(PREPARE_SEQ2SEQ_BATCH_DOCSTRING)

src/transformers/models/bert/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_flax_available, is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig
-from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer
+from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer, BERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_bert_fast import BertTokenizerFast
+    from .tokenization_bert_fast import BertTokenizerFast, BERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_bert import (

src/transformers/models/bert/tokenization_bert.py

@@ -28,70 +28,27 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+BERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "bert-base-uncased",
-        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    "bert-large-uncased",
-        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
+    "bert-base-cased",
-        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
+    "bert-large-cased",
-        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
+    "bert-base-multilingual-uncased",
-        "bert-base-multilingual-uncased": "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt",
+    "bert-base-multilingual-cased",
-        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
+    "bert-base-chinese",
-        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
+    "bert-base-german-cased",
-        "bert-base-german-cased": "https://int-deepset-models-bert.s3.eu-central-1.amazonaws.com/pytorch/bert-base-german-cased-vocab.txt",
+    "bert-large-uncased-whole-word-masking",
-        "bert-large-uncased-whole-word-masking": "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt",
+    "bert-large-cased-whole-word-masking",
-        "bert-large-cased-whole-word-masking": "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt",
+    "bert-large-uncased-whole-word-masking-finetuned-squad",
-        "bert-large-uncased-whole-word-masking-finetuned-squad": "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt",
+    "bert-large-cased-whole-word-masking-finetuned-squad",
-        "bert-large-cased-whole-word-masking-finetuned-squad": "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt",
+    "bert-base-cased-finetuned-mrpc",
-        "bert-base-cased-finetuned-mrpc": "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt",
+    "bert-base-german-dbmdz-cased",
-        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
+    "bert-base-german-dbmdz-uncased",
-        "bert-base-german-dbmdz-uncased": "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt",
+    "TurkuNLP/bert-base-finnish-cased-v1",
-        "TurkuNLP/bert-base-finnish-cased-v1": "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt",
+    "TurkuNLP/bert-base-finnish-uncased-v1",
-        "TurkuNLP/bert-base-finnish-uncased-v1": "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt",
+    "wietsedv/bert-base-dutch-cased",
-        "wietsedv/bert-base-dutch-cased": "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt",
+    # See all BERT models at https://huggingface.co/models?filter=bert
-    }
+]
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "bert-base-uncased": 512,
-    "bert-large-uncased": 512,
-    "bert-base-cased": 512,
-    "bert-large-cased": 512,
-    "bert-base-multilingual-uncased": 512,
-    "bert-base-multilingual-cased": 512,
-    "bert-base-chinese": 512,
-    "bert-base-german-cased": 512,
-    "bert-large-uncased-whole-word-masking": 512,
-    "bert-large-cased-whole-word-masking": 512,
-    "bert-large-uncased-whole-word-masking-finetuned-squad": 512,
-    "bert-large-cased-whole-word-masking-finetuned-squad": 512,
-    "bert-base-cased-finetuned-mrpc": 512,
-    "bert-base-german-dbmdz-cased": 512,
-    "bert-base-german-dbmdz-uncased": 512,
-    "TurkuNLP/bert-base-finnish-cased-v1": 512,
-    "TurkuNLP/bert-base-finnish-uncased-v1": 512,
-    "wietsedv/bert-base-dutch-cased": 512,
-}
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "bert-base-uncased": {"do_lower_case": True},
-    "bert-large-uncased": {"do_lower_case": True},
-    "bert-base-cased": {"do_lower_case": False},
-    "bert-large-cased": {"do_lower_case": False},
-    "bert-base-multilingual-uncased": {"do_lower_case": True},
-    "bert-base-multilingual-cased": {"do_lower_case": False},
-    "bert-base-chinese": {"do_lower_case": False},
-    "bert-base-german-cased": {"do_lower_case": False},
-    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
-    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
-    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
-    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
-    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
-    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
-    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
-    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
-    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
-    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
-}
 
 
 def load_vocab(vocab_file):
@@ -157,9 +114,7 @@ class BertTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(
         self,

src/transformers/models/bert/tokenization_bert_fast.py

@@ -28,90 +28,27 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+BERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "bert-base-uncased",
-        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    "bert-large-uncased",
-        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
+    "bert-base-cased",
-        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
+    "bert-large-cased",
-        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
+    "bert-base-multilingual-uncased",
-        "bert-base-multilingual-uncased": "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt",
+    "bert-base-multilingual-cased",
-        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
+    "bert-base-chinese",
-        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
+    "bert-base-german-cased",
-        "bert-base-german-cased": "https://int-deepset-models-bert.s3.eu-central-1.amazonaws.com/pytorch/bert-base-german-cased-vocab.txt",
+    "bert-large-uncased-whole-word-masking",
-        "bert-large-uncased-whole-word-masking": "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt",
+    "bert-large-cased-whole-word-masking",
-        "bert-large-cased-whole-word-masking": "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt",
+    "bert-large-uncased-whole-word-masking-finetuned-squad",
-        "bert-large-uncased-whole-word-masking-finetuned-squad": "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt",
+    "bert-large-cased-whole-word-masking-finetuned-squad",
-        "bert-large-cased-whole-word-masking-finetuned-squad": "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt",
+    "bert-base-cased-finetuned-mrpc",
-        "bert-base-cased-finetuned-mrpc": "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt",
+    "bert-base-german-dbmdz-cased",
-        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
+    "bert-base-german-dbmdz-uncased",
-        "bert-base-german-dbmdz-uncased": "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt",
+    "TurkuNLP/bert-base-finnish-cased-v1",
-        "TurkuNLP/bert-base-finnish-cased-v1": "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt",
+    "TurkuNLP/bert-base-finnish-uncased-v1",
-        "TurkuNLP/bert-base-finnish-uncased-v1": "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt",
+    "wietsedv/bert-base-dutch-cased",
-        "wietsedv/bert-base-dutch-cased": "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt",
+    # See all BERT models at https://huggingface.co/models?filter=bert
-    },
+]
-    "tokenizer_file": {
-        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
-        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
-        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
-        "bert-base-multilingual-uncased": "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json",
-        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json",
-        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
-        "bert-base-german-cased": "https://int-deepset-models-bert.s3.eu-central-1.amazonaws.com/pytorch/bert-base-german-cased-tokenizer.json",
-        "bert-large-uncased-whole-word-masking": "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json",
-        "bert-large-cased-whole-word-masking": "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json",
-        "bert-large-uncased-whole-word-masking-finetuned-squad": "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json",
-        "bert-large-cased-whole-word-masking-finetuned-squad": "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json",
-        "bert-base-cased-finetuned-mrpc": "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json",
-        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json",
-        "bert-base-german-dbmdz-uncased": "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json",
-        "TurkuNLP/bert-base-finnish-cased-v1": "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json",
-        "TurkuNLP/bert-base-finnish-uncased-v1": "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json",
-        "wietsedv/bert-base-dutch-cased": "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json",
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "bert-base-uncased": 512,
-    "bert-large-uncased": 512,
-    "bert-base-cased": 512,
-    "bert-large-cased": 512,
-    "bert-base-multilingual-uncased": 512,
-    "bert-base-multilingual-cased": 512,
-    "bert-base-chinese": 512,
-    "bert-base-german-cased": 512,
-    "bert-large-uncased-whole-word-masking": 512,
-    "bert-large-cased-whole-word-masking": 512,
-    "bert-large-uncased-whole-word-masking-finetuned-squad": 512,
-    "bert-large-cased-whole-word-masking-finetuned-squad": 512,
-    "bert-base-cased-finetuned-mrpc": 512,
-    "bert-base-german-dbmdz-cased": 512,
-    "bert-base-german-dbmdz-uncased": 512,
-    "TurkuNLP/bert-base-finnish-cased-v1": 512,
-    "TurkuNLP/bert-base-finnish-uncased-v1": 512,
-    "wietsedv/bert-base-dutch-cased": 512,
-}
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "bert-base-uncased": {"do_lower_case": True},
-    "bert-large-uncased": {"do_lower_case": True},
-    "bert-base-cased": {"do_lower_case": False},
-    "bert-large-cased": {"do_lower_case": False},
-    "bert-base-multilingual-uncased": {"do_lower_case": True},
-    "bert-base-multilingual-cased": {"do_lower_case": False},
-    "bert-base-chinese": {"do_lower_case": False},
-    "bert-base-german-cased": {"do_lower_case": False},
-    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
-    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
-    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
-    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
-    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
-    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
-    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
-    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
-    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
-    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
-}
 
 
 class BertTokenizerFast(PreTrainedTokenizerFast):
@@ -155,9 +92,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     slow_tokenizer_class = BertTokenizer
 
     def __init__(

src/transformers/models/bert_generation/__init__.py

@@ -7,7 +7,7 @@ from .configuration_bert_generation import BertGenerationConfig
 
 
 if is_sentencepiece_available():
-    from .tokenization_bert_generation import BertGenerationTokenizer
+    from .tokenization_bert_generation import BertGenerationTokenizer, BERT_GENERATION_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_bert_generation import (

src/transformers/models/bert_generation/tokenization_bert_generation.py

@@ -29,7 +29,10 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}
 
-tokenizer_url = "https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model"
+
+BERT_GENERATION_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
+    "google/bert_for_seq_generation_L-24_bbc_encoder",
+]
 
 
 class BertGenerationTokenizer(PreTrainedTokenizer):
@@ -55,8 +58,7 @@ class BertGenerationTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = {"vocab_file": {"bert_for_seq_generation": tokenizer_url}}
+    max_model_input_sizes = 512
-    max_model_input_sizes = {"bert_for_seq_generation": 512}
     prefix_tokens: List[int] = []
 
     def __init__(

src/transformers/models/bert_japanese/__init__.py

@@ -2,4 +2,4 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer
+from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer, BERT_JAPANESE_PRETRAINED_TOKENIZER_ARCHIVE_LIST

src/transformers/models/bert_japanese/tokenization_bert_japanese.py

@@ -29,53 +29,19 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+BERT_JAPANESE_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+        "cl-tohoku/bert-base-japanese",
-        "cl-tohoku/bert-base-japanese": "https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/vocab.txt",
+        "cl-tohoku/bert-base-japanese-whole-word-masking",
-        "cl-tohoku/bert-base-japanese-whole-word-masking": "https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/vocab.txt",
+        "cl-tohoku/bert-base-japanese-char",
-        "cl-tohoku/bert-base-japanese-char": "https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/vocab.txt",
+        "cl-tohoku/bert-base-japanese-char-whole-word-masking",
-        "cl-tohoku/bert-base-japanese-char-whole-word-masking": "https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/vocab.txt",
+]
-    }
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "cl-tohoku/bert-base-japanese": 512,
-    "cl-tohoku/bert-base-japanese-whole-word-masking": 512,
-    "cl-tohoku/bert-base-japanese-char": 512,
-    "cl-tohoku/bert-base-japanese-char-whole-word-masking": 512,
-}
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "cl-tohoku/bert-base-japanese": {
-        "do_lower_case": False,
-        "word_tokenizer_type": "mecab",
-        "subword_tokenizer_type": "wordpiece",
-    },
-    "cl-tohoku/bert-base-japanese-whole-word-masking": {
-        "do_lower_case": False,
-        "word_tokenizer_type": "mecab",
-        "subword_tokenizer_type": "wordpiece",
-    },
-    "cl-tohoku/bert-base-japanese-char": {
-        "do_lower_case": False,
-        "word_tokenizer_type": "mecab",
-        "subword_tokenizer_type": "character",
-    },
-    "cl-tohoku/bert-base-japanese-char-whole-word-masking": {
-        "do_lower_case": False,
-        "word_tokenizer_type": "mecab",
-        "subword_tokenizer_type": "character",
-    },
-}
 
 
 class BertJapaneseTokenizer(BertTokenizer):
     """BERT tokenizer for Japanese text"""
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(
         self,

src/transformers/models/bertweet/__init__.py

@@ -2,4 +2,4 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-from .tokenization_bertweet import BertweetTokenizer
+from .tokenization_bertweet import BertweetTokenizer, BERT_TWEET_PRETRAINED_TOKENIZER_ARCHIVE_LIST

src/transformers/models/bertweet/tokenization_bertweet.py

@@ -35,18 +35,9 @@ VOCAB_FILES_NAMES = {
     "merges_file": "bpe.codes",
 }
 
-PRETRAINED_VOCAB_FILES_MAP = {
+BERT_TWEET_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "vinai/bertweet-base",
-        "vinai/bertweet-base": "https://huggingface.co/vinai/bertweet-base/resolve/main/vocab.txt",
+]
-    },
-    "merges_file": {
-        "vinai/bertweet-base": "https://huggingface.co/vinai/bertweet-base/resolve/main/bpe.codes",
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "vinai/bertweet-base": 128,
-}
 
 
 def get_pairs(word):
@@ -111,8 +102,7 @@ class BertweetTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 128
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(
         self,

src/transformers/models/blenderbot/__init__.py

@@ -4,7 +4,7 @@
 
 from ...file_utils import is_tf_available, is_torch_available
 from .configuration_blenderbot import BLENDERBOT_PRETRAINED_CONFIG_ARCHIVE_MAP, BlenderbotConfig
-from .tokenization_blenderbot import BlenderbotSmallTokenizer, BlenderbotTokenizer
+from .tokenization_blenderbot import BlenderbotSmallTokenizer, BlenderbotTokenizer, BLENDERBOT_LARGE_PRETRAINED_TOKENIZER_ARCHIVE_LIST, BLENDERBOT_SMALL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_torch_available():

src/transformers/models/blenderbot/tokenization_blenderbot.py

@@ -32,9 +32,17 @@ logger = logging.get_logger(__name__)
 VOCAB_FILES_NAMES = {
     "vocab_file": "vocab.json",
     "merges_file": "merges.txt",
-    # "tokenizer_config_file": "tokenizer_config.json",
 }
-CKPT_3B = "facebook/blenderbot-3B"
+
+BLENDERBOT_LARGE_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
+    "facebook/blenderbot-3B",
+    # See all BLENDERBOT models at https://huggingface.co/models?filter=blenderbot
+]
+
+BLENDERBOT_SMALL_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
+    "facebook/blenderbot-90M",
+    # See all BLENDERBOT models at https://huggingface.co/models?filter=blenderbot
+]
 
 
 class BlenderbotTokenizer(RobertaTokenizer):
@@ -48,17 +56,8 @@ class BlenderbotTokenizer(RobertaTokenizer):
     Refer to superclass :class:`~transformers.RobertaTokenizer` for usage examples and documentation concerning
     parameters.
     """
-    vocab_files_names = {
+    vocab_files_names = VOCAB_FILES_NAMES
-        "vocab_file": "vocab.json",
+    max_model_input_sizes = 128
-        "merges_file": "merges.txt",
-        "tokenizer_config_file": "tokenizer_config.json",
-    }
-    pretrained_vocab_files_map = {
-        "vocab_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/vocab.json"},
-        "merges_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/merges.txt"},
-        "tokenizer_config_file": {CKPT_3B: "https://cdn.huggingface.co/facebook/blenderbot-3B/tokenizer_config.json"},
-    }
-    max_model_input_sizes = {"facebook/blenderbot-3B": 128}
 
     def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: List[int] = None):
         """
@@ -121,11 +120,7 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
-    pretrained_vocab_files_map = {
+    max_model_input_sizes = 512
-        "vocab_file": {"facebook/blenderbot-90M": "https://cdn.huggingface.co/facebook/blenderbot-90M/vocab.json"},
-        "merges_file": {"facebook/blenderbot-90M": "https://cdn.huggingface.co/facebook/blenderbot-90M/merges.txt"},
-    }
-    max_model_input_sizes = {"facebook/blenderbot-90M": 512}
 
     def __init__(
         self,

src/transformers/models/camembert/__init__.py

@@ -7,10 +7,10 @@ from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, Ca
 
 
 if is_sentencepiece_available():
-    from .tokenization_camembert import CamembertTokenizer
+    from .tokenization_camembert import CamembertTokenizer, CAMEMBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_tokenizers_available():
-    from .tokenization_camembert_fast import CamembertTokenizerFast
+    from .tokenization_camembert_fast import CamembertTokenizerFast, CAMEMBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_camembert import (

src/transformers/models/camembert/tokenization_camembert.py

@@ -29,21 +29,9 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.bpe.model"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+CAMEMBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "camembert-base",
-        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model",
+    # See all CAMEMBERT models at https://huggingface.co/models?filter=camembert
-    }
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "camembert-base": 512,
-}
-
-SHARED_MODEL_IDENTIFIERS = [
-    # Load with
-    # `tokenizer = AutoTokenizer.from_pretrained("username/pretrained_model")`
-    "Musixmatch/umberto-commoncrawl-cased-v1",
-    "Musixmatch/umberto-wikipedia-uncased-v1",
 ]
 
 SPIECE_UNDERLINE = "▁"
@@ -98,8 +86,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     model_input_names = ["attention_mask"]
 
     def __init__(

src/transformers/models/camembert/tokenization_camembert_fast.py

@@ -34,26 +34,12 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+CAMEMBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "camembert-base",
-        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model",
+    # See all CAMEMBERT models at https://huggingface.co/models?filter=camembert
-    },
-    "tokenizer_file": {
-        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/tokenizer.json",
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "camembert-base": 512,
-}
-
-SHARED_MODEL_IDENTIFIERS = [
-    # Load with
-    # `tokenizer = AutoTokenizer.from_pretrained("username/pretrained_model")`
-    "Musixmatch/umberto-commoncrawl-cased-v1",
-    "Musixmatch/umberto-wikipedia-uncased-v1",
 ]
 
+
 SPIECE_UNDERLINE = "▁"
 
 
@@ -108,8 +94,7 @@ class CamembertTokenizerFast(PreTrainedTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     model_input_names = ["attention_mask"]
     slow_tokenizer_class = CamembertTokenizer
 

src/transformers/models/ctrl/__init__.py

@@ -4,7 +4,7 @@
 
 from ...file_utils import is_tf_available, is_torch_available
 from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
-from .tokenization_ctrl import CTRLTokenizer
+from .tokenization_ctrl import CTRLTokenizer, CTRL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_torch_available():

src/transformers/models/ctrl/tokenization_ctrl.py

@@ -28,18 +28,14 @@ from ...utils import logging
 logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {
-    "vocab_file": "vocab.json",
+    "vocab_file": "ctrl-vocab.json",
-    "merges_file": "merges.txt",
+    "merges_file": "ctrl-merges.txt",
 }
 
-PRETRAINED_VOCAB_FILES_MAP = {
+CTRL_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
+    "salesforce/ctrl",
-    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
+    # See all CTRL models at https://huggingface.co/models?filter=ctrl
-}
+]
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "ctrl": 256,
-}
 
 CONTROL_CODES = {
     "Pregnancy": 168629,
@@ -134,8 +130,7 @@ class CTRLTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 256
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     control_codes = CONTROL_CODES
 
     def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):

src/transformers/models/deberta/__init__.py

@@ -4,7 +4,7 @@
 
 from ...file_utils import is_torch_available
 from .configuration_deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig
-from .tokenization_deberta import DebertaTokenizer
+from .tokenization_deberta import DebertaTokenizer, DEBERTA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_torch_available():

src/transformers/models/deberta/tokenization_deberta.py

@@ -40,22 +40,11 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "bpe_encoder.bin"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+DEBERTA_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "microsoft/deberta-base",
-        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/bpe_encoder.bin",
+    "microsoft/deberta-large",
-        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/bpe_encoder.bin",
+    # See all DEBERTA models at https://huggingface.co/models?search=deberta
-    }
+]
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "microsoft/deberta-base": 512,
-    "microsoft/deberta-large": 512,
-}
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "microsoft/deberta-base": {"do_lower_case": False},
-    "microsoft/deberta-large": {"do_lower_case": False},
-}
 
 __all__ = ["DebertaTokenizer"]
 
@@ -514,9 +503,7 @@ class DebertaTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(
         self,

src/transformers/models/distilbert/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig
-from .tokenization_distilbert import DistilBertTokenizer
+from .tokenization_distilbert import DistilBertTokenizer, DISTILBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_distilbert_fast import DistilBertTokenizerFast
+    from .tokenization_distilbert_fast import DistilBertTokenizerFast, DISTILBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_distilbert import (

src/transformers/models/distilbert/tokenization_distilbert.py

@@ -22,35 +22,15 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+DISTILBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "distilbert-base-uncased",
-        "distilbert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    "distilbert-base-uncased-distilled-squad",
-        "distilbert-base-uncased-distilled-squad": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
+    "distilbert-base-cased",
-        "distilbert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
+    "distilbert-base-cased-distilled-squad",
-        "distilbert-base-cased-distilled-squad": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
+    "distilbert-base-german-cased",
-        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
+    "distilbert-base-multilingual-cased",
-        "distilbert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
+    # See all DISTILBERT models at https://huggingface.co/models?filter=distilbert
-    }
+]
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "distilbert-base-uncased": 512,
-    "distilbert-base-uncased-distilled-squad": 512,
-    "distilbert-base-cased": 512,
-    "distilbert-base-cased-distilled-squad": 512,
-    "distilbert-base-german-cased": 512,
-    "distilbert-base-multilingual-cased": 512,
-}
-
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "distilbert-base-uncased": {"do_lower_case": True},
-    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
-    "distilbert-base-cased": {"do_lower_case": False},
-    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
-    "distilbert-base-german-cased": {"do_lower_case": False},
-    "distilbert-base-multilingual-cased": {"do_lower_case": False},
-}
 
 
 class DistilBertTokenizer(BertTokenizer):
@@ -65,7 +45,5 @@ class DistilBertTokenizer(BertTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
     model_input_names = ["attention_mask"]

src/transformers/models/distilbert/tokenization_distilbert_fast.py

@@ -23,43 +23,15 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+DISTILBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "distilbert-base-uncased",
-        "distilbert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    "distilbert-base-uncased-distilled-squad",
-        "distilbert-base-uncased-distilled-squad": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
+    "distilbert-base-cased",
-        "distilbert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
+    "distilbert-base-cased-distilled-squad",
-        "distilbert-base-cased-distilled-squad": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
+    "distilbert-base-german-cased",
-        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
+    "distilbert-base-multilingual-cased",
-        "distilbert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
+    # See all DISTILBERT models at https://huggingface.co/models?filter=distilbert
-    },
+]
-    "tokenizer_file": {
-        "distilbert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-        "distilbert-base-uncased-distilled-squad": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
-        "distilbert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
-        "distilbert-base-cased-distilled-squad": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
-        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json",
-        "distilbert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json",
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "distilbert-base-uncased": 512,
-    "distilbert-base-uncased-distilled-squad": 512,
-    "distilbert-base-cased": 512,
-    "distilbert-base-cased-distilled-squad": 512,
-    "distilbert-base-german-cased": 512,
-    "distilbert-base-multilingual-cased": 512,
-}
-
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "distilbert-base-uncased": {"do_lower_case": True},
-    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
-    "distilbert-base-cased": {"do_lower_case": False},
-    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
-    "distilbert-base-german-cased": {"do_lower_case": False},
-    "distilbert-base-multilingual-cased": {"do_lower_case": False},
-}
 
 
 class DistilBertTokenizerFast(BertTokenizerFast):
@@ -74,8 +46,6 @@ class DistilBertTokenizerFast(BertTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
     model_input_names = ["attention_mask"]
     slow_tokenizer_class = DistilBertTokenizer

src/transformers/models/dpr/__init__.py

@@ -9,14 +9,19 @@ from .tokenization_dpr import (
     DPRQuestionEncoderTokenizer,
     DPRReaderOutput,
     DPRReaderTokenizer,
+    DPR_CONTEXT_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+    DPR_QUESTION_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+    DPR_READER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 )
 
-
 if is_tokenizers_available():
     from .tokenization_dpr_fast import (
         DPRContextEncoderTokenizerFast,
         DPRQuestionEncoderTokenizerFast,
         DPRReaderTokenizerFast,
+        DPR_CONTEXT_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+        DPR_QUESTION_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+        DPR_READER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     )
 
 if is_torch_available():

src/transformers/models/dpr/tokenization_dpr.py

@@ -28,63 +28,23 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
-CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP = {
+DPR_CONTEXT_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "facebook/dpr-ctx_encoder-single-nq-base",
-        "facebook/dpr-ctx_encoder-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    "facebook/dpr-ctx_encoder-multiset-base",
-        "facebook/dpr-ctx_encoder-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    # See all DPR models at https://huggingface.co/models?filter=dpr
-    },
+]
-    "tokenizer_file": {
-        "facebook/dpr-ctx_encoder-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-        "facebook/dpr-ctx_encoder-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-    },
-}
-QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {
-        "facebook/dpr-question_encoder-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-        "facebook/dpr-question_encoder-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-    },
-    "tokenizer_file": {
-        "facebook/dpr-question_encoder-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-        "facebook/dpr-question_encoder-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-    },
-}
-READER_PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {
-        "facebook/dpr-reader-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-        "facebook/dpr-reader-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-    },
-    "tokenizer_file": {
-        "facebook/dpr-reader-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-        "facebook/dpr-reader-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-    },
-}
 
-CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+DPR_QUESTION_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "facebook/dpr-ctx_encoder-single-nq-base": 512,
+    "facebook/dpr-question_encoder-single-nq-base",
-    "facebook/dpr-ctx_encoder-multiset-base": 512,
+    "facebook/dpr-question_encoder-multiset-base",
-}
+    # See all DPR models at https://huggingface.co/models?filter=dpr
-QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+]
-    "facebook/dpr-question_encoder-single-nq-base": 512,
-    "facebook/dpr-question_encoder-multiset-base": 512,
-}
-READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "facebook/dpr-reader-single-nq-base": 512,
-    "facebook/dpr-reader-multiset-base": 512,
-}
 
-
+DPR_READER_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION = {
+    "facebook/dpr-reader-single-nq-base",
-    "facebook/dpr-ctx_encoder-single-nq-base": {"do_lower_case": True},
+    "facebook/dpr-reader-multiset-base",
-    "facebook/dpr-ctx_encoder-multiset-base": {"do_lower_case": True},
+    # See all DPR models at https://huggingface.co/models?filter=dpr
-}
+]
-QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION = {
-    "facebook/dpr-question_encoder-single-nq-base": {"do_lower_case": True},
-    "facebook/dpr-question_encoder-multiset-base": {"do_lower_case": True},
-}
-READER_PRETRAINED_INIT_CONFIGURATION = {
-    "facebook/dpr-reader-single-nq-base": {"do_lower_case": True},
-    "facebook/dpr-reader-multiset-base": {"do_lower_case": True},
-}
 
 
 class DPRContextEncoderTokenizer(BertTokenizer):
@@ -99,9 +59,7 @@ class DPRContextEncoderTokenizer(BertTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION
 
 
 class DPRQuestionEncoderTokenizer(BertTokenizer):
@@ -116,9 +74,7 @@ class DPRQuestionEncoderTokenizer(BertTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
 
 
 DPRSpanPrediction = collections.namedtuple(
@@ -380,7 +336,5 @@ class DPRReaderTokenizer(CustomDPRReaderTokenizerMixin, BertTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = READER_PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = READER_PRETRAINED_INIT_CONFIGURATION
     model_input_names = ["attention_mask"]

src/transformers/models/dpr/tokenization_dpr_fast.py

@@ -29,63 +29,23 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
-CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP = {
+DPR_CONTEXT_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "facebook/dpr-ctx_encoder-single-nq-base",
-        "facebook/dpr-ctx_encoder-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    "facebook/dpr-ctx_encoder-multiset-base",
-        "facebook/dpr-ctx_encoder-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    # See all DPR models at https://huggingface.co/models?filter=dpr
-    },
+]
-    "tokenizer_file": {
-        "facebook/dpr-ctx_encoder-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-        "facebook/dpr-ctx_encoder-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-    },
-}
-QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {
-        "facebook/dpr-question_encoder-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-        "facebook/dpr-question_encoder-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-    },
-    "tokenizer_file": {
-        "facebook/dpr-question_encoder-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-        "facebook/dpr-question_encoder-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-    },
-}
-READER_PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {
-        "facebook/dpr-reader-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-        "facebook/dpr-reader-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-    },
-    "tokenizer_file": {
-        "facebook/dpr-reader-single-nq-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-        "facebook/dpr-reader-multiset-base": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-    },
-}
 
-CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+DPR_QUESTION_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "facebook/dpr-ctx_encoder-single-nq-base": 512,
+    "facebook/dpr-question_encoder-single-nq-base",
-    "facebook/dpr-ctx_encoder-multiset-base": 512,
+    "facebook/dpr-question_encoder-multiset-base",
-}
+    # See all DPR models at https://huggingface.co/models?filter=dpr
-QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+]
-    "facebook/dpr-question_encoder-single-nq-base": 512,
-    "facebook/dpr-question_encoder-multiset-base": 512,
-}
-READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "facebook/dpr-reader-single-nq-base": 512,
-    "facebook/dpr-reader-multiset-base": 512,
-}
 
-
+DPR_READER_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION = {
+    "facebook/dpr-reader-single-nq-base",
-    "facebook/dpr-ctx_encoder-single-nq-base": {"do_lower_case": True},
+    "facebook/dpr-reader-multiset-base",
-    "facebook/dpr-ctx_encoder-multiset-base": {"do_lower_case": True},
+    # See all DPR models at https://huggingface.co/models?filter=dpr
-}
+]
-QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION = {
-    "facebook/dpr-question_encoder-single-nq-base": {"do_lower_case": True},
-    "facebook/dpr-question_encoder-multiset-base": {"do_lower_case": True},
-}
-READER_PRETRAINED_INIT_CONFIGURATION = {
-    "facebook/dpr-reader-single-nq-base": {"do_lower_case": True},
-    "facebook/dpr-reader-multiset-base": {"do_lower_case": True},
-}
 
 
 class DPRContextEncoderTokenizerFast(BertTokenizerFast):
@@ -100,9 +60,7 @@ class DPRContextEncoderTokenizerFast(BertTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION
     slow_tokenizer_class = DPRContextEncoderTokenizer
 
 
@@ -118,9 +76,7 @@ class DPRQuestionEncoderTokenizerFast(BertTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
     slow_tokenizer_class = DPRQuestionEncoderTokenizer
 
 
@@ -382,8 +338,6 @@ class DPRReaderTokenizerFast(CustomDPRReaderTokenizerMixin, BertTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = READER_PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = READER_PRETRAINED_INIT_CONFIGURATION
     model_input_names = ["attention_mask"]
     slow_tokenizer_class = DPRReaderTokenizer

src/transformers/models/electra/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig
-from .tokenization_electra import ElectraTokenizer
+from .tokenization_electra import ElectraTokenizer, ELECTRA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_electra_fast import ElectraTokenizerFast
+    from .tokenization_electra_fast import ElectraTokenizerFast, ELECTRA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_electra import (

src/transformers/models/electra/tokenization_electra.py

@@ -18,35 +18,15 @@ from ..bert.tokenization_bert import BertTokenizer
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+ELECTRA_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "google/electra-small-generator",
-        "google/electra-small-generator": "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt",
+    "google/electra-base-generator",
-        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
+    "google/electra-large-generator",
-        "google/electra-large-generator": "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt",
+    "google/electra-small-discriminator",
-        "google/electra-small-discriminator": "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt",
+    "google/electra-base-discriminator",
-        "google/electra-base-discriminator": "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt",
+    "google/electra-large-discriminator",
-        "google/electra-large-discriminator": "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt",
+    # See all ELECTRA models at https://huggingface.co/models?filter=electra
-    }
+]
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "google/electra-small-generator": 512,
-    "google/electra-base-generator": 512,
-    "google/electra-large-generator": 512,
-    "google/electra-small-discriminator": 512,
-    "google/electra-base-discriminator": 512,
-    "google/electra-large-discriminator": 512,
-}
-
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "google/electra-small-generator": {"do_lower_case": True},
-    "google/electra-base-generator": {"do_lower_case": True},
-    "google/electra-large-generator": {"do_lower_case": True},
-    "google/electra-small-discriminator": {"do_lower_case": True},
-    "google/electra-base-discriminator": {"do_lower_case": True},
-    "google/electra-large-discriminator": {"do_lower_case": True},
-}
 
 
 class ElectraTokenizer(BertTokenizer):
@@ -61,6 +41,4 @@ class ElectraTokenizer(BertTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION

src/transformers/models/electra/tokenization_electra_fast.py

@@ -19,43 +19,15 @@ from .tokenization_electra import ElectraTokenizer
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+ELECTRA_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "google/electra-small-generator",
-        "google/electra-small-generator": "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt",
+    "google/electra-base-generator",
-        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
+    "google/electra-large-generator",
-        "google/electra-large-generator": "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt",
+    "google/electra-small-discriminator",
-        "google/electra-small-discriminator": "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt",
+    "google/electra-base-discriminator",
-        "google/electra-base-discriminator": "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt",
+    "google/electra-large-discriminator",
-        "google/electra-large-discriminator": "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt",
+    # See all ELECTRA models at https://huggingface.co/models?filter=electra
-    },
+]
-    "tokenizer_file": {
-        "google/electra-small-generator": "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json",
-        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json",
-        "google/electra-large-generator": "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json",
-        "google/electra-small-discriminator": "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json",
-        "google/electra-base-discriminator": "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json",
-        "google/electra-large-discriminator": "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json",
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "google/electra-small-generator": 512,
-    "google/electra-base-generator": 512,
-    "google/electra-large-generator": 512,
-    "google/electra-small-discriminator": 512,
-    "google/electra-base-discriminator": 512,
-    "google/electra-large-discriminator": 512,
-}
-
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "google/electra-small-generator": {"do_lower_case": True},
-    "google/electra-base-generator": {"do_lower_case": True},
-    "google/electra-large-generator": {"do_lower_case": True},
-    "google/electra-small-discriminator": {"do_lower_case": True},
-    "google/electra-base-discriminator": {"do_lower_case": True},
-    "google/electra-large-discriminator": {"do_lower_case": True},
-}
 
 
 class ElectraTokenizerFast(BertTokenizerFast):
@@ -69,7 +41,5 @@ class ElectraTokenizerFast(BertTokenizerFast):
     parameters.
     """
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
     slow_tokenizer_class = ElectraTokenizer

src/transformers/models/flaubert/__init__.py

@@ -4,7 +4,7 @@
 
 from ...file_utils import is_tf_available, is_torch_available
 from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig
-from .tokenization_flaubert import FlaubertTokenizer
+from .tokenization_flaubert import FlaubertTokenizer, FLAUBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_torch_available():

src/transformers/models/flaubert/tokenization_flaubert.py

@@ -30,34 +30,13 @@ VOCAB_FILES_NAMES = {
     "merges_file": "merges.txt",
 }
 
-PRETRAINED_VOCAB_FILES_MAP = {
+FLAUBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "flaubert/flaubert_small_cased",
-        "flaubert/flaubert_small_cased": "https://huggingface.co/flaubert/flaubert_small_cased/resolve/main/vocab.json",
+    "flaubert/flaubert_base_uncased",
-        "flaubert/flaubert_base_uncased": "https://huggingface.co/flaubert/flaubert_base_uncased/resolve/main/vocab.json",
+    "flaubert/flaubert_base_cased",
-        "flaubert/flaubert_base_cased": "https://huggingface.co/flaubert/flaubert_base_cased/resolve/main/vocab.json",
+    "flaubert/flaubert_large_cased",
-        "flaubert/flaubert_large_cased": "https://huggingface.co/flaubert/flaubert_large_cased/resolve/main/vocab.json",
+    # See all FLAUBERT models at https://huggingface.co/models?filter=flaubert
-    },
+]
-    "merges_file": {
-        "flaubert/flaubert_small_cased": "https://huggingface.co/flaubert/flaubert_small_cased/resolve/main/merges.txt",
-        "flaubert/flaubert_base_uncased": "https://huggingface.co/flaubert/flaubert_base_uncased/resolve/main/merges.txt",
-        "flaubert/flaubert_base_cased": "https://huggingface.co/flaubert/flaubert_base_cased/resolve/main/merges.txt",
-        "flaubert/flaubert_large_cased": "https://huggingface.co/flaubert/flaubert_large_cased/resolve/main/merges.txt",
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "flaubert/flaubert_small_cased": 512,
-    "flaubert/flaubert_base_uncased": 512,
-    "flaubert/flaubert_base_cased": 512,
-    "flaubert/flaubert_large_cased": 512,
-}
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "flaubert/flaubert_small_cased": {"do_lowercase": False},
-    "flaubert/flaubert_base_uncased": {"do_lowercase": True},
-    "flaubert/flaubert_base_cased": {"do_lowercase": False},
-    "flaubert/flaubert_large_cased": {"do_lowercase": False},
-}
 
 
 def convert_to_unicode(text):
@@ -91,9 +70,7 @@ class FlaubertTokenizer(XLMTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(self, do_lowercase=False, **kwargs):
         super().__init__(**kwargs)

src/transformers/models/fsmt/__init__.py

@@ -4,7 +4,7 @@
 
 from ...file_utils import is_torch_available
 from .configuration_fsmt import FSMT_PRETRAINED_CONFIG_ARCHIVE_MAP, FSMTConfig
-from .tokenization_fsmt import FSMTTokenizer
+from .tokenization_fsmt import FSMTTokenizer, FSMT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_torch_available():

src/transformers/models/fsmt/tokenization_fsmt.py

@@ -37,21 +37,10 @@ VOCAB_FILES_NAMES = {
     "merges_file": "merges.txt",
 }
 
-PRETRAINED_VOCAB_FILES_MAP = {
+FSMT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "src_vocab_file": {"stas/tiny-wmt19-en-de": "https://cdn.huggingface.co/stas/tiny-wmt19-en-de/vocab-src.json"},
+    "stas/tiny-wmt19-en-de",
-    "tgt_vocab_file": {"stas/tiny-wmt19-en-de": "https://cdn.huggingface.co/stas/tiny-wmt19-en-de/vocab-tgt.json"},
+    # See all FSMT models at https://huggingface.co/models?filter=fsmt
-    "merges_file": {"stas/tiny-wmt19-en-de": "https://cdn.huggingface.co/stas/tiny-wmt19-en-de/merges.txt"},
+]
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"stas/tiny-wmt19-en-de": 1024}
-PRETRAINED_INIT_CONFIGURATION = {
-    "stas/tiny-wmt19-en-de": {
-        "langs": ["en", "de"],
-        "model_max_length": 1024,
-        "special_tokens_map_file": None,
-        "full_tokenizer_file": None,
-    }
-}
 
 
 def get_pairs(word):
@@ -176,9 +165,7 @@ class FSMTTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 1024
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(
         self,

src/transformers/models/funnel/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
-from .tokenization_funnel import FunnelTokenizer
+from .tokenization_funnel import FunnelTokenizer, FUNNEL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_funnel_fast import FunnelTokenizerFast
+    from .tokenization_funnel_fast import FunnelTokenizerFast, FUNNEL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_funnel import (

src/transformers/models/funnel/tokenization_funnel.py

@@ -24,36 +24,20 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
 
-_model_names = [
+FUNNEL_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "small",
+    "funnel-transformer/small",
-    "small-base",
+    "funnel-transformer/small-base",
-    "medium",
+    "funnel-transformer/medium",
-    "medium-base",
+    "funnel-transformer/medium-base",
-    "intermediate",
+    "funnel-transformer/intermediate",
-    "intermediate-base",
+    "funnel-transformer/intermediate-base",
-    "large",
+    "funnel-transformer/large",
-    "large-base",
+    "funnel-transformer/large-base",
-    "xlarge",
+    "funnel-transformer/xlarge",
-    "xlarge-base",
+    "funnel-transformer/xlarge-base",
+    # See all FUNNEL models at https://huggingface.co/models?filter=funnel
 ]
 
-PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {
-        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
-        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
-        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
-        "funnel-transformer/medium-base": "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt",
-        "funnel-transformer/intermediate": "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt",
-        "funnel-transformer/intermediate-base": "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt",
-        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
-        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
-        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
-        "funnel-transformer/xlarge-base": "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt",
-    }
-}
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {f"funnel-transformer/{name}": 512 for name in _model_names}
-PRETRAINED_INIT_CONFIGURATION = {f"funnel-transformer/{name}": {"do_lower_case": True} for name in _model_names}
-
 
 class FunnelTokenizer(BertTokenizer):
     r"""
@@ -67,9 +51,7 @@ class FunnelTokenizer(BertTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
     cls_token_type_id: int = 2
 
     def __init__(

src/transformers/models/funnel/tokenization_funnel_fast.py

@@ -25,48 +25,20 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
-_model_names = [
+FUNNEL_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "small",
+    "funnel-transformer/small",
-    "small-base",
+    "funnel-transformer/small-base",
-    "medium",
+    "funnel-transformer/medium",
-    "medium-base",
+    "funnel-transformer/medium-base",
-    "intermediate",
+    "funnel-transformer/intermediate",
-    "intermediate-base",
+    "funnel-transformer/intermediate-base",
-    "large",
+    "funnel-transformer/large",
-    "large-base",
+    "funnel-transformer/large-base",
-    "xlarge",
+    "funnel-transformer/xlarge",
-    "xlarge-base",
+    "funnel-transformer/xlarge-base",
+    # See all FUNNEL models at https://huggingface.co/models?filter=funnel
 ]
 
-PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {
-        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
-        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
-        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
-        "funnel-transformer/medium-base": "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt",
-        "funnel-transformer/intermediate": "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt",
-        "funnel-transformer/intermediate-base": "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt",
-        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
-        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
-        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
-        "funnel-transformer/xlarge-base": "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt",
-    },
-    "tokenizer_file": {
-        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
-        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json",
-        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
-        "funnel-transformer/medium-base": "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json",
-        "funnel-transformer/intermediate": "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json",
-        "funnel-transformer/intermediate-base": "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json",
-        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
-        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json",
-        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
-        "funnel-transformer/xlarge-base": "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json",
-    },
-}
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {f"funnel-transformer/{name}": 512 for name in _model_names}
-PRETRAINED_INIT_CONFIGURATION = {f"funnel-transformer/{name}": {"do_lower_case": True} for name in _model_names}
-
 
 class FunnelTokenizerFast(BertTokenizerFast):
     r"""
@@ -80,9 +52,7 @@ class FunnelTokenizerFast(BertTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
     slow_tokenizer_class = FunnelTokenizer
     cls_token_type_id: int = 2
 

src/transformers/models/gpt2/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config
-from .tokenization_gpt2 import GPT2Tokenizer
+from .tokenization_gpt2 import GPT2Tokenizer, GPT2_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_gpt2_fast import GPT2TokenizerFast
+    from .tokenization_gpt2_fast import GPT2TokenizerFast, GPT2_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_gpt2 import (

src/transformers/models/gpt2/tokenization_gpt2.py

@@ -33,30 +33,14 @@ VOCAB_FILES_NAMES = {
     "merges_file": "merges.txt",
 }
 
-PRETRAINED_VOCAB_FILES_MAP = {
+GPT2_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "gpt2",
-        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
+    "gpt2-medium",
-        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
+    "gpt2-large",
-        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
+    "gpt2-xl",
-        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
+    "distilgpt2",
-        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
+    # See all GPT2 models at https://huggingface.co/models?filter=gpt2
-    },
+]
-    "merges_file": {
-        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
-        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
-        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
-        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
-        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "gpt2": 1024,
-    "gpt2-medium": 1024,
-    "gpt2-large": 1024,
-    "gpt2-xl": 1024,
-    "distilgpt2": 1024,
-}
 
 
 @lru_cache()
@@ -146,8 +130,7 @@ class GPT2Tokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 1024
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     model_input_names = ["attention_mask"]
 
     def __init__(

src/transformers/models/gpt2/tokenization_gpt2_fast.py

@@ -30,37 +30,14 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+GPT2_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "gpt2",
-        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
+    "gpt2-medium",
-        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
+    "gpt2-large",
-        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
+    "gpt2-xl",
-        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
+    "distilgpt2",
-        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
+    # See all GPT2 models at https://huggingface.co/models?filter=gpt2
-    },
+]
-    "merges_file": {
-        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
-        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
-        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
-        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
-        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
-    },
-    "tokenizer_file": {
-        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
-        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
-        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
-        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
-        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "gpt2": 1024,
-    "gpt2-medium": 1024,
-    "gpt2-large": 1024,
-    "gpt2-xl": 1024,
-    "distilgpt2": 1024,
-}
 
 
 class GPT2TokenizerFast(PreTrainedTokenizerFast):
@@ -114,8 +91,7 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 1024
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     model_input_names = ["attention_mask"]
     slow_tokenizer_class = GPT2Tokenizer
 

src/transformers/models/herbert/__init__.py

@@ -3,8 +3,8 @@
 # module, but to preserve other warnings. So, don't check this module at all.
 
 from ...file_utils import is_tokenizers_available
-from .tokenization_herbert import HerbertTokenizer
+from .tokenization_herbert import HerbertTokenizer, HERBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_herbert_fast import HerbertTokenizerFast
+    from .tokenization_herbert_fast import HerbertTokenizerFast, HERBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST

src/transformers/models/herbert/tokenization_herbert.py

@@ -25,13 +25,10 @@ VOCAB_FILES_NAMES = {
     "merges_file": "merges.txt",
 }
 
-PRETRAINED_VOCAB_FILES_MAP = {
+HERBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {"allegro/herbert-base-cased": "https://cdn.huggingface.co/allegro/herbert-base-cased/vocab.json"},
+    "allegro/herbert-base-cased",
-    "merges_file": {"allegro/herbert-base-cased": "https://cdn.huggingface.co/allegro/herbert-base-cased/merges.txt"},
+    # See all HERBERT models at https://huggingface.co/models?filter=herbert
-}
+]
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"allegro/herbert-base-cased": 514}
-PRETRAINED_INIT_CONFIGURATION = {}
 
 
 class HerbertTokenizer(XLMTokenizer):
@@ -50,9 +47,7 @@ class HerbertTokenizer(XLMTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 514
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(self, **kwargs):
 

src/transformers/models/herbert/tokenization_herbert_fast.py

@@ -18,9 +18,6 @@ from typing import List, Optional, Tuple
 from ...tokenization_utils_fast import PreTrainedTokenizerFast
 from ...utils import logging
 from .tokenization_herbert import (
-    PRETRAINED_INIT_CONFIGURATION,
-    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES,
-    PRETRAINED_VOCAB_FILES_MAP,
     HerbertTokenizer,
 )
 
@@ -32,6 +29,10 @@ VOCAB_FILES_NAMES = {
     "merges_file": "merges.txt",
 }
 
+HERBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
+    "allegro/herbert-base-cased",
+    # See all HERBERT models at https://huggingface.co/models?filter=herbert
+]
 
 class HerbertTokenizerFast(PreTrainedTokenizerFast):
     """
@@ -53,9 +54,7 @@ class HerbertTokenizerFast(PreTrainedTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 514
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     slow_tokenizer_class = HerbertTokenizer
 
     def __init__(self, vocab_file, merges_file, tokenizer_file=None, **kwargs):

src/transformers/models/layoutlm/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tokenizers_available, is_torch_available
 from .configuration_layoutlm import LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMConfig
-from .tokenization_layoutlm import LayoutLMTokenizer
+from .tokenization_layoutlm import LayoutLMTokenizer, LAYOUTLM_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_layoutlm_fast import LayoutLMTokenizerFast
+    from .tokenization_layoutlm_fast import LayoutLMTokenizerFast, LAYOUTLM_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_layoutlm import (

src/transformers/models/layoutlm/tokenization_layoutlm.py

@@ -23,25 +23,11 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+LAYOUTLM_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "microsoft/layoutlm-base-uncased",
-        "microsoft/layoutlm-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    "microsoft/layoutlm-large-uncased",
-        "microsoft/layoutlm-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
+    # See all LAYOUTLM models at https://huggingface.co/models?filter=layoutlm
-    }
+]
-}
-
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "microsoft/layoutlm-base-uncased": 512,
-    "microsoft/layoutlm-large-uncased": 512,
-}
-
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
-    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
-}
-
 
 class LayoutLMTokenizer(BertTokenizer):
     r"""
@@ -55,6 +41,4 @@ class LayoutLMTokenizer(BertTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

src/transformers/models/layoutlm/tokenization_layoutlm_fast.py

@@ -24,29 +24,11 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+LAYOUTLM_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {
+    "microsoft/layoutlm-base-uncased",
-        "microsoft/layoutlm-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
+    "microsoft/layoutlm-large-uncased",
-        "microsoft/layoutlm-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
+    # See all LAYOUTLM models at https://huggingface.co/models?filter=layoutlm
-    },
+]
-    "tokenizer_file": {
-        "microsoft/layoutlm-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-        "microsoft/layoutlm-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
-    },
-}
-
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "microsoft/layoutlm-base-uncased": 512,
-    "microsoft/layoutlm-large-uncased": 512,
-}
-
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
-    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
-}
-
 
 class LayoutLMTokenizerFast(BertTokenizerFast):
     r"""
@@ -60,7 +42,5 @@ class LayoutLMTokenizerFast(BertTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
     slow_tokenizer_class = LayoutLMTokenizer

src/transformers/models/longformer/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig
-from .tokenization_longformer import LongformerTokenizer
+from .tokenization_longformer import LongformerTokenizer, LONGFORMER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_longformer_fast import LongformerTokenizerFast
+    from .tokenization_longformer_fast import LongformerTokenizerFast, LONGFORMER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_longformer import (

src/transformers/models/longformer/tokenization_longformer.py

@@ -20,27 +20,15 @@ from ..roberta.tokenization_roberta import RobertaTokenizer
 logger = logging.get_logger(__name__)
 
 
-# vocab and merges same as roberta
+LONGFORMER_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-vocab_url = "https://huggingface.co/roberta-large/resolve/main/vocab.json"
-merges_url = "https://huggingface.co/roberta-large/resolve/main/merges.txt"
-_all_longformer_models = [
     "allenai/longformer-base-4096",
     "allenai/longformer-large-4096",
     "allenai/longformer-large-4096-finetuned-triviaqa",
     "allenai/longformer-base-4096-extra.pos.embd.only",
     "allenai/longformer-large-4096-extra.pos.embd.only",
+    # See all LONGFORMER models at https://huggingface.co/models?filter=longformer
 ]
 
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "allenai/longformer-base-4096": 4096,
-    "allenai/longformer-large-4096": 4096,
-    "allenai/longformer-large-4096-finetuned-triviaqa": 4096,
-    "allenai/longformer-base-4096-extra.pos.embd.only": 4096,
-    "allenai/longformer-large-4096-extra.pos.embd.only": 4096,
-}
-
-
 class LongformerTokenizer(RobertaTokenizer):
     r"""
     Construct a Longformer tokenizer.
@@ -49,8 +37,4 @@ class LongformerTokenizer(RobertaTokenizer):
     superclass for usage examples and documentation concerning parameters.
     """
     # merges and vocab same as Roberta
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    max_model_input_sizes = 4096
-    pretrained_vocab_files_map = {
-        "vocab_file": {m: vocab_url for m in _all_longformer_models},
-        "merges_file": {m: merges_url for m in _all_longformer_models},
-    }

src/transformers/models/longformer/tokenization_longformer_fast.py

@@ -21,28 +21,16 @@ from .tokenization_longformer import LongformerTokenizer
 logger = logging.get_logger(__name__)
 
 
-# vocab and merges same as roberta
+LONGFORMER_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-vocab_url = "https://huggingface.co/roberta-large/resolve/main/vocab.json"
-merges_url = "https://huggingface.co/roberta-large/resolve/main/merges.txt"
-tokenizer_url = "https://huggingface.co/roberta-large/resolve/main/tokenizer.json"
-_all_longformer_models = [
     "allenai/longformer-base-4096",
     "allenai/longformer-large-4096",
     "allenai/longformer-large-4096-finetuned-triviaqa",
     "allenai/longformer-base-4096-extra.pos.embd.only",
     "allenai/longformer-large-4096-extra.pos.embd.only",
+    # See all LONGFORMER models at https://huggingface.co/models?filter=longformer
 ]
 
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "allenai/longformer-base-4096": 4096,
-    "allenai/longformer-large-4096": 4096,
-    "allenai/longformer-large-4096-finetuned-triviaqa": 4096,
-    "allenai/longformer-base-4096-extra.pos.embd.only": 4096,
-    "allenai/longformer-large-4096-extra.pos.embd.only": 4096,
-}
-
-
 class LongformerTokenizerFast(RobertaTokenizerFast):
     r"""
     Construct a "fast" Longformer tokenizer (backed by HuggingFace's `tokenizers` library).
@@ -51,10 +39,5 @@ class LongformerTokenizerFast(RobertaTokenizerFast):
     to the superclass for usage examples and documentation concerning parameters.
     """
     # merges and vocab same as Roberta
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    max_model_input_sizes = 4096
-    pretrained_vocab_files_map = {
-        "vocab_file": {m: vocab_url for m in _all_longformer_models},
-        "merges_file": {m: merges_url for m in _all_longformer_models},
-        "tokenizer_file": {m: tokenizer_url for m in _all_longformer_models},
-    }
     slow_tokenizer_class = LongformerTokenizer

src/transformers/models/lxmert/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
-from .tokenization_lxmert import LxmertTokenizer
+from .tokenization_lxmert import LxmertTokenizer, LXMERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_lxmert_fast import LxmertTokenizerFast
+    from .tokenization_lxmert_fast import LxmertTokenizerFast, LXMERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_lxmert import (

src/transformers/models/lxmert/tokenization_lxmert.py

@@ -16,36 +16,12 @@
 from ..bert.tokenization_bert import BertTokenizer
 
 
-####################################################
-# Mapping from the keyword arguments names of Tokenizer `__init__`
-# to file names for serializing Tokenizer instances
-####################################################
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
 
-####################################################
+LXMERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-# Mapping from the keyword arguments names of Tokenizer `__init__`
+    "unc-nlp/lxmert-base-uncased",
-# to pretrained vocabulary URL for all the model ids.
+    # See all LXMERT models at https://huggingface.co/models?filter=lxmert
-####################################################
+]
-PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {
-        "unc-nlp/lxmert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-    }
-}
-
-####################################################
-# Mapping from model ids to max length of inputs
-####################################################
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "unc-nlp/lxmert-base-uncased": 512,
-}
-####################################################
-# Mapping from model ids to a dictionary of additional
-# keyword arguments for Tokenizer `__init__`.
-# To be used for checkpoint specific configurations.
-####################################################
-PRETRAINED_INIT_CONFIGURATION = {
-    "unc-nlp/lxmert-base-uncased": {"do_lower_case": True},
-}
 
 
 class LxmertTokenizer(BertTokenizer):
@@ -60,6 +36,4 @@ class LxmertTokenizer(BertTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION

src/transformers/models/lxmert/tokenization_lxmert_fast.py

@@ -17,39 +17,12 @@ from ..bert.tokenization_bert_fast import BertTokenizerFast
 from .tokenization_lxmert import LxmertTokenizer
 
 
-####################################################
-# Mapping from the keyword arguments names of Tokenizer `__init__`
-# to file names for serializing Tokenizer instances
-####################################################
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
-####################################################
+LXMERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-# Mapping from the keyword arguments names of Tokenizer `__init__`
+    "unc-nlp/lxmert-base-uncased",
-# to pretrained vocabulary URL for all the model ids.
+    # See all LXMERT models at https://huggingface.co/models?filter=lxmert
-####################################################
+]
-PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {
-        "unc-nlp/lxmert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
-    },
-    "tokenizer_file": {
-        "unc-nlp/lxmert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
-    },
-}
-
-####################################################
-# Mapping from model ids to max length of inputs
-####################################################
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "unc-nlp/lxmert-base-uncased": 512,
-}
-####################################################
-# Mapping from model ids to a dictionary of additional
-# keyword arguments for Tokenizer `__init__`.
-# To be used for checkpoint specific configurations.
-####################################################
-PRETRAINED_INIT_CONFIGURATION = {
-    "unc-nlp/lxmert-base-uncased": {"do_lower_case": True},
-}
 
 
 class LxmertTokenizerFast(BertTokenizerFast):
@@ -63,7 +36,5 @@ class LxmertTokenizerFast(BertTokenizerFast):
     parameters.
     """
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
     slow_tokenizer_class = LxmertTokenizer

src/transformers/models/marian/__init__.py

@@ -7,7 +7,7 @@ from .configuration_marian import MarianConfig
 
 
 if is_sentencepiece_available():
-    from .tokenization_marian import MarianTokenizer
+    from .tokenization_marian import MarianTokenizer, MARIAN_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_marian import MarianMTModel

src/transformers/models/marian/tokenization_marian.py

@@ -16,22 +16,12 @@ vocab_files_names = {
     "source_spm": "source.spm",
     "target_spm": "target.spm",
     "vocab": "vocab.json",
-    "tokenizer_config_file": "tokenizer_config.json",
 }
 
-PRETRAINED_VOCAB_FILES_MAP = {
+MARIAN_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "source_spm": {"Helsinki-NLP/opus-mt-en-de": "https://cdn.huggingface.co/Helsinki-NLP/opus-mt-en-de/source.spm"},
+    "Helsinki-NLP/opus-mt-en-de",
-    "target_spm": {"Helsinki-NLP/opus-mt-en-de": "https://cdn.huggingface.co/Helsinki-NLP/opus-mt-en-de/target.spm"},
+    # See all MARIAN models at https://huggingface.co/models?filter=marian
-    "vocab": {"Helsinki-NLP/opus-mt-en-de": "https://cdn.huggingface.co/Helsinki-NLP/opus-mt-en-de/vocab.json"},
+]
-    "tokenizer_config_file": {
-        "Helsinki-NLP/opus-mt-en-de": "https://cdn.huggingface.co/Helsinki-NLP/opus-mt-en-de/tokenizer_config.json"
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"Helsinki-NLP/opus-mt-en-de": 512}
-PRETRAINED_INIT_CONFIGURATION = {}
-
-# Example URL https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json
 
 
 class MarianTokenizer(PreTrainedTokenizer):
@@ -76,9 +66,7 @@ class MarianTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = vocab_files_names
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     model_input_names = ["attention_mask"]
     language_code_re = re.compile(">>.+<<")  # type: re.Pattern
 

src/transformers/models/mbart/__init__.py

@@ -7,10 +7,10 @@ from .configuration_mbart import MBartConfig
 
 
 if is_sentencepiece_available():
-    from .tokenization_mbart import MBartTokenizer
+    from .tokenization_mbart import MBartTokenizer, MBART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_tokenizers_available():
-    from .tokenization_mbart_fast import MBartTokenizerFast
+    from .tokenization_mbart_fast import MBartTokenizerFast, MBART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_mbart import MBartForConditionalGeneration

src/transformers/models/mbart/tokenization_mbart.py

@@ -24,8 +24,11 @@ from ..xlm_roberta.tokenization_xlm_roberta import XLMRobertaTokenizer
 
 logger = logging.get_logger(__name__)
 
-_all_mbart_models = ["facebook/mbart-large-en-ro", "facebook/mbart-large-cc25"]
+MBART_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-SPM_URL = "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentence.bpe.model"
+    "facebook/mbart-large-en-ro",
+    "facebook/mbart-large-cc25"
+    # See all MBART models at https://huggingface.co/models?filter=mbart
+]
 
 FAIRSEQ_LANGUAGE_CODES = [
     "ar_AR",
@@ -87,8 +90,7 @@ class MBartTokenizer(XLMRobertaTokenizer):
     """
 
     vocab_files_names = {"vocab_file": "sentencepiece.bpe.model"}
-    max_model_input_sizes = {m: 1024 for m in _all_mbart_models}
+    max_model_input_sizes = 1024
-    pretrained_vocab_files_map = {"vocab_file": {m: SPM_URL for m in _all_mbart_models}}
 
     prefix_tokens: List[int] = []
     suffix_tokens: List[int] = []

src/transformers/models/mbart/tokenization_mbart_fast.py

@@ -32,9 +32,11 @@ else:
 
 logger = logging.get_logger(__name__)
 
-_all_mbart_models = ["facebook/mbart-large-en-ro", "facebook/mbart-large-cc25"]
+MBART_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-SPM_URL = "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentence.bpe.model"
+    "facebook/mbart-large-en-ro",
-tokenizer_URL = "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json"
+    "facebook/mbart-large-cc25"
+    # See all MBART models at https://huggingface.co/models?filter=mbart
+]
 
 FAIRSEQ_LANGUAGE_CODES = [
     "ar_AR",
@@ -94,8 +96,7 @@ class MBartTokenizerFast(XLMRobertaTokenizerFast):
     """
 
     vocab_files_names = {"vocab_file": "sentencepiece.bpe.model"}
-    max_model_input_sizes = {m: 1024 for m in _all_mbart_models}
+    max_model_input_sizes = 1024
-    pretrained_vocab_files_map = {"vocab_file": {m: SPM_URL for m in _all_mbart_models}}
     slow_tokenizer_class = MBartTokenizer
 
     prefix_tokens: List[int] = []

src/transformers/models/mobilebert/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig
-from .tokenization_mobilebert import MobileBertTokenizer
+from .tokenization_mobilebert import MobileBertTokenizer, MOBILEBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_mobilebert_fast import MobileBertTokenizerFast
+    from .tokenization_mobilebert_fast import MobileBertTokenizerFast, MOBILEBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_mobilebert import (

src/transformers/models/mobilebert/tokenization_mobilebert.py

@@ -21,15 +21,10 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+MOBILEBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {"mobilebert-uncased": "https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"}
+    "google/mobilebert-uncased",
-}
+    # See all MOBILEBERT models at https://huggingface.co/models?search=mobilebert
-
+]
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"mobilebert-uncased": 512}
-
-
-PRETRAINED_INIT_CONFIGURATION = {}
-
 
 class MobileBertTokenizer(BertTokenizer):
     r"""
@@ -43,6 +38,4 @@ class MobileBertTokenizer(BertTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION

src/transformers/models/mobilebert/tokenization_mobilebert_fast.py

@@ -22,17 +22,10 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+MOBILEBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {"mobilebert-uncased": "https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"},
+    "google/mobilebert-uncased",
-    "tokenizer_file": {
+    # See all MOBILEBERT models at https://huggingface.co/models?search=mobilebert
-        "mobilebert-uncased": "https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"
+]
-    },
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"mobilebert-uncased": 512}
-
-
-PRETRAINED_INIT_CONFIGURATION = {}
 
 
 class MobileBertTokenizerFast(BertTokenizerFast):
@@ -47,7 +40,5 @@ class MobileBertTokenizerFast(BertTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
     slow_tokenizer_class = MobileBertTokenizer

src/transformers/models/openai/__init__.py

@@ -4,11 +4,11 @@
 
 from ...file_utils import is_tf_available, is_tokenizers_available, is_torch_available
 from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
-from .tokenization_openai import OpenAIGPTTokenizer
+from .tokenization_openai import OpenAIGPTTokenizer, OPENAIGPT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 
 if is_tokenizers_available():
-    from .tokenization_openai_fast import OpenAIGPTTokenizerFast
+    from .tokenization_openai_fast import OpenAIGPTTokenizerFast, OPENAIGPT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 if is_torch_available():
     from .modeling_openai import (

src/transformers/models/openai/tokenization_openai.py

@@ -32,14 +32,10 @@ VOCAB_FILES_NAMES = {
     "merges_file": "merges.txt",
 }
 
-PRETRAINED_VOCAB_FILES_MAP = {
+OPENAIGPT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
+    "openai-gpt",
-    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
+    # See all OPENAIGPT models at https://huggingface.co/models?filter=openai-gpt
-}
+]
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "openai-gpt": 512,
-}
 
 
 def get_pairs(word):
@@ -92,8 +88,7 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     model_input_names = ["attention_mask"]
 
     def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):

src/transformers/models/openai/tokenization_openai_fast.py

@@ -26,15 +26,10 @@ logger = logging.get_logger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}
 
-PRETRAINED_VOCAB_FILES_MAP = {
+OPENAIGPT_PRETRAINED_TOKENIZER_ARCHIVE_LIST = [
-    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
+    "openai-gpt",
-    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
+    # See all OPENAIGPT models at https://huggingface.co/models?filter=openai-gpt
-    "tokenizer_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/tokenizer.json"},
+]
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "openai-gpt": 512,
-}
 
 
 class OpenAIGPTTokenizerFast(PreTrainedTokenizerFast):
@@ -59,8 +54,7 @@ class OpenAIGPTTokenizerFast(PreTrainedTokenizerFast):
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = 512
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     model_input_names = ["attention_mask"]
     slow_tokenizer_class = OpenAIGPTTokenizer
 

src/transformers/tokenization_utils.py

@@ -21,7 +21,7 @@ import re
 import unicodedata
 from typing import Any, Dict, List, Optional, Tuple, Union, overload
 
-from .file_utils import add_end_docstrings
+from .file_utils import add_end_docstrings, TOKENIZER_CONFIG_NAME
 from .tokenization_utils_base import (
     ENCODE_KWARGS_DOCSTRING,
     ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING,
@@ -33,7 +33,7 @@ from .tokenization_utils_base import (
     PaddingStrategy,
     PreTokenizedInput,
     PreTokenizedInputPair,
-    PreTrainedTokenizerBase,
+    PreTrainedTokenizerBase, TOKENIZER_CONFIG_FILE,
     TensorType,
     TextInput,
     TextInputPair,
@@ -47,8 +47,7 @@ logger = logging.get_logger(__name__)
 # Slow tokenizers are saved in a vocabulary plus three separated files
 SPECIAL_TOKENS_MAP_FILE = "special_tokens_map.json"
 ADDED_TOKENS_FILE = "added_tokens.json"
-TOKENIZER_CONFIG_FILE = "tokenizer_config.json"
+TOKENIZER_CONFIG_FILE = TOKENIZER_CONFIG_NAME  # Backward compatibility
-
 
 def _is_whitespace(char):
     """Checks whether `char` is a whitespace character."""

src/transformers/tokenization_utils_base.py

@@ -1515,9 +1515,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
     """
 
     vocab_files_names: Dict[str, str] = {}
-    pretrained_vocab_files_map: Dict[str, Dict[str, str]] = {}
+    max_model_input_sizes: Optional[int] = None
-    pretrained_init_configuration: Dict[str, Dict[str, Any]] = {}
-    max_model_input_sizes: Dict[str, Optional[int]] = {}
     model_input_names: List[str] = ["token_type_ids", "attention_mask"]
     padding_side: str = "right"
     slow_tokenizer_class = None
@@ -1526,6 +1524,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
         # inputs and kwargs for saving and re-loading (see ``from_pretrained`` and ``save_pretrained``)
         self.init_inputs = ()
         self.init_kwargs = copy.deepcopy(kwargs)
+        self.init_kwargs["tokenizer_class_name"] = self.__class__.__name__  # Used by AutoTokenizer to rebuild the tokenizer
+
         self.name_or_path = kwargs.pop("name_or_path", "")
 
         # For backward compatibility we fallback to set model_max_length from max_len if provided
@@ -1682,68 +1682,48 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
         revision = kwargs.pop("revision", None)
         subfolder = kwargs.pop("subfolder", None)
 
-        s3_models = list(cls.max_model_input_sizes.keys())
         vocab_files = {}
         init_configuration = {}
-        if pretrained_model_name_or_path in s3_models:
+        if os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
-            # Get the vocabulary from AWS S3 bucket
+            if len(cls.vocab_files_names) > 1:
-            for file_id, map_list in cls.pretrained_vocab_files_map.items():
+                raise ValueError(
-                vocab_files[file_id] = map_list[pretrained_model_name_or_path]
+                    "Calling {}.from_pretrained() with the path to a single file or url is not supported."
-            if (
+                    "Use a model identifier or the path to a directory instead.".format(cls.__name__)
-                cls.pretrained_init_configuration
+                )
-                and pretrained_model_name_or_path in cls.pretrained_init_configuration
+            logger.warning(
-            ):
+                "Calling {}.from_pretrained() with the path to a single file or url is deprecated".format(
-                init_configuration = cls.pretrained_init_configuration[pretrained_model_name_or_path].copy()
+                    cls.__name__
-        else:
-            # Get the vocabulary from local files
-            logger.info(
-                "Model name '{}' not found in model shortcut name list ({}). "
-                "Assuming '{}' is a path, a model identifier, or url to a directory containing tokenizer files.".format(
-                    pretrained_model_name_or_path, ", ".join(s3_models), pretrained_model_name_or_path
                 )
             )
-
+            file_id = list(cls.vocab_files_names.keys())[0]
-            if os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
+            vocab_files[file_id] = pretrained_model_name_or_path
-                if len(cls.vocab_files_names) > 1:
+        else:
-                    raise ValueError(
+            additional_files_names = {
-                        "Calling {}.from_pretrained() with the path to a single file or url is not supported."
+                "added_tokens_file": ADDED_TOKENS_FILE,
-                        "Use a model identifier or the path to a directory instead.".format(cls.__name__)
+                "special_tokens_map_file": SPECIAL_TOKENS_MAP_FILE,
-                    )
+                "tokenizer_config_file": TOKENIZER_CONFIG_FILE,
-                logger.warning(
+                "tokenizer_file": FULL_TOKENIZER_FILE,
-                    "Calling {}.from_pretrained() with the path to a single file or url is deprecated".format(
+            }
-                        cls.__name__
+            # Look for the tokenizer files
-                    )
+            for file_id, file_name in {**cls.vocab_files_names, **additional_files_names}.items():
-                )
+                if os.path.isdir(pretrained_model_name_or_path):
-                file_id = list(cls.vocab_files_names.keys())[0]
+                    if subfolder is not None:
-                vocab_files[file_id] = pretrained_model_name_or_path
+                        full_file_name = os.path.join(pretrained_model_name_or_path, subfolder, file_name)
-            else:
-                # At this point pretrained_model_name_or_path is either a directory or a model identifier name
-                additional_files_names = {
-                    "added_tokens_file": ADDED_TOKENS_FILE,
-                    "special_tokens_map_file": SPECIAL_TOKENS_MAP_FILE,
-                    "tokenizer_config_file": TOKENIZER_CONFIG_FILE,
-                    "tokenizer_file": FULL_TOKENIZER_FILE,
-                }
-                # Look for the tokenizer files
-                for file_id, file_name in {**cls.vocab_files_names, **additional_files_names}.items():
-                    if os.path.isdir(pretrained_model_name_or_path):
-                        if subfolder is not None:
-                            full_file_name = os.path.join(pretrained_model_name_or_path, subfolder, file_name)
-                        else:
-                            full_file_name = os.path.join(pretrained_model_name_or_path, file_name)
-                        if not os.path.exists(full_file_name):
-                            logger.info("Didn't find file {}. We won't load it.".format(full_file_name))
-                            full_file_name = None
                     else:
-                        full_file_name = hf_bucket_url(
+                        full_file_name = os.path.join(pretrained_model_name_or_path, file_name)
-                            pretrained_model_name_or_path,
+                    if not os.path.exists(full_file_name):
-                            filename=file_name,
+                        logger.info("Didn't find file {}. We won't load it.".format(full_file_name))
-                            subfolder=subfolder,
+                        full_file_name = None
-                            revision=revision,
+                else:
-                            mirror=None,
+                    full_file_name = hf_bucket_url(
-                        )
+                        pretrained_model_name_or_path,
+                        filename=file_name,
+                        subfolder=subfolder,
+                        revision=revision,
+                        mirror=None,
+                    )
 
-                    vocab_files[file_id] = full_file_name
+                vocab_files[file_id] = full_file_name
 
         # Get files from url, cache, or disk depending on the case
         resolved_vocab_files = {}
@@ -1833,13 +1813,14 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
 
         init_kwargs = convert_added_tokens(init_kwargs)
 
-        # Set max length if needed
+        # if we're using a pretrained model, ensure the tokenizer
-        if pretrained_model_name_or_path in cls.max_model_input_sizes:
+        # wont index sequences longer than the number of positional embeddings
-            # if we're using a pretrained model, ensure the tokenizer
+        # We use the model_max_length secified in the config if possible
-            # wont index sequences longer than the number of positional embeddings
+        # If not we default to the class default length
-            model_max_length = cls.max_model_input_sizes[pretrained_model_name_or_path]
+        model_max_length = init_kwargs.get("model_max_length", None)
-            if model_max_length is not None and isinstance(model_max_length, (int, float)):
+        if not isinstance(model_max_length, (int, float)):
-                init_kwargs["model_max_length"] = min(init_kwargs.get("model_max_length", int(1e30)), model_max_length)
+            model_max_length = cls.max_model_input_sizes
+        init_kwargs["model_max_length"] = model_max_length
 
         # Merge resolved_vocab_files arguments in init_kwargs.
         added_tokens_file = resolved_vocab_files.pop("added_tokens_file", None)

tests/test_tokenization_albert.py

@@ -17,7 +17,7 @@
 import os
 import unittest
 
-from transformers import AlbertTokenizer, AlbertTokenizerFast
+from transformers import AlbertTokenizer, AlbertTokenizerFast, ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.testing_utils import require_sentencepiece, require_tokenizers
 
 from .test_tokenization_common import TokenizerTesterMixin
@@ -30,6 +30,7 @@ SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixture
 @require_tokenizers
 class AlbertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = AlbertTokenizer
     rust_tokenizer_class = AlbertTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_bart.py

@@ -2,7 +2,7 @@ import json
 import os
 import unittest
 
-from transformers import BartTokenizer, BartTokenizerFast, BatchEncoding
+from transformers import BartTokenizer, BartTokenizerFast, BatchEncoding, BART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.file_utils import cached_property
 from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
 from transformers.testing_utils import require_tokenizers, require_torch
@@ -12,6 +12,7 @@ from .test_tokenization_common import TokenizerTesterMixin, filter_roberta_detec
 
 @require_tokenizers
 class TestTokenizationBart(TokenizerTesterMixin, unittest.TestCase):
+    pretrained_vocab_checkpoints = BART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = BartTokenizer
     rust_tokenizer_class = BartTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_bert.py

@@ -17,7 +17,7 @@
 import os
 import unittest
 
-from transformers import BertTokenizerFast
+from transformers import BertTokenizerFast, BERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.models.bert.tokenization_bert import (
     VOCAB_FILES_NAMES,
     BasicTokenizer,
@@ -35,6 +35,7 @@ from .test_tokenization_common import TokenizerTesterMixin, filter_non_english
 @require_tokenizers
 class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = BERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = BertTokenizer
     rust_tokenizer_class = BertTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_bert_generation.py

@@ -17,7 +17,7 @@
 import os
 import unittest
 
-from transformers import BertGenerationTokenizer
+from transformers import BertGenerationTokenizer, BERT_GENERATION_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_torch, slow
 
@@ -32,6 +32,7 @@ SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixture
 @require_sentencepiece
 class BertGenerationTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = BERT_GENERATION_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = BertGenerationTokenizer
 
     def setUp(self):

tests/test_tokenization_bert_japanese.py

@@ -20,6 +20,7 @@ import unittest
 
 from transformers import AutoTokenizer
 from transformers.models.bert_japanese.tokenization_bert_japanese import (
+    BERT_JAPANESE_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
     VOCAB_FILES_NAMES,
     BertJapaneseTokenizer,
     CharacterTokenizer,
@@ -34,6 +35,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 @custom_tokenizers
 class BertJapaneseTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = BERT_JAPANESE_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = BertJapaneseTokenizer
     space_between_special_tokens = True
 

tests/test_tokenization_bertweet.py

@@ -16,13 +16,14 @@
 import os
 import unittest
 
-from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer
+from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer, BERT_TWEET_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 from .test_tokenization_common import TokenizerTesterMixin
 
 
 class BertweetTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = BERT_TWEET_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = BertweetTokenizer
 
     def setUp(self):

tests/test_tokenization_blenderbot.py

@@ -21,6 +21,8 @@ import unittest
 
 from transformers.file_utils import cached_property
 from transformers.models.blenderbot.tokenization_blenderbot import (
+    BLENDERBOT_LARGE_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+    BLENDERBOT_SMALL_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
     VOCAB_FILES_NAMES,
     BlenderbotSmallTokenizer,
     BlenderbotTokenizer,
@@ -31,6 +33,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 
 class BlenderbotSmallTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = BLENDERBOT_SMALL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = BlenderbotSmallTokenizer
 
     def setUp(self):
@@ -92,7 +95,7 @@ class BlenderbotSmallTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
 class Blenderbot3BTokenizerTests(unittest.TestCase):
     @cached_property
     def tokenizer_3b(self):
-        return BlenderbotTokenizer.from_pretrained("facebook/blenderbot-3B")
+        return BlenderbotTokenizer.from_pretrained(BLENDERBOT_LARGE_PRETRAINED_TOKENIZER_ARCHIVE_LIST[0])
 
     def test_encode_decode_cycle(self):
         tok = self.tokenizer_3b

tests/test_tokenization_camembert.py

@@ -17,7 +17,7 @@
 import os
 import unittest
 
-from transformers import CamembertTokenizer, CamembertTokenizerFast
+from transformers import CamembertTokenizer, CamembertTokenizerFast, CAMEMBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.testing_utils import _torch_available, require_sentencepiece, require_tokenizers
 
 from .test_tokenization_common import TokenizerTesterMixin
@@ -32,6 +32,7 @@ FRAMEWORK = "pt" if _torch_available else "tf"
 @require_tokenizers
 class CamembertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = CAMEMBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = CamembertTokenizer
     rust_tokenizer_class = CamembertTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_common.py

@@ -76,6 +76,7 @@ def merge_model_tokenizer_mappings(
 
 class TokenizerTesterMixin:
 
+    pretrained_vocab_checkpoints = None
     tokenizer_class = None
     rust_tokenizer_class = None
     test_rust_tokenizer = False
@@ -94,9 +95,7 @@ class TokenizerTesterMixin:
                     pretrained_name,
                     self.from_pretrained_kwargs if self.from_pretrained_kwargs is not None else {},
                 )
-                for pretrained_name in self.rust_tokenizer_class.pretrained_vocab_files_map[
+                for pretrained_name in self.pretrained_vocab_checkpoints
-                    self.from_pretrained_vocab_key
-                ].keys()
                 if self.from_pretrained_filter is None
                 or (self.from_pretrained_filter is not None and self.from_pretrained_filter(pretrained_name))
             ]
@@ -181,6 +180,8 @@ class TokenizerTesterMixin:
         signature = inspect.signature(self.tokenizer_class.__init__)
         tokenizer = self.get_tokenizer()
 
+        self.assertEqual(tokenizer.init_kwargs["tokenizer_class_name"], tokenizer.__class__.__name__)
+
         for parameter_name, parameter in signature.parameters.items():
             if parameter.default != inspect.Parameter.empty:
                 self.assertIn(parameter_name, tokenizer.init_kwargs)
@@ -192,6 +193,8 @@ class TokenizerTesterMixin:
         signature = inspect.signature(self.rust_tokenizer_class.__init__)
         tokenizer = self.get_rust_tokenizer()
 
+        self.assertEqual(tokenizer.init_kwargs["tokenizer_class_name"], tokenizer.__class__.__name__)
+
         for parameter_name, parameter in signature.parameters.items():
             if parameter.default != inspect.Parameter.empty:
                 self.assertIn(parameter_name, tokenizer.init_kwargs)
@@ -545,21 +548,7 @@ class TokenizerTesterMixin:
 
     def test_pretrained_model_lists(self):
         # We should have at least one default checkpoint for each tokenizer
-        # We should specify the max input length as well (used in some part to list the pretrained checkpoints)
+        self.assertGreaterEqual(len(self.pretrained_vocab_checkpoints), 1)
-        self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map), 1)
-        self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0]), 1)
-        self.assertEqual(
-            len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0]),
-            len(self.tokenizer_class.max_model_input_sizes),
-        )
-
-        weights_list = list(self.tokenizer_class.max_model_input_sizes.keys())
-        weights_lists_2 = []
-        for file_id, map_list in self.tokenizer_class.pretrained_vocab_files_map.items():
-            weights_lists_2.append(list(map_list.keys()))
-
-        for weights_list_2 in weights_lists_2:
-            self.assertListEqual(weights_list, weights_list_2)
 
     def test_mask_output(self):
         tokenizers = self.get_tokenizers(fast=False, do_lower_case=False)

tests/test_tokenization_ctrl.py

@@ -17,13 +17,14 @@ import json
 import os
 import unittest
 
-from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer
+from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer, CTRL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 
 from .test_tokenization_common import TokenizerTesterMixin
 
 
 class CTRLTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = CTRL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = CTRLTokenizer
     test_rust_tokenizer = False
 

tests/test_tokenization_deberta.py

@@ -18,7 +18,7 @@ import re
 import unittest
 from typing import Tuple
 
-from transformers.models.deberta.tokenization_deberta import DebertaTokenizer
+from transformers.models.deberta.tokenization_deberta import DebertaTokenizer, DEBERTA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.testing_utils import require_torch
 
 from .test_tokenization_common import TokenizerTesterMixin
@@ -27,6 +27,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 @require_torch
 class DebertaTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = DEBERTA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = DebertaTokenizer
 
     def setUp(self):

tests/test_tokenization_distilbert.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 
-from transformers import DistilBertTokenizer, DistilBertTokenizerFast
+from transformers import DistilBertTokenizer, DistilBertTokenizerFast, DISTILBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.testing_utils import require_tokenizers, slow
 
 from .test_tokenization_bert import BertTokenizationTest
@@ -23,6 +23,7 @@ from .test_tokenization_bert import BertTokenizationTest
 @require_tokenizers
 class DistilBertTokenizationTest(BertTokenizationTest):
 
+    pretrained_vocab_checkpoints = DISTILBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = DistilBertTokenizer
     rust_tokenizer_class = DistilBertTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_dpr.py

@@ -22,6 +22,9 @@ from transformers import (
     DPRReaderOutput,
     DPRReaderTokenizer,
     DPRReaderTokenizerFast,
+        DPR_CONTEXT_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+        DPR_QUESTION_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST,
+        DPR_READER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 )
 from transformers.testing_utils import require_tokenizers, slow
 from transformers.tokenization_utils_base import BatchEncoding
@@ -32,6 +35,7 @@ from .test_tokenization_bert import BertTokenizationTest
 @require_tokenizers
 class DPRContextEncoderTokenizationTest(BertTokenizationTest):
 
+    pretrained_vocab_checkpoints = DPR_CONTEXT_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = DPRContextEncoderTokenizer
     rust_tokenizer_class = DPRContextEncoderTokenizerFast
     test_rust_tokenizer = True
@@ -40,6 +44,7 @@ class DPRContextEncoderTokenizationTest(BertTokenizationTest):
 @require_tokenizers
 class DPRQuestionEncoderTokenizationTest(BertTokenizationTest):
 
+    pretrained_vocab_checkpoints = DPR_QUESTION_ENCODER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = DPRQuestionEncoderTokenizer
     rust_tokenizer_class = DPRQuestionEncoderTokenizerFast
     test_rust_tokenizer = True
@@ -48,6 +53,7 @@ class DPRQuestionEncoderTokenizationTest(BertTokenizationTest):
 @require_tokenizers
 class DPRReaderTokenizationTest(BertTokenizationTest):
 
+    pretrained_vocab_checkpoints = DPR_READER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = DPRReaderTokenizer
     rust_tokenizer_class = DPRReaderTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_electra.py

@@ -0,0 +1,33 @@
+# coding=utf-8
+# Copyright 2020 Huggingface
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from transformers import (
+    ElectraTokenizer,
+    ElectraTokenizerFast,
+    ELECTRA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+)
+from transformers.testing_utils import require_tokenizers
+
+from .test_tokenization_bert import BertTokenizationTest
+
+
+@require_tokenizers
+class ElectraTokenizationTest(BertTokenizationTest):
+
+    pretrained_vocab_checkpoints = ELECTRA_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+    tokenizer_class = ElectraTokenizer
+    rust_tokenizer_class = ElectraTokenizerFast
+    test_rust_tokenizer = True

tests/test_tokenization_flaubert.py

@@ -0,0 +1,28 @@
+# coding=utf-8
+# Copyright 2020 Huggingface
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from transformers import (
+    FlaubertTokenizer,
+    FLAUBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+)
+
+from .test_tokenization_xlm import XLMTokenizationTest
+
+
+class FlaubertTokenizationTest(XLMTokenizationTest):
+
+    pretrained_vocab_checkpoints = FLAUBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+    tokenizer_class = FlaubertTokenizer

tests/test_tokenization_fsmt.py

@@ -19,7 +19,7 @@ import os
 import unittest
 
 from transformers.file_utils import cached_property
-from transformers.models.fsmt.tokenization_fsmt import VOCAB_FILES_NAMES, FSMTTokenizer
+from transformers.models.fsmt.tokenization_fsmt import VOCAB_FILES_NAMES, FSMTTokenizer, FSMT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.testing_utils import slow
 
 from .test_tokenization_common import TokenizerTesterMixin
@@ -30,6 +30,8 @@ FSMT_TINY2 = "stas/tiny-wmt19-en-ru"
 
 
 class FSMTTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
+
+    pretrained_vocab_checkpoints = FSMT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = FSMTTokenizer
 
     def setUp(self):

tests/test_tokenization_funnel.py

@@ -17,7 +17,7 @@
 import os
 import unittest
 
-from transformers import FunnelTokenizer, FunnelTokenizerFast
+from transformers import FunnelTokenizer, FunnelTokenizerFast, FUNNEL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
 from transformers.testing_utils import require_tokenizers
 
@@ -27,6 +27,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 @require_tokenizers
 class FunnelTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = FUNNEL_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = FunnelTokenizer
     rust_tokenizer_class = FunnelTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_gpt2.py

@@ -18,7 +18,7 @@ import json
 import os
 import unittest
 
-from transformers import GPT2Tokenizer, GPT2TokenizerFast
+from transformers import GPT2Tokenizer, GPT2TokenizerFast, GPT2_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.models.gpt2.tokenization_gpt2 import VOCAB_FILES_NAMES
 from transformers.testing_utils import require_tokenizers
 
@@ -28,6 +28,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 @require_tokenizers
 class GPT2TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = GPT2_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = GPT2Tokenizer
     rust_tokenizer_class = GPT2TokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_herbert.py

@@ -18,7 +18,7 @@ import json
 import os
 import unittest
 
-from transformers import HerbertTokenizer, HerbertTokenizerFast
+from transformers import HerbertTokenizer, HerbertTokenizerFast, HERBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
 from transformers.testing_utils import get_tests_dir, require_tokenizers, slow
 
@@ -28,6 +28,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 @require_tokenizers
 class HerbertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = HERBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = HerbertTokenizer
     rust_tokenizer_class = HerbertTokenizerFast
     test_rust_tokenizer = True
@@ -114,6 +115,9 @@ class HerbertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         rust_ids = rust_tokenizer.encode(sequence)
         self.assertListEqual(ids, rust_ids)
 
+    def test_padding_to_multiple_of(self):
+        pass  # Because HerBert has a max length of 514 which is not a multiple of 8
+
     @slow
     def test_sequence_builders(self):
         tokenizer = self.tokenizer_class.from_pretrained("allegro/herbert-base-cased")

tests/test_tokenization_layoutlm.py

@@ -17,7 +17,7 @@
 import os
 import unittest
 
-from transformers import LayoutLMTokenizer, LayoutLMTokenizerFast
+from transformers import LayoutLMTokenizer, LayoutLMTokenizerFast, LAYOUTLM_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.models.layoutlm.tokenization_layoutlm import VOCAB_FILES_NAMES
 from transformers.testing_utils import require_tokenizers
 
@@ -27,6 +27,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 @require_tokenizers
 class LayoutLMTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = LAYOUTLM_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = LayoutLMTokenizer
     rust_tokenizer_class = LayoutLMTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_longformer.py

@@ -0,0 +1,33 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+import unittest
+
+from transformers import LongformerTokenizer, LongformerTokenizerFast, LONGFORMER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+from transformers.testing_utils import require_tokenizers
+
+from .test_tokenization_roberta import RobertaTokenizationTest
+
+
+@require_tokenizers
+class LongformerTokenizationTest(RobertaTokenizationTest):
+
+    pretrained_vocab_checkpoints = LONGFORMER_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+    tokenizer_class = LongformerTokenizer
+    rust_tokenizer_class = LongformerTokenizerFast
+    test_rust_tokenizer = True

tests/test_tokenization_lxmert.py

@@ -17,7 +17,7 @@
 import os
 import unittest
 
-from transformers import LxmertTokenizer, LxmertTokenizerFast
+from transformers import LxmertTokenizer, LxmertTokenizerFast, LXMERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
 from transformers.testing_utils import require_tokenizers
 
@@ -27,6 +27,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 @require_tokenizers
 class LxmertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = LXMERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = LxmertTokenizer
     rust_tokenizer_class = LxmertTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_marian.py

@@ -20,7 +20,7 @@ import unittest
 from pathlib import Path
 from shutil import copyfile
 
-from transformers import BatchEncoding, MarianTokenizer
+from transformers import BatchEncoding, MarianTokenizer, MARIAN_PRETRAINED_TOKENIZER_ARCHIVE_LIST, TOKENIZER_CONFIG_NAME
 from transformers.testing_utils import _sentencepiece_available, _torch_available, require_sentencepiece
 
 
@@ -41,6 +41,7 @@ FRAMEWORK = "pt" if _torch_available else "tf"
 @require_sentencepiece
 class MarianTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = MARIAN_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = MarianTokenizer
     test_rust_tokenizer = False
 
@@ -50,7 +51,7 @@ class MarianTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         vocab_tokens = dict(zip(vocab, range(len(vocab))))
         save_dir = Path(self.tmpdirname)
         save_json(vocab_tokens, save_dir / vocab_files_names["vocab"])
-        save_json(mock_tokenizer_config, save_dir / vocab_files_names["tokenizer_config_file"])
+        save_json(mock_tokenizer_config, save_dir / TOKENIZER_CONFIG_NAME)
         if not (save_dir / vocab_files_names["source_spm"]).exists():
             copyfile(SAMPLE_SP, save_dir / vocab_files_names["source_spm"])
             copyfile(SAMPLE_SP, save_dir / vocab_files_names["target_spm"])

tests/test_tokenization_mbart.py

@@ -1,7 +1,7 @@
 import tempfile
 import unittest
 
-from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
+from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available, MBART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.testing_utils import (
     _sentencepiece_available,
     require_sentencepiece,
@@ -26,6 +26,8 @@ RO_CODE = 250020
 @require_sentencepiece
 @require_tokenizers
 class MBartTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
+
+    pretrained_vocab_checkpoints = MBART_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = MBartTokenizer
     rust_tokenizer_class = MBartTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_mobilebert.py

@@ -0,0 +1,34 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors, Allegro.pl and The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+import unittest
+
+from transformers import MobileBertTokenizer, MobileBertTokenizerFast, MOBILEBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+from transformers.testing_utils import get_tests_dir, require_tokenizers, slow
+
+from .test_tokenization_bert import BertTokenizationTest
+from .test_tokenization_common import TokenizerTesterMixin
+
+
+@require_tokenizers
+class MobilebertTokenizationTest(BertTokenizationTest):
+
+    pretrained_vocab_checkpoints = MOBILEBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
+    tokenizer_class = MobileBertTokenizer
+    rust_tokenizer_class = MobileBertTokenizerFast
+    test_rust_tokenizer = True

tests/test_tokenization_openai.py

@@ -18,7 +18,7 @@ import json
 import os
 import unittest
 
-from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
+from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast, OPENAIGPT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
 from transformers.testing_utils import require_tokenizers
 
@@ -28,6 +28,7 @@ from .test_tokenization_common import TokenizerTesterMixin
 @require_tokenizers
 class OpenAIGPTTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
+    pretrained_vocab_checkpoints = OPENAIGPT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
     tokenizer_class = OpenAIGPTTokenizer
     rust_tokenizer_class = OpenAIGPTTokenizerFast
     test_rust_tokenizer = True

tests/test_tokenization_utils.py

@@ -19,24 +19,11 @@ from typing import Callable, Optional
 import numpy as np
 
 from transformers import BatchEncoding, BertTokenizer, BertTokenizerFast, PreTrainedTokenizer, TensorType, TokenSpan
-from transformers.models.gpt2.tokenization_gpt2 import GPT2Tokenizer
+from transformers.models.albert.tokenization_albert import AlbertTokenizer, ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST
 from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow
 
 
 class TokenizerUtilsTest(unittest.TestCase):
-    def check_tokenizer_from_pretrained(self, tokenizer_class):
-        s3_models = list(tokenizer_class.max_model_input_sizes.keys())
-        for model_name in s3_models[:1]:
-            tokenizer = tokenizer_class.from_pretrained(model_name)
-            self.assertIsNotNone(tokenizer)
-            self.assertIsInstance(tokenizer, tokenizer_class)
-            self.assertIsInstance(tokenizer, PreTrainedTokenizer)
-
-            for special_tok in tokenizer.all_special_tokens:
-                self.assertIsInstance(special_tok, str)
-                special_tok_id = tokenizer.convert_tokens_to_ids(special_tok)
-                self.assertIsInstance(special_tok_id, int)
-
     def assert_dump_and_restore(self, be_original: BatchEncoding, equal_op: Optional[Callable] = None):
         batch_encoding_str = pickle.dumps(be_original)
         self.assertIsNotNone(batch_encoding_str)
@@ -61,7 +48,16 @@ class TokenizerUtilsTest(unittest.TestCase):
 
     @slow
     def test_pretrained_tokenizers(self):
-        self.check_tokenizer_from_pretrained(GPT2Tokenizer)
+        for model_name in ALBERT_PRETRAINED_TOKENIZER_ARCHIVE_LIST[:1]:
+            tokenizer = AlbertTokenizer.from_pretrained(model_name)
+            self.assertIsNotNone(tokenizer)
+            self.assertIsInstance(tokenizer, AlbertTokenizer)
+            self.assertIsInstance(tokenizer, PreTrainedTokenizer)
+
+            for special_tok in tokenizer.all_special_tokens:
+                self.assertIsInstance(special_tok, str)
+                special_tok_id = tokenizer.convert_tokens_to_ids(special_tok)
+                self.assertIsInstance(special_tok_id, int)
 
     def test_tensor_type_from_str(self):
         self.assertEqual(TensorType("tf"), TensorType.TENSORFLOW)