Fix style

Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>

docs/source/model_doc/albert.rst

@@ -1,7 +1,7 @@
 ALBERT
 ----------------------------------------------------
 
-``AlbrtConfig``
+``AlbertConfig``
 ~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.AlbertConfig

docs/source/model_doc/xlm.rst

@@ -34,6 +34,13 @@ XLM
     :members:
 
 
+``XLMForQuestionAnsweringSimple``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLMForQuestionAnsweringSimple
+    :members:
+
+
 ``XLMForQuestionAnswering``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/xlnet.rst

@@ -36,6 +36,27 @@ XLNet
     :members:
 
 
+``XLNetForTokenClassification``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLNetForTokenClassification
+    :members:
+
+
+``XLNetForMultipleChoice``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLNetForMultipleChoice
+    :members:
+
+
+``XLNetForQuestionAnsweringSimple``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLNetForQuestionAnsweringSimple
+    :members:
+
+
 ``XLNetForQuestionAnswering``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

examples/README.md

@@ -24,6 +24,8 @@ pip install -r ./examples/requirements.txt
 | [Multiple Choice](#multiple-choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. 
 | [Named Entity Recognition](#named-entity-recognition) | Using BERT for Named Entity Recognition (NER) on the CoNLL 2003 dataset, examples with distributed training.                                                                                  |
 | [XNLI](#xnli) | Examples running BERT/XLM on the XNLI benchmark. |
+| [Adversarial evaluation of model performances](#adversarial-evaluation-of-model-performances) | Testing a model with adversarial evaluation of natural language
+inference on the Heuristic Analysis for NLI Systems (HANS) dataset (McCoy et al., 2019.) |
 
 ## TensorFlow 2.0 Bert models on GLUE
 
@@ -758,4 +760,42 @@ python run_mmimdb.py \
     --patience 5
 ```
 
+## Adversarial evaluation of model performances
 
+Here is an example on evaluating a model using adversarial evaluation of natural language inference with the Heuristic Analysis for NLI Systems (HANS) dataset [McCoy et al., 2019](https://arxiv.org/abs/1902.01007). The example was gracefully provided by [Nafise Sadat Moosavi](https://github.com/ns-moosavi).
+
+The HANS dataset can be downloaded from [this location](https://github.com/tommccoy1/hans).
+
+This is an example of using test_hans.py:
+
+```bash
+export HANS_DIR=path-to-hans
+export MODEL_TYPE=type-of-the-model-e.g.-bert-roberta-xlnet-etc
+export MODEL_PATH=path-to-the-model-directory-that-is-trained-on-NLI-e.g.-by-using-run_glue.py
+
+python examples/test_hans.py \
+        --task_name hans \
+        --model_type $MODEL_TYPE \
+        --do_eval \
+        --do_lower_case \
+        --data_dir $HANS_DIR \
+        --model_name_or_path $MODEL_PATH \
+        --max_seq_length 128 \
+        -output_dir $MODEL_PATH \
+```
+
+This will create the hans_predictions.txt file in MODEL_PATH, which can then be evaluated using hans/evaluate_heur_output.py from the HANS dataset.
+
+The results of the BERT-base model that is trained on MNLI using batch size 8 and the random seed 42 on the HANS dataset is as follows:
+
+```bash
+Heuristic entailed results:
+lexical_overlap: 0.9702
+subsequence: 0.9942
+constituent: 0.9962
+
+Heuristic non-entailed results:
+lexical_overlap: 0.199
+subsequence: 0.0396
+constituent: 0.118
+```

examples/distillation/lm_seqs_dataset.py

@@ -42,6 +42,7 @@ class LmSeqsDataset(Dataset):
         self.check()
         self.remove_long_sequences()
         self.remove_empty_sequences()
+        self.remove_unknown_sequences()
         self.check()
         self.print_statistics()
 
@@ -109,6 +110,22 @@ class LmSeqsDataset(Dataset):
         new_size = len(self)
         logger.info(f"Remove {init_size - new_size} too short (<=11 tokens) sequences.")
 
+    def remove_unknown_sequences(self):
+        """
+        Remove sequences with a (too) high level of unknown tokens.
+        """
+        if "unk_token" not in self.params.special_tok_ids:
+            return
+        else:
+            unk_token_id = self.params.special_tok_ids["unk_token"]
+        init_size = len(self)
+        unk_occs = np.array([np.count_nonzero(a == unk_token_id) for a in self.token_ids])
+        indices = (unk_occs / self.lengths) < 0.5
+        self.token_ids = self.token_ids[indices]
+        self.lengths = self.lengths[indices]
+        new_size = len(self)
+        logger.info(f"Remove {init_size - new_size} sequences with a high level of unknown tokens (50%).")
+
     def print_statistics(self):
         """
         Print some statistics on the corpus. Only the master process.

examples/distillation/training_configs/distilbert-base-multilingual-cased.json

@@ -0,0 +1,15 @@
+{
+	"activation": "gelu",
+	"attention_dropout": 0.1,
+	"dim": 768,
+	"dropout": 0.1,
+	"hidden_dim": 3072,
+	"initializer_range": 0.02,
+	"max_position_embeddings": 512,
+	"n_heads": 12,
+	"n_layers": 6,
+	"sinusoidal_pos_embds": true,
+	"tie_weights_": true,
+	"vocab_size": 119547
+  }
+  

examples/distillation/training_configs/distilroberta-base.json

@@ -0,0 +1,14 @@
+{
+    "vocab_size": 50265,
+    "hidden_size": 768,
+    "num_hidden_layers": 6,
+    "num_attention_heads": 12,
+    "intermediate_size": 3072,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "attention_probs_dropout_prob": 0.1,
+    "max_position_embeddings": 514,
+    "type_vocab_size": 1,
+    "initializer_range": 0.02,
+    "layer_norm_eps": 0.00001
+}

examples/hans/hans_processors.py

@@ -0,0 +1,221 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" GLUE processors and helpers """
+
+import logging
+import os
+
+from transformers.file_utils import is_tf_available
+from utils_hans import DataProcessor, InputExample, InputFeatures
+
+
+if is_tf_available():
+    import tensorflow as tf
+
+logger = logging.getLogger(__name__)
+
+
+def hans_convert_examples_to_features(
+    examples,
+    tokenizer,
+    max_length=512,
+    task=None,
+    label_list=None,
+    output_mode=None,
+    pad_on_left=False,
+    pad_token=0,
+    pad_token_segment_id=0,
+    mask_padding_with_zero=True,
+):
+    """
+    Loads a data file into a list of ``InputFeatures``
+
+    Args:
+        examples: List of ``InputExamples`` or ``tf.data.Dataset`` containing the examples.
+        tokenizer: Instance of a tokenizer that will tokenize the examples
+        max_length: Maximum example length
+        task: HANS
+        label_list: List of labels. Can be obtained from the processor using the ``processor.get_labels()`` method
+        output_mode: String indicating the output mode. Either ``regression`` or ``classification``
+        pad_on_left: If set to ``True``, the examples will be padded on the left rather than on the right (default)
+        pad_token: Padding token
+        pad_token_segment_id: The segment ID for the padding token (It is usually 0, but can vary such as for XLNet where it is 4)
+        mask_padding_with_zero: If set to ``True``, the attention mask will be filled by ``1`` for actual values
+            and by ``0`` for padded values. If set to ``False``, inverts it (``1`` for padded values, ``0`` for
+            actual values)
+
+    Returns:
+        If the ``examples`` input is a ``tf.data.Dataset``, will return a ``tf.data.Dataset``
+        containing the task-specific features. If the input is a list of ``InputExamples``, will return
+        a list of task-specific ``InputFeatures`` which can be fed to the model.
+
+    """
+    is_tf_dataset = False
+    if is_tf_available() and isinstance(examples, tf.data.Dataset):
+        is_tf_dataset = True
+
+    if task is not None:
+        processor = glue_processors[task]()
+        if label_list is None:
+            label_list = processor.get_labels()
+            logger.info("Using label list %s for task %s" % (label_list, task))
+        if output_mode is None:
+            output_mode = glue_output_modes[task]
+            logger.info("Using output mode %s for task %s" % (output_mode, task))
+
+    label_map = {label: i for i, label in enumerate(label_list)}
+
+    features = []
+    for (ex_index, example) in enumerate(examples):
+        if ex_index % 10000 == 0:
+            logger.info("Writing example %d" % (ex_index))
+        if is_tf_dataset:
+            example = processor.get_example_from_tensor_dict(example)
+            example = processor.tfds_map(example)
+
+        inputs = tokenizer.encode_plus(example.text_a, example.text_b, add_special_tokens=True, max_length=max_length,)
+        input_ids, token_type_ids = inputs["input_ids"], inputs["token_type_ids"]
+
+        # The mask has 1 for real tokens and 0 for padding tokens. Only real
+        # tokens are attended to.
+        attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
+
+        # Zero-pad up to the sequence length.
+        padding_length = max_length - len(input_ids)
+        if pad_on_left:
+            input_ids = ([pad_token] * padding_length) + input_ids
+            attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
+            token_type_ids = ([pad_token_segment_id] * padding_length) + token_type_ids
+        else:
+            input_ids = input_ids + ([pad_token] * padding_length)
+            attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)
+            token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length)
+
+        assert len(input_ids) == max_length, "Error with input length {} vs {}".format(len(input_ids), max_length)
+        assert len(attention_mask) == max_length, "Error with input length {} vs {}".format(
+            len(attention_mask), max_length
+        )
+        assert len(token_type_ids) == max_length, "Error with input length {} vs {}".format(
+            len(token_type_ids), max_length
+        )
+
+        if output_mode == "classification":
+            label = label_map[example.label] if example.label in label_map else 0
+        elif output_mode == "regression":
+            label = float(example.label)
+        else:
+            raise KeyError(output_mode)
+        pairID = str(example.pairID)
+
+        if ex_index < 10:
+            logger.info("*** Example ***")
+            logger.info("text_a: %s" % (example.text_a))
+            logger.info("text_b: %s" % (example.text_b))
+            logger.info("guid: %s" % (example.guid))
+            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
+            logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask]))
+            logger.info("token_type_ids: %s" % " ".join([str(x) for x in token_type_ids]))
+            logger.info("label: %s (id = %d)" % (example.label, label))
+
+        features.append(
+            InputFeatures(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                token_type_ids=token_type_ids,
+                label=label,
+                pairID=pairID,
+            )
+        )
+
+    if is_tf_available() and is_tf_dataset:
+
+        def gen():
+            for ex in features:
+                yield (
+                    {
+                        "input_ids": ex.input_ids,
+                        "attention_mask": ex.attention_mask,
+                        "token_type_ids": ex.token_type_ids,
+                    },
+                    ex.label,
+                )
+
+        return tf.data.Dataset.from_generator(
+            gen,
+            ({"input_ids": tf.int32, "attention_mask": tf.int32, "token_type_ids": tf.int32}, tf.int64),
+            (
+                {
+                    "input_ids": tf.TensorShape([None]),
+                    "attention_mask": tf.TensorShape([None]),
+                    "token_type_ids": tf.TensorShape([None]),
+                },
+                tf.TensorShape([]),
+            ),
+        )
+
+    return features
+
+
+class HansProcessor(DataProcessor):
+    """Processor for the HANS data set."""
+
+    def get_example_from_tensor_dict(self, tensor_dict):
+        """See base class."""
+        return InputExample(
+            tensor_dict["idx"].numpy(),
+            tensor_dict["premise"].numpy().decode("utf-8"),
+            tensor_dict["hypothesis"].numpy().decode("utf-8"),
+            str(tensor_dict["label"].numpy()),
+        )
+
+    def get_train_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(self._read_tsv(os.path.join(data_dir, "heuristics_train_set.txt")), "train")
+
+    def get_dev_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(self._read_tsv(os.path.join(data_dir, "heuristics_evaluation_set.txt")), "dev")
+
+    def get_labels(self):
+        """See base class."""
+        return ["contradiction", "entailment", "neutral"]
+
+    def _create_examples(self, lines, set_type):
+        """Creates examples for the training and dev sets."""
+        examples = []
+        for (i, line) in enumerate(lines):
+            if i == 0:
+                continue
+            guid = "%s-%s" % (set_type, line[0])
+            text_a = line[5]
+            text_b = line[6]
+            pairID = line[7][2:] if line[7].startswith("ex") else line[7]
+            label = line[-1]
+            examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label, pairID=pairID))
+        return examples
+
+
+glue_tasks_num_labels = {
+    "hans": 3,
+}
+
+glue_processors = {
+    "hans": HansProcessor,
+}
+
+glue_output_modes = {
+    "hans": "classification",
+}

examples/hans/test_hans.py

@@ -0,0 +1,643 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning the library models for sequence classification on GLUE (Bert, XLM, XLNet, RoBERTa)."""
+
+from __future__ import absolute_import, division, print_function
+
+import argparse
+import glob
+import logging
+import os
+import random
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
+from torch.utils.data.distributed import DistributedSampler
+from tqdm import tqdm, trange
+
+from hans_processors import glue_output_modes as output_modes
+from hans_processors import glue_processors as processors
+from hans_processors import hans_convert_examples_to_features as convert_examples_to_features
+from transformers import (
+    WEIGHTS_NAME,
+    AdamW,
+    AlbertConfig,
+    AlbertForSequenceClassification,
+    AlbertTokenizer,
+    BertConfig,
+    BertForSequenceClassification,
+    BertTokenizer,
+    DistilBertConfig,
+    DistilBertForSequenceClassification,
+    DistilBertTokenizer,
+    RobertaConfig,
+    RobertaForSequenceClassification,
+    RobertaTokenizer,
+    XLMConfig,
+    XLMForSequenceClassification,
+    XLMTokenizer,
+    XLNetConfig,
+    XLNetForSequenceClassification,
+    XLNetTokenizer,
+    get_linear_schedule_with_warmup,
+)
+
+
+try:
+    from torch.utils.tensorboard import SummaryWriter
+except ImportError:
+    from tensorboardX import SummaryWriter
+
+
+logger = logging.getLogger(__name__)
+
+ALL_MODELS = sum(
+    (
+        tuple(conf.pretrained_config_archive_map.keys())
+        for conf in (BertConfig, XLNetConfig, XLMConfig, RobertaConfig, DistilBertConfig)
+    ),
+    (),
+)
+
+MODEL_CLASSES = {
+    "bert": (BertConfig, BertForSequenceClassification, BertTokenizer),
+    "xlnet": (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
+    "xlm": (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
+    "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
+    "distilbert": (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer),
+    "albert": (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer),
+}
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def train(args, train_dataset, model, tokenizer):
+    """ Train the model """
+    if args.local_rank in [-1, 0]:
+        tb_writer = SummaryWriter()
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
+    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
+
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+    else:
+        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
+
+    # Prepare optimizer and schedule (linear warmup and decay)
+    no_decay = ["bias", "LayerNorm.weight"]
+    optimizer_grouped_parameters = [
+        {
+            "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
+            "weight_decay": args.weight_decay,
+        },
+        {"params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0},
+    ]
+
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
+    scheduler = get_linear_schedule_with_warmup(
+        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total
+    )
+    if args.fp16:
+        try:
+            from apex import amp
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
+
+    # multi-gpu training (should be after apex fp16 initialization)
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
+    # Distributed training (should be after apex fp16 initialization)
+    if args.local_rank != -1:
+        model = torch.nn.parallel.DistributedDataParallel(
+            model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True
+        )
+
+    # Train!
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info(
+        "  Total train batch size (w. parallel, distributed & accumulation) = %d",
+        args.train_batch_size
+        * args.gradient_accumulation_steps
+        * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),
+    )
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
+    set_seed(args)  # Added here for reproductibility (even between python 2 and 3)
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
+        for step, batch in enumerate(epoch_iterator):
+            model.train()
+            batch = tuple(t.to(args.device) for t in batch)
+            inputs = {"input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3]}
+            if args.model_type != "distilbert":
+                inputs["token_type_ids"] = (
+                    batch[2] if args.model_type in ["bert", "xlnet"] else None
+                )  # XLM, DistilBERT and RoBERTa don't use segment_ids
+            outputs = model(**inputs)
+            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)
+
+            if args.n_gpu > 1:
+                loss = loss.mean()  # mean() to average on multi-gpu parallel training
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+
+            if args.fp16:
+                with amp.scale_loss(loss, optimizer) as scaled_loss:
+                    scaled_loss.backward()
+            else:
+                loss.backward()
+
+            tr_loss += loss.item()
+            if (step + 1) % args.gradient_accumulation_steps == 0:
+                if args.fp16:
+                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+                else:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
+                optimizer.step()
+                scheduler.step()  # Update learning rate schedule
+                model.zero_grad()
+                global_step += 1
+
+                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
+                    logs = {}
+                    if (
+                        args.local_rank == -1 and args.evaluate_during_training
+                    ):  # Only evaluate when single GPU otherwise metrics may not average well
+                        results = evaluate(args, model, tokenizer)
+                        for key, value in results.items():
+                            eval_key = "eval_{}".format(key)
+                            logs[eval_key] = value
+
+                    loss_scalar = (tr_loss - logging_loss) / args.logging_steps
+                    learning_rate_scalar = scheduler.get_lr()[0]
+                    logs["learning_rate"] = learning_rate_scalar
+                    logs["loss"] = loss_scalar
+                    logging_loss = tr_loss
+
+                    for key, value in logs.items():
+                        tb_writer.add_scalar(key, value, global_step)
+                    # print(json.dumps({**logs, **{'step': global_step}}))
+
+                if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
+                    # Save model checkpoint
+                    output_dir = os.path.join(args.output_dir, "checkpoint-{}".format(global_step))
+                    if not os.path.exists(output_dir):
+                        os.makedirs(output_dir)
+                    model_to_save = (
+                        model.module if hasattr(model, "module") else model
+                    )  # Take care of distributed/parallel training
+                    model_to_save.save_pretrained(output_dir)
+                    torch.save(args, os.path.join(output_dir, "training_args.bin"))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+
+    if args.local_rank in [-1, 0]:
+        tb_writer.close()
+
+    return global_step, tr_loss / global_step
+
+
+def evaluate(args, model, tokenizer, prefix=""):
+    # Loop to handle MNLI double evaluation (matched, mis-matched)
+    eval_task_names = ("mnli", "mnli-mm") if args.task_name == "mnli" else (args.task_name,)
+    eval_outputs_dirs = (args.output_dir, args.output_dir + "-MM") if args.task_name == "mnli" else (args.output_dir,)
+
+    results = {}
+    for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs):
+        eval_dataset, label_list = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True)
+
+        if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(eval_output_dir)
+
+        args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+        # Note that DistributedSampler samples randomly
+        eval_sampler = SequentialSampler(eval_dataset)
+        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
+
+        # multi-gpu eval
+        if args.n_gpu > 1:
+            model = torch.nn.DataParallel(model)
+
+        # Eval!
+        logger.info("***** Running evaluation {} *****".format(prefix))
+        logger.info("  Num examples = %d", len(eval_dataset))
+        logger.info("  Batch size = %d", args.eval_batch_size)
+        eval_loss = 0.0
+        nb_eval_steps = 0
+        preds = None
+        out_label_ids = None
+        for batch in tqdm(eval_dataloader, desc="Evaluating"):
+            model.eval()
+            batch = tuple(t.to(args.device) for t in batch)
+
+            with torch.no_grad():
+                inputs = {"input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3]}
+                if args.model_type != "distilbert":
+                    inputs["token_type_ids"] = (
+                        batch[2] if args.model_type in ["bert", "xlnet"] else None
+                    )  # XLM, DistilBERT and RoBERTa don't use segment_ids
+                outputs = model(**inputs)
+                tmp_eval_loss, logits = outputs[:2]
+
+                eval_loss += tmp_eval_loss.mean().item()
+            nb_eval_steps += 1
+            if preds is None:
+                preds = logits.detach().cpu().numpy()
+                out_label_ids = inputs["labels"].detach().cpu().numpy()
+                pair_ids = batch[4].detach().cpu().numpy()
+            else:
+                preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
+                out_label_ids = np.append(out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0)
+                pair_ids = np.append(pair_ids, batch[4].detach().cpu().numpy(), axis=0)
+
+        eval_loss = eval_loss / nb_eval_steps
+        if args.output_mode == "classification":
+            preds = np.argmax(preds, axis=1)
+        elif args.output_mode == "regression":
+            preds = np.squeeze(preds)
+
+        output_eval_file = os.path.join(eval_output_dir, "hans_predictions.txt")
+        with open(output_eval_file, "w") as writer:
+            writer.write("pairID,gld_label\n")
+            for pid, pred in zip(pair_ids, preds):
+                writer.write("ex" + str(pid) + "," + label_list[int(pred)] + "\n")
+
+    return results
+
+
+def load_and_cache_examples(args, task, tokenizer, evaluate=False):
+    if args.local_rank not in [-1, 0] and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    processor = processors[task]()
+    output_mode = output_modes[task]
+    # Load data features from cache or dataset file
+    cached_features_file = os.path.join(
+        args.data_dir,
+        "cached_{}_{}_{}_{}".format(
+            "dev" if evaluate else "train",
+            list(filter(None, args.model_name_or_path.split("/"))).pop(),
+            str(args.max_seq_length),
+            str(task),
+        ),
+    )
+
+    label_list = processor.get_labels()
+
+    if os.path.exists(cached_features_file) and not args.overwrite_cache:
+        logger.info("Loading features from cached file %s", cached_features_file)
+        features = torch.load(cached_features_file)
+    else:
+        logger.info("Creating features from dataset file at %s", args.data_dir)
+        if task in ["mnli", "mnli-mm"] and args.model_type in ["roberta"]:
+            # HACK(label indices are swapped in RoBERTa pretrained model)
+            label_list[1], label_list[2] = label_list[2], label_list[1]
+        examples = (
+            processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
+        )
+        features = convert_examples_to_features(
+            examples,
+            tokenizer,
+            label_list=label_list,
+            max_length=args.max_seq_length,
+            output_mode=output_mode,
+            pad_on_left=bool(args.model_type in ["xlnet"]),  # pad on the left for xlnet
+            pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
+            pad_token_segment_id=4 if args.model_type in ["xlnet"] else 0,
+        )
+        if args.local_rank in [-1, 0]:
+            logger.info("Saving features into cached file %s", cached_features_file)
+            torch.save(features, cached_features_file)
+
+    if args.local_rank == 0 and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    # Convert to Tensors and build dataset
+    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
+    all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)
+    all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)
+    if output_mode == "classification":
+        all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
+    elif output_mode == "regression":
+        all_labels = torch.tensor([f.label for f in features], dtype=torch.float)
+    all_pair_ids = torch.tensor([int(f.pairID) for f in features], dtype=torch.long)
+
+    dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels, all_pair_ids)
+    return dataset, label_list
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    # Required parameters
+    parser.add_argument(
+        "--data_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The input data dir. Should contain the .tsv files (or other data files) for the task.",
+    )
+    parser.add_argument(
+        "--model_type",
+        default=None,
+        type=str,
+        required=True,
+        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        default=None,
+        type=str,
+        required=True,
+        help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS),
+    )
+    parser.add_argument(
+        "--task_name",
+        default=None,
+        type=str,
+        required=True,
+        help="The name of the task to train selected in the list: " + ", ".join(processors.keys()),
+    )
+    parser.add_argument(
+        "--output_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The output directory where the model predictions and checkpoints will be written.",
+    )
+
+    # Other parameters
+    parser.add_argument(
+        "--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name"
+    )
+    parser.add_argument(
+        "--tokenizer_name",
+        default="",
+        type=str,
+        help="Pretrained tokenizer name or path if not the same as model_name",
+    )
+    parser.add_argument(
+        "--cache_dir",
+        default="",
+        type=str,
+        help="Where do you want to store the pre-trained models downloaded from s3",
+    )
+    parser.add_argument(
+        "--max_seq_length",
+        default=128,
+        type=int,
+        help="The maximum total input sequence length after tokenization. Sequences longer "
+        "than this will be truncated, sequences shorter will be padded.",
+    )
+    parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
+    parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.")
+    parser.add_argument(
+        "--evaluate_during_training", action="store_true", help="Rul evaluation during training at each logging step."
+    )
+    parser.add_argument(
+        "--do_lower_case", action="store_true", help="Set this flag if you are using an uncased model."
+    )
+
+    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.")
+    parser.add_argument(
+        "--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for evaluation."
+    )
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Number of updates steps to accumulate before performing a backward/update pass.",
+    )
+    parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
+    parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
+    parser.add_argument(
+        "--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform."
+    )
+    parser.add_argument(
+        "--max_steps",
+        default=-1,
+        type=int,
+        help="If > 0: set total number of training steps to perform. Override num_train_epochs.",
+    )
+    parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")
+
+    parser.add_argument("--logging_steps", type=int, default=50, help="Log every X updates steps.")
+    parser.add_argument("--save_steps", type=int, default=50, help="Save checkpoint every X updates steps.")
+    parser.add_argument(
+        "--eval_all_checkpoints",
+        action="store_true",
+        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number",
+    )
+    parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
+    parser.add_argument(
+        "--overwrite_output_dir", action="store_true", help="Overwrite the content of the output directory"
+    )
+    parser.add_argument(
+        "--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets"
+    )
+    parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
+
+    parser.add_argument(
+        "--fp16",
+        action="store_true",
+        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
+    )
+    parser.add_argument(
+        "--fp16_opt_level",
+        type=str,
+        default="O1",
+        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+        "See details at https://nvidia.github.io/apex/amp.html",
+    )
+    parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank")
+    parser.add_argument("--server_ip", type=str, default="", help="For distant debugging.")
+    parser.add_argument("--server_port", type=str, default="", help="For distant debugging.")
+    args = parser.parse_args()
+
+    if (
+        os.path.exists(args.output_dir)
+        and os.listdir(args.output_dir)
+        and args.do_train
+        and not args.overwrite_output_dir
+    ):
+        raise ValueError(
+            "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
+                args.output_dir
+            )
+        )
+
+    # Setup distant debugging if needed
+    if args.server_ip and args.server_port:
+        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
+        import ptvsd
+
+        print("Waiting for debugger attach")
+        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
+        ptvsd.wait_for_attach()
+
+    # Setup CUDA, GPU & distributed training
+    if args.local_rank == -1 or args.no_cuda:
+        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+        args.n_gpu = torch.cuda.device_count()
+    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+        torch.cuda.set_device(args.local_rank)
+        device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend="nccl")
+        args.n_gpu = 1
+    args.device = device
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
+    )
+    logger.warning(
+        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+        args.local_rank,
+        device,
+        args.n_gpu,
+        bool(args.local_rank != -1),
+        args.fp16,
+    )
+
+    # Set seed
+    set_seed(args)
+
+    # Prepare GLUE task
+    args.task_name = args.task_name.lower()
+    if args.task_name not in processors:
+        raise ValueError("Task not found: %s" % (args.task_name))
+    processor = processors[args.task_name]()
+    args.output_mode = output_modes[args.task_name]
+    label_list = processor.get_labels()
+    num_labels = len(label_list)
+
+    # Load pretrained model and tokenizer
+    if args.local_rank not in [-1, 0]:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    args.model_type = args.model_type.lower()
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(
+        args.config_name if args.config_name else args.model_name_or_path,
+        num_labels=num_labels,
+        finetuning_task=args.task_name,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+    tokenizer = tokenizer_class.from_pretrained(
+        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+        do_lower_case=args.do_lower_case,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+    model = model_class.from_pretrained(
+        args.model_name_or_path,
+        from_tf=bool(".ckpt" in args.model_name_or_path),
+        config=config,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+
+    if args.local_rank == 0:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    model.to(args.device)
+
+    logger.info("Training/evaluation parameters %s", args)
+
+    # Training
+    if args.do_train:
+        train_dataset, _ = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)
+        global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+
+    # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
+        # Create output directory if needed
+        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(args.output_dir)
+
+        logger.info("Saving model checkpoint to %s", args.output_dir)
+        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
+        # They can then be reloaded using `from_pretrained()`
+        model_to_save = (
+            model.module if hasattr(model, "module") else model
+        )  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(args.output_dir)
+        tokenizer.save_pretrained(args.output_dir)
+
+        # Good practice: save your training arguments together with the trained model
+        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
+
+        # Load a trained model and vocabulary that you have fine-tuned
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
+        model.to(args.device)
+
+    # Evaluation
+    results = {}
+    if args.do_eval and args.local_rank in [-1, 0]:
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        checkpoints = [args.output_dir]
+        if args.eval_all_checkpoints:
+            checkpoints = list(
+                os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME, recursive=True))
+            )
+            logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN)  # Reduce logging
+        logger.info("Evaluate the following checkpoints: %s", checkpoints)
+        for checkpoint in checkpoints:
+            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
+            prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""
+
+            model = model_class.from_pretrained(checkpoint)
+            model.to(args.device)
+            result = evaluate(args, model, tokenizer, prefix=prefix)
+            result = dict((k + "_{}".format(global_step), v) for k, v in result.items())
+            results.update(result)
+
+    return results
+
+
+if __name__ == "__main__":
+    main()

examples/hans/utils_hans.py

@@ -0,0 +1,121 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import csv
+import json
+
+
+class InputExample(object):
+    """
+    A single training/test example for simple sequence classification.
+
+    Args:
+        guid: Unique id for the example.
+        text_a: string. The untokenized text of the first sequence. For single
+        sequence tasks, only this sequence must be specified.
+        text_b: (Optional) string. The untokenized text of the second sequence.
+        Only must be specified for sequence pair tasks.
+        label: (Optional) string. The label of the example. This should be
+        specified for train and dev examples, but not for test examples.
+    """
+
+    def __init__(self, guid, text_a, text_b=None, label=None, pairID=None):
+        self.guid = guid
+        self.text_a = text_a
+        self.text_b = text_b
+        self.label = label
+        self.pairID = pairID
+
+    def __repr__(self):
+        return str(self.to_json_string())
+
+    def to_dict(self):
+        """Serializes this instance to a Python dictionary."""
+        output = copy.deepcopy(self.__dict__)
+        return output
+
+    def to_json_string(self):
+        """Serializes this instance to a JSON string."""
+        return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n"
+
+
+class InputFeatures(object):
+    """
+    A single set of features of data.
+
+    Args:
+        input_ids: Indices of input sequence tokens in the vocabulary.
+        attention_mask: Mask to avoid performing attention on padding token indices.
+            Mask values selected in ``[0, 1]``:
+            Usually  ``1`` for tokens that are NOT MASKED, ``0`` for MASKED (padded) tokens.
+        token_type_ids: Segment token indices to indicate first and second portions of the inputs.
+        label: Label corresponding to the input
+    """
+
+    def __init__(self, input_ids, attention_mask, token_type_ids, label, pairID=None):
+        self.input_ids = input_ids
+        self.attention_mask = attention_mask
+        self.token_type_ids = token_type_ids
+        self.label = label
+        self.pairID = pairID
+
+    def __repr__(self):
+        return str(self.to_json_string())
+
+    def to_dict(self):
+        """Serializes this instance to a Python dictionary."""
+        output = copy.deepcopy(self.__dict__)
+        return output
+
+    def to_json_string(self):
+        """Serializes this instance to a JSON string."""
+        return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n"
+
+
+class DataProcessor(object):
+    """Base class for data converters for sequence classification data sets."""
+
+    def get_example_from_tensor_dict(self, tensor_dict):
+        """Gets an example from a dict with tensorflow tensors
+
+        Args:
+            tensor_dict: Keys and values should match the corresponding Glue
+                tensorflow_dataset examples.
+        """
+        raise NotImplementedError()
+
+    def get_train_examples(self, data_dir):
+        """Gets a collection of `InputExample`s for the train set."""
+        raise NotImplementedError()
+
+    def get_dev_examples(self, data_dir):
+        """Gets a collection of `InputExample`s for the dev set."""
+        raise NotImplementedError()
+
+    def get_labels(self):
+        """Gets the list of labels for this data set."""
+        raise NotImplementedError()
+
+    @classmethod
+    def _read_tsv(cls, input_file, quotechar=None):
+        """Reads a tab separated value file."""
+        with open(input_file, "r", encoding="utf-8-sig") as f:
+            reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
+            lines = []
+            for line in reader:
+                lines.append(line)
+            return lines

examples/mm-imdb/utils_mmimdb.py

@@ -31,7 +31,7 @@ POOLING_BREAKDOWN = {1: (1, 1), 2: (2, 1), 3: (3, 1), 4: (2, 2), 5: (5, 1), 6: (
 
 class ImageEncoder(nn.Module):
     def __init__(self, args):
-        super(ImageEncoder, self).__init__()
+        super().__init__()
         model = torchvision.models.resnet152(pretrained=True)
         modules = list(model.children())[:-2]
         self.model = nn.Sequential(*modules)

examples/pplm/pplm_classification_head.py

@@ -5,7 +5,7 @@ class ClassificationHead(torch.nn.Module):
     """Classification Head for  transformer encoders"""
 
     def __init__(self, class_size, embed_size):
-        super(ClassificationHead, self).__init__()
+        super().__init__()
         self.class_size = class_size
         self.embed_size = embed_size
         # self.mlp1 = torch.nn.Linear(embed_size, embed_size)

examples/pplm/run_pplm.py

@@ -344,6 +344,7 @@ def full_text_generation(
     gamma=1.5,
     gm_scale=0.9,
     kl_scale=0.01,
+    repetition_penalty=1.0,
     **kwargs
 ):
     classifier, class_id = get_classifier(discrim, class_label, device)
@@ -368,7 +369,14 @@ def full_text_generation(
         raise Exception("Specify either a bag of words or a discriminator")
 
     unpert_gen_tok_text, _, _ = generate_text_pplm(
-        model=model, tokenizer=tokenizer, context=context, device=device, length=length, sample=sample, perturb=False
+        model=model,
+        tokenizer=tokenizer,
+        context=context,
+        device=device,
+        length=length,
+        sample=sample,
+        perturb=False,
+        repetition_penalty=repetition_penalty,
     )
     if device == "cuda":
         torch.cuda.empty_cache()
@@ -401,6 +409,7 @@ def full_text_generation(
             gamma=gamma,
             gm_scale=gm_scale,
             kl_scale=kl_scale,
+            repetition_penalty=repetition_penalty,
         )
         pert_gen_tok_texts.append(pert_gen_tok_text)
         if classifier is not None:
@@ -437,6 +446,7 @@ def generate_text_pplm(
     gamma=1.5,
     gm_scale=0.9,
     kl_scale=0.01,
+    repetition_penalty=1.0,
 ):
     output_so_far = None
     if context:
@@ -508,6 +518,13 @@ def generate_text_pplm(
 
         pert_logits, past, pert_all_hidden = model(last, past=pert_past)
         pert_logits = pert_logits[:, -1, :] / temperature  # + SMALL_CONST
+
+        for token_idx in set(output_so_far[0].tolist()):
+            if pert_logits[0, token_idx] < 0:
+                pert_logits[0, token_idx] *= repetition_penalty
+            else:
+                pert_logits[0, token_idx] /= repetition_penalty
+
         pert_probs = F.softmax(pert_logits, dim=-1)
 
         if classifier is not None:
@@ -588,6 +605,7 @@ def run_pplm_example(
     seed=0,
     no_cuda=False,
     colorama=False,
+    repetition_penalty=1.0,
 ):
     # set Random seed
     torch.manual_seed(seed)
@@ -655,6 +673,7 @@ def run_pplm_example(
         gamma=gamma,
         gm_scale=gm_scale,
         kl_scale=kl_scale,
+        repetition_penalty=repetition_penalty,
     )
 
     # untokenize unperturbed text
@@ -767,6 +786,9 @@ if __name__ == "__main__":
     parser.add_argument("--seed", type=int, default=0)
     parser.add_argument("--no_cuda", action="store_true", help="no cuda")
     parser.add_argument("--colorama", action="store_true", help="colors keywords")
+    parser.add_argument(
+        "--repetition_penalty", type=float, default=1.0, help="Penalize repetition. More than 1.0 -> less repetition",
+    )
 
     args = parser.parse_args()
     run_pplm_example(**vars(args))

examples/pplm/run_pplm_discrim_train.py

@@ -46,7 +46,7 @@ class Discriminator(torch.nn.Module):
     """Transformer encoder followed by a Classification Head"""
 
     def __init__(self, class_size, pretrained_model="gpt2-medium", cached_mode=False, device="cpu"):
-        super(Discriminator, self).__init__()
+        super().__init__()
         self.tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model)
         self.encoder = GPT2LMHeadModel.from_pretrained(pretrained_model)
         self.embed_size = self.encoder.transformer.config.hidden_size

examples/run_squad.py

@@ -586,7 +586,7 @@ def main():
     parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
     parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.")
     parser.add_argument(
-        "--evaluate_during_training", action="store_true", help="Rul evaluation during training at each logging step."
+        "--evaluate_during_training", action="store_true", help="Run evaluation during training at each logging step."
     )
     parser.add_argument(
         "--do_lower_case", action="store_true", help="Set this flag if you are using an uncased model."
@@ -671,6 +671,13 @@ def main():
     parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features")
     args = parser.parse_args()
 
+    if args.doc_stride >= args.max_seq_length - args.max_query_length:
+        logger.warning(
+            "WARNING - You've set a doc stride which may be superior to the document length in some "
+            "examples. This could result in errors when building features from the examples. Please reduce the doc "
+            "stride or increase the maximum length to ensure the features are correctly built."
+        )
+
     if (
         os.path.exists(args.output_dir)
         and os.listdir(args.output_dir)

examples/summarization/configuration_bertabs.py

@@ -62,6 +62,7 @@ class BertAbsConfig(PretrainedConfig):
     """
 
     pretrained_config_archive_map = BERTABS_FINETUNED_CONFIG_MAP
+    model_type = "bertabs"
 
     def __init__(
         self,
@@ -79,7 +80,7 @@ class BertAbsConfig(PretrainedConfig):
         dec_dropout=0.2,
         **kwargs,
     ):
-        super(BertAbsConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.vocab_size = vocab_size
         self.max_pos = max_pos

examples/summarization/modeling_bertabs.py

@@ -47,7 +47,7 @@ class BertAbsPreTrainedModel(PreTrainedModel):
 
 class BertAbs(BertAbsPreTrainedModel):
     def __init__(self, args, checkpoint=None, bert_extractive_checkpoint=None):
-        super(BertAbs, self).__init__(args)
+        super().__init__(args)
         self.args = args
         self.bert = Bert()
 
@@ -122,7 +122,7 @@ class Bert(nn.Module):
     """
 
     def __init__(self):
-        super(Bert, self).__init__()
+        super().__init__()
         config = BertConfig.from_pretrained("bert-base-uncased")
         self.model = BertModel(config)
 
@@ -151,7 +151,7 @@ class TransformerDecoder(nn.Module):
     """
 
     def __init__(self, num_layers, d_model, heads, d_ff, dropout, embeddings, vocab_size):
-        super(TransformerDecoder, self).__init__()
+        super().__init__()
 
         # Basic attributes.
         self.decoder_type = "transformer"
@@ -261,7 +261,7 @@ class PositionalEncoding(nn.Module):
         pe[:, 0::2] = torch.sin(position.float() * div_term)
         pe[:, 1::2] = torch.cos(position.float() * div_term)
         pe = pe.unsqueeze(0)
-        super(PositionalEncoding, self).__init__()
+        super().__init__()
         self.register_buffer("pe", pe)
         self.dropout = nn.Dropout(p=dropout)
         self.dim = dim
@@ -293,7 +293,7 @@ class TransformerDecoderLayer(nn.Module):
     """
 
     def __init__(self, d_model, heads, d_ff, dropout):
-        super(TransformerDecoderLayer, self).__init__()
+        super().__init__()
 
         self.self_attn = MultiHeadedAttention(heads, d_model, dropout=dropout)
 
@@ -410,7 +410,7 @@ class MultiHeadedAttention(nn.Module):
         self.dim_per_head = model_dim // head_count
         self.model_dim = model_dim
 
-        super(MultiHeadedAttention, self).__init__()
+        super().__init__()
         self.head_count = head_count
 
         self.linear_keys = nn.Linear(model_dim, head_count * self.dim_per_head)
@@ -639,7 +639,7 @@ class PositionwiseFeedForward(nn.Module):
     """
 
     def __init__(self, d_model, d_ff, dropout=0.1):
-        super(PositionwiseFeedForward, self).__init__()
+        super().__init__()
         self.w_1 = nn.Linear(d_model, d_ff)
         self.w_2 = nn.Linear(d_ff, d_model)
         self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)

setup.cfg

@@ -4,18 +4,23 @@ force_grid_wrap = 0
 include_trailing_comma = True
 known_first_party = transformers
 known_third_party =
+    absl
     fairseq
     fastprogress
     git
+    h5py
     MeCab
     nltk
+    numpy
     packaging
     PIL
     psutil
     seqeval
     sklearn
     tensorboardX
+    tensorflow
     tensorflow_datasets
+    torch
     torchtext
     torchvision
 

setup.py

@@ -94,7 +94,7 @@ setup(
         # for downloading models over HTTPS
         "requests",
         # progress bars in model download and training scripts
-        "tqdm",
+        "tqdm >= 4.27",
         # for OpenAI GPT
         "regex != 2019.12.17",
         # for XLNet

src/transformers/commands/serving.py

@@ -1,7 +1,8 @@
 import logging
 from argparse import ArgumentParser, Namespace
-from typing import Any, List, Optional, Union
+from typing import Any, List, Optional
 
+from starlette.responses import JSONResponse
 from transformers import Pipeline
 from transformers.commands import BaseTransformersCLICommand
 from transformers.pipelines import SUPPORTED_TASKS, pipeline
@@ -10,6 +11,7 @@ from transformers.pipelines import SUPPORTED_TASKS, pipeline
 try:
     from uvicorn import run
     from fastapi import FastAPI, HTTPException, Body
+    from fastapi.routing import APIRoute
     from pydantic import BaseModel
 
     _serve_dependancies_installed = True
@@ -37,7 +39,7 @@ def serve_command_factory(args: Namespace):
         tokenizer=args.tokenizer,
         device=args.device,
     )
-    return ServeCommand(nlp, args.host, args.port)
+    return ServeCommand(nlp, args.host, args.port, args.workers)
 
 
 class ServeModelInfoResult(BaseModel):
@@ -89,6 +91,7 @@ class ServeCommand(BaseTransformersCLICommand):
         )
         serve_parser.add_argument("--host", type=str, default="localhost", help="Interface the server will listen on.")
         serve_parser.add_argument("--port", type=int, default=8888, help="Port the serving will listen to.")
+        serve_parser.add_argument("--workers", type=int, default=1, help="Number of http workers")
         serve_parser.add_argument("--model", type=str, help="Model's name or path to stored model.")
         serve_parser.add_argument("--config", type=str, help="Model's config name or path to stored model.")
         serve_parser.add_argument("--tokenizer", type=str, help="Tokenizer name to use.")
@@ -100,12 +103,14 @@ class ServeCommand(BaseTransformersCLICommand):
         )
         serve_parser.set_defaults(func=serve_command_factory)
 
-    def __init__(self, pipeline: Pipeline, host: str, port: int):
+    def __init__(self, pipeline: Pipeline, host: str, port: int, workers: int):
 
         self._pipeline = pipeline
 
-        self._host = host
-        self._port = port
+        self.host = host
+        self.port = port
+        self.workers = workers
+
         if not _serve_dependancies_installed:
             raise RuntimeError(
                 "Using serve command requires FastAPI and unicorn. "
@@ -114,18 +119,42 @@ class ServeCommand(BaseTransformersCLICommand):
             )
         else:
             logger.info("Serving model over {}:{}".format(host, port))
-            self._app = FastAPI()
-
-            # Register routes
-            self._app.add_api_route("/", self.model_info, response_model=ServeModelInfoResult, methods=["GET"])
-            self._app.add_api_route("/tokenize", self.tokenize, response_model=ServeTokenizeResult, methods=["POST"])
-            self._app.add_api_route(
-                "/detokenize", self.detokenize, response_model=ServeDeTokenizeResult, methods=["POST"]
+            self._app = FastAPI(
+                routes=[
+                    APIRoute(
+                        "/",
+                        self.model_info,
+                        response_model=ServeModelInfoResult,
+                        response_class=JSONResponse,
+                        methods=["GET"],
+                    ),
+                    APIRoute(
+                        "/tokenize",
+                        self.tokenize,
+                        response_model=ServeTokenizeResult,
+                        response_class=JSONResponse,
+                        methods=["POST"],
+                    ),
+                    APIRoute(
+                        "/detokenize",
+                        self.detokenize,
+                        response_model=ServeDeTokenizeResult,
+                        response_class=JSONResponse,
+                        methods=["POST"],
+                    ),
+                    APIRoute(
+                        "/forward",
+                        self.forward,
+                        response_model=ServeForwardResult,
+                        response_class=JSONResponse,
+                        methods=["POST"],
+                    ),
+                ],
+                timeout=600,
             )
-            self._app.add_api_route("/forward", self.forward, response_model=ServeForwardResult, methods=["POST"])
 
     def run(self):
-        run(self._app, host=self._host, port=self._port)
+        run(self._app, host=self.host, port=self.port, workers=self.workers)
 
     def model_info(self):
         return ServeModelInfoResult(infos=vars(self._pipeline.model.config))
@@ -166,7 +195,7 @@ class ServeCommand(BaseTransformersCLICommand):
         except Exception as e:
             raise HTTPException(status_code=500, detail={"model": "", "error": str(e)})
 
-    def forward(self, inputs: Union[str, dict, List[str], List[int], List[dict]] = Body(None, embed=True)):
+    async def forward(self, inputs=Body(None, embed=True)):
         """
         **inputs**:
         **attention_mask**:

src/transformers/configuration_albert.py

@@ -31,12 +31,77 @@ ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 
 class AlbertConfig(PretrainedConfig):
-    """Configuration for `AlbertModel`.
+    r"""
+        This is the configuration class to store the configuration of an :class:`~transformers.AlbertModel`.
+        It is used to instantiate an ALBERT model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the ALBERT `xxlarge <https://huggingface.co/albert-xxlarge-v2>`__ architecture.
 
-    The default settings match the configuration of model `albert_xxlarge`.
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
+
+
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 30000):
+                Vocabulary size of the ALBERT model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.AlbertModel`.
+            embedding_size (:obj:`int`, optional, defaults to 128):
+                Dimensionality of vocabulary embeddings.
+            hidden_size (:obj:`int`, optional, defaults to 4096):
+                Dimensionality of the encoder layers and the pooler layer.
+            num_hidden_layers (:obj:`int`, optional, defaults to 12):
+                Number of hidden layers in the Transformer encoder.
+            num_hidden_groups (:obj:`int`, optional, defaults to 1):
+                Number of groups for the hidden layers, parameters in the same group are shared.
+            num_attention_heads (:obj:`int`, optional, defaults to 64):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            intermediate_size (:obj:`int`, optional, defaults to 16384):
+                The dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+            inner_group_num (:obj:`int`, optional, defaults to 1):
+                The number of inner repetition of attention and ffn.
+            hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "gelu_new"):
+                The non-linear activation function (function or string) in the encoder and pooler.
+                If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            hidden_dropout_prob (:obj:`float`, optional, defaults to 0):
+                The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0):
+                The dropout ratio for the attention probabilities.
+            max_position_embeddings (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might ever be used with. Typically set this to something
+                large (e.g., 512 or 1024 or 2048).
+            type_vocab_size (:obj:`int`, optional, defaults to 2):
+                The vocabulary size of the `token_type_ids` passed into :class:`~transformers.AlbertModel`.
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
+                The epsilon used by the layer normalization layers.
+
+        Example::
+
+            # Initializing an ALBERT-xxlarge style configuration
+            albert_xxlarge_configuration = AlbertConfig()
+
+            # Initializing an ALBERT-base style configuration
+            albert_base_configuration = AlbertConfig(
+                hidden_size=768,
+                num_attention_heads=12,
+                intermediate_size=3072,
+            )
+
+            # Initializing a model from the ALBERT-base style configuration
+            model = AlbertModel(bert_base_configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
     """
 
     pretrained_config_archive_map = ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "albert"
 
     def __init__(
         self,
@@ -57,36 +122,7 @@ class AlbertConfig(PretrainedConfig):
         layer_norm_eps=1e-12,
         **kwargs
     ):
-        """Constructs AlbertConfig.
-
-        Args:
-            vocab_size: Vocabulary size of `inputs_ids` in `AlbertModel`.
-            embedding_size: size of voc embeddings.
-            hidden_size: Size of the encoder layers and the pooler layer.
-            num_hidden_layers: Number of hidden layers in the Transformer encoder.
-            num_hidden_groups: Number of group for the hidden layers, parameters in
-                the same group are shared.
-            num_attention_heads: Number of attention heads for each attention layer in
-                the Transformer encoder.
-            intermediate_size: The size of the "intermediate" (i.e., feed-forward)
-                layer in the Transformer encoder.
-            inner_group_num: int, number of inner repetition of attention and ffn.
-            down_scale_factor: float, the scale to apply
-            hidden_act: The non-linear activation function (function or string) in the
-                encoder and pooler.
-            hidden_dropout_prob: The dropout probability for all fully connected
-                layers in the embeddings, encoder, and pooler.
-            attention_probs_dropout_prob: The dropout ratio for the attention
-                probabilities.
-            max_position_embeddings: The maximum sequence length that this model might
-                ever be used with. Typically set this to something large just in case
-                (e.g., 512 or 1024 or 2048).
-            type_vocab_size: The vocabulary size of the `token_type_ids` passed into
-                `AlbertModel`.
-            initializer_range: The stdev of the truncated_normal_initializer for
-                initializing all weight matrices.
-        """
-        super(AlbertConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.vocab_size = vocab_size
         self.embedding_size = embedding_size

src/transformers/configuration_auto.py

@@ -12,10 +12,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" Auto Model class. """
+""" Auto Config class. """
 
 
 import logging
+from collections import OrderedDict
 
 from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
 from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig
@@ -27,6 +28,7 @@ from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, Open
 from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig
 from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config
 from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
+from .configuration_utils import PretrainedConfig
 from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig
 from .configuration_xlm_roberta import XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMRobertaConfig
 from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig
@@ -56,30 +58,34 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
 )
 
 
+CONFIG_MAPPING = OrderedDict(
+    [
+        ("t5", T5Config,),
+        ("distilbert", DistilBertConfig,),
+        ("albert", AlbertConfig,),
+        ("camembert", CamembertConfig,),
+        ("xlm-roberta", XLMRobertaConfig,),
+        ("roberta", RobertaConfig,),
+        ("bert", BertConfig,),
+        ("openai-gpt", OpenAIGPTConfig,),
+        ("gpt2", GPT2Config,),
+        ("transfo-xl", TransfoXLConfig,),
+        ("xlnet", XLNetConfig,),
+        ("xlm", XLMConfig,),
+        ("ctrl", CTRLConfig,),
+    ]
+)
+
+
 class AutoConfig(object):
-    r""":class:`~transformers.AutoConfig` is a generic configuration class
+    r"""
+        :class:`~transformers.AutoConfig` is a generic configuration class
         that will be instantiated as one of the configuration classes of the library
-        when created with the `AutoConfig.from_pretrained(pretrained_model_name_or_path)`
-        class method.
+        when created with the :func:`~transformers.AutoConfig.from_pretrained` class method.
 
-        The `from_pretrained()` method take care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
-
-        The base model class to instantiate is selected as the first pattern matching
-        in the `pretrained_model_name_or_path` string (in the following order):
-            - contains `distilbert`: DistilBertConfig (DistilBERT model)
-            - contains `albert`: AlbertConfig (ALBERT model)
-            - contains `camembert`: CamembertConfig (CamemBERT model)
-            - contains `xlm-roberta`: XLMRobertaConfig (XLM-RoBERTa model)
-            - contains `roberta`: RobertaConfig (RoBERTa model)
-            - contains `bert`: BertConfig (Bert model)
-            - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model)
-            - contains `gpt2`: GPT2Config (OpenAI GPT-2 model)
-            - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model)
-            - contains `xlnet`: XLNetConfig (XLNet model)
-            - contains `xlm`: XLMConfig (XLM model)
-            - contains `ctrl` : CTRLConfig (CTRL model)
-        This class cannot be instantiated using `__init__()` (throw an error).
+        The :func:`~transformers.AutoConfig.from_pretrained` method takes care of returning the correct model class instance
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
     """
 
     def __init__(self):
@@ -90,89 +96,72 @@ class AutoConfig(object):
 
     @classmethod
     def for_model(cls, model_type, *args, **kwargs):
-        if "distilbert" in model_type:
-            return DistilBertConfig(*args, **kwargs)
-        elif "roberta" in model_type:
-            return RobertaConfig(*args, **kwargs)
-        elif "bert" in model_type:
-            return BertConfig(*args, **kwargs)
-        elif "openai-gpt" in model_type:
-            return OpenAIGPTConfig(*args, **kwargs)
-        elif "gpt2" in model_type:
-            return GPT2Config(*args, **kwargs)
-        elif "transfo-xl" in model_type:
-            return TransfoXLConfig(*args, **kwargs)
-        elif "xlnet" in model_type:
-            return XLNetConfig(*args, **kwargs)
-        elif "xlm" in model_type:
-            return XLMConfig(*args, **kwargs)
-        elif "ctrl" in model_type:
-            return CTRLConfig(*args, **kwargs)
-        elif "albert" in model_type:
-            return AlbertConfig(*args, **kwargs)
-        elif "camembert" in model_type:
-            return CamembertConfig(*args, **kwargs)
+        for pattern, config_class in CONFIG_MAPPING.items():
+            if pattern in model_type:
+                return config_class(*args, **kwargs)
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'distilbert', 'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-            "'xlm', 'roberta', 'ctrl', 'camembert', 'albert'".format(model_type)
+            "Unrecognized model identifier in {}. Should contain one of {}".format(
+                model_type, ", ".join(CONFIG_MAPPING.keys())
+            )
         )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
-        r""" Instantiate a one of the configuration classes of the library
+        r""" Instantiates one of the configuration classes of the library
         from a pre-trained model configuration.
 
-        The configuration class to instantiate is selected as the first pattern matching
-        in the `pretrained_model_name_or_path` string (in the following order):
-            - contains `t5`: T5Config (T5 model)
-            - contains `distilbert`: DistilBertConfig (DistilBERT model)
-            - contains `albert`: AlbertConfig (ALBERT model)
-            - contains `camembert`: CamembertConfig (CamemBERT model)
-            - contains `xlm-roberta`: XLMRobertaConfig (XLM-RoBERTa model)
-            - contains `roberta`: RobertaConfig (RoBERTa model)
-            - contains `bert`: BertConfig (Bert model)
-            - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model)
-            - contains `gpt2`: GPT2Config (OpenAI GPT-2 model)
-            - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model)
-            - contains `xlnet`: XLNetConfig (XLNet model)
-            - contains `xlm`: XLMConfig (XLM model)
-            - contains `ctrl` : CTRLConfig (CTRL model)
-        Params:
-            pretrained_model_name_or_path: either:
+        The configuration class to instantiate is selected
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
+            - contains `t5`: :class:`~transformers.T5Config` (T5 model)
+            - contains `distilbert`: :class:`~transformers.DistilBertConfig` (DistilBERT model)
+            - contains `albert`: :class:`~transformers.AlbertConfig` (ALBERT model)
+            - contains `camembert`: :class:`~transformers.CamembertConfig` (CamemBERT model)
+            - contains `xlm-roberta`: :class:`~transformers.XLMRobertaConfig` (XLM-RoBERTa model)
+            - contains `roberta`: :class:`~transformers.RobertaConfig` (RoBERTa model)
+            - contains `bert`: :class:`~transformers.BertConfig` (Bert model)
+            - contains `openai-gpt`: :class:`~transformers.OpenAIGPTConfig` (OpenAI GPT model)
+            - contains `gpt2`: :class:`~transformers.GPT2Config` (OpenAI GPT-2 model)
+            - contains `transfo-xl`: :class:`~transformers.TransfoXLConfig` (Transformer-XL model)
+            - contains `xlnet`: :class:`~transformers.XLNetConfig` (XLNet model)
+            - contains `xlm`: :class:`~transformers.XLMConfig` (XLM model)
+            - contains `ctrl` : :class:`~transformers.CTRLConfig` (CTRL model)
 
-                - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``.
-                - a string with the `identifier name` of a pre-trained model configuration that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``.
-                - a path to a `directory` containing a configuration file saved using the :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``.
-                - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``.
 
-            cache_dir: (`optional`) string:
+        Args:
+            pretrained_model_name_or_path (:obj:`string`):
+                Is either: \
+                    - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``.
+                    - a string with the `identifier name` of a pre-trained model configuration that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``.
+                    - a path to a `directory` containing a configuration file saved using the :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``.
+                    - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``.
+
+            cache_dir (:obj:`string`, optional, defaults to `None`):
                 Path to a directory in which a downloaded pre-trained model
                 configuration should be cached if the standard cache should not be used.
 
-            kwargs: (`optional`) dict: key/value pairs with which to update the configuration object after loading.
+            force_download (:obj:`boolean`, optional, defaults to `False`):
+                Force to (re-)download the model weights and configuration files and override the cached versions if they exist.
 
-                - The values in kwargs of any keys which are configuration attributes will be used to override the loaded values.
-                - Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled by the `return_unused_kwargs` keyword parameter.
+            resume_download (:obj:`boolean`, optional, defaults to `False`):
+                Do not delete incompletely received file. Attempt to resume the download if such a file exists.
 
-            force_download: (`optional`) boolean, default False:
-                Force to (re-)download the model weights and configuration files and override the cached versions if they exists.
-
-            resume_download: (`optional`) boolean, default False:
-                Do not delete incompletely recieved file. Attempt to resume the download if such a file exists.
-
-            proxies: (`optional`) dict, default None:
-                A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.
-                The proxies are used on each request.
-
-            return_unused_kwargs: (`optional`) bool:
+            proxies (:obj:`Dict[str, str]`, optional, defaults to `None`):
+                A dictionary of proxy servers to use by protocol or endpoint, e.g.: :obj:`{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`.
+                The proxies are used on each request. See `the requests documentation <https://requests.readthedocs.io/en/master/user/advanced/#proxies>`__ for usage.
 
+            return_unused_kwargs (:obj:`boolean`, optional, defaults to `False`):
                 - If False, then this function returns just the final configuration object.
                 - If True, then this functions returns a tuple `(config, unused_kwargs)` where `unused_kwargs` is a dictionary consisting of the key/value pairs whose keys are not configuration attributes: ie the part of kwargs which has not been used to update `config` and is otherwise ignored.
 
+            kwargs (:obj:`Dict[str, any]`, optional, defaults to `{}`): key/value pairs with which to update the configuration object after loading.
+                - The values in kwargs of any keys which are configuration attributes will be used to override the loaded values.
+                - Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled by the `return_unused_kwargs` keyword parameter.
+
+
         Examples::
 
-            config = AutoConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
+            config = AutoConfig.from_pretrained('bert-base-uncased')  # Download configuration from S3 and cache.
             config = AutoConfig.from_pretrained('./test/bert_saved_model/')  # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')`
             config = AutoConfig.from_pretrained('./test/bert_saved_model/my_configuration.json')
             config = AutoConfig.from_pretrained('bert-base-uncased', output_attention=True, foo=False)
@@ -183,36 +172,21 @@ class AutoConfig(object):
             assert unused_kwargs == {'foo': False}
 
         """
-        if "t5" in pretrained_model_name_or_path:
-            return T5Config.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "distilbert" in pretrained_model_name_or_path:
-            return DistilBertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "albert" in pretrained_model_name_or_path:
-            return AlbertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "camembert" in pretrained_model_name_or_path:
-            return CamembertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "xlm-roberta" in pretrained_model_name_or_path:
-            return XLMRobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "roberta" in pretrained_model_name_or_path:
-            return RobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "bert" in pretrained_model_name_or_path:
-            return BertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "openai-gpt" in pretrained_model_name_or_path:
-            return OpenAIGPTConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "gpt2" in pretrained_model_name_or_path:
-            return GPT2Config.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "transfo-xl" in pretrained_model_name_or_path:
-            return TransfoXLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return XLNetConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "xlm" in pretrained_model_name_or_path:
-            return XLMConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-        elif "ctrl" in pretrained_model_name_or_path:
-            return CTRLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+        config_dict, _ = PretrainedConfig.get_config_dict(
+            pretrained_model_name_or_path, pretrained_config_archive_map=ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, **kwargs
+        )
+
+        if "model_type" in config_dict:
+            config_class = CONFIG_MAPPING[config_dict["model_type"]]
+            return config_class.from_dict(config_dict, **kwargs)
+        else:
+            # Fallback: use pattern matching on the string.
+            for pattern, config_class in CONFIG_MAPPING.items():
+                if pattern in pretrained_model_name_or_path:
+                    return config_class.from_dict(config_dict, **kwargs)
+
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-            "'xlm-roberta', 'xlm', 'roberta', 'distilbert', 'camembert', 'ctrl', 'albert'".format(
-                pretrained_model_name_or_path
-            )
+            "Unrecognized model in {}. "
+            "Should have a `model_type` key in its config.json, or contain one of the following strings "
+            "in its name: {}".format(pretrained_model_name_or_path, ", ".join(CONFIG_MAPPING.keys()))
         )

src/transformers/configuration_bert.py

@@ -50,34 +50,64 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 class BertConfig(PretrainedConfig):
     r"""
-        :class:`~transformers.BertConfig` is the configuration class to store the configuration of a
-        `BertModel`.
+        This is the configuration class to store the configuration of a :class:`~transformers.BertModel`.
+        It is used to instantiate an BERT model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the BERT `bert-base-uncased <https://huggingface.co/bert-base-uncased>`__ architecture.
+
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
 
 
-        Arguments:
-            vocab_size: Vocabulary size of `inputs_ids` in `BertModel`.
-            hidden_size: Size of the encoder layers and the pooler layer.
-            num_hidden_layers: Number of hidden layers in the Transformer encoder.
-            num_attention_heads: Number of attention heads for each attention layer in
-                the Transformer encoder.
-            intermediate_size: The size of the "intermediate" (i.e., feed-forward)
-                layer in the Transformer encoder.
-            hidden_act: The non-linear activation function (function or string) in the
-                encoder and pooler. If string, "gelu", "relu", "swish" and "gelu_new" are supported.
-            hidden_dropout_prob: The dropout probabilitiy for all fully connected
-                layers in the embeddings, encoder, and pooler.
-            attention_probs_dropout_prob: The dropout ratio for the attention
-                probabilities.
-            max_position_embeddings: The maximum sequence length that this model might
-                ever be used with. Typically set this to something large just in case
-                (e.g., 512 or 1024 or 2048).
-            type_vocab_size: The vocabulary size of the `token_type_ids` passed into
-                `BertModel`.
-            initializer_range: The sttdev of the truncated_normal_initializer for
-                initializing all weight matrices.
-            layer_norm_eps: The epsilon used by LayerNorm.
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 30522):
+                Vocabulary size of the BERT model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.BertModel`.
+            hidden_size (:obj:`int`, optional, defaults to 768):
+                Dimensionality of the encoder layers and the pooler layer.
+            num_hidden_layers (:obj:`int`, optional, defaults to 12):
+                Number of hidden layers in the Transformer encoder.
+            num_attention_heads (:obj:`int`, optional, defaults to 12):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            intermediate_size (:obj:`int`, optional, defaults to 3072):
+                Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+            hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
+                The non-linear activation function (function or string) in the encoder and pooler.
+                If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            hidden_dropout_prob (:obj:`float`, optional, defaults to 0.1):
+                The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention probabilities.
+            max_position_embeddings (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            type_vocab_size (:obj:`int`, optional, defaults to 2):
+                The vocabulary size of the `token_type_ids` passed into :class:`~transformers.BertModel`.
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
+                The epsilon used by the layer normalization layers.
+
+        Example::
+
+            from transformers import BertModel, BertConfig
+
+            # Initializing a BERT bert-base-uncased style configuration
+            configuration = BertConfig()
+
+            # Initializing a model from the bert-base-uncased style configuration
+            model = BertModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
     """
     pretrained_config_archive_map = BERT_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "bert"
 
     def __init__(
         self,
@@ -95,7 +125,8 @@ class BertConfig(PretrainedConfig):
         layer_norm_eps=1e-12,
         **kwargs
     ):
-        super(BertConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
+
         self.vocab_size = vocab_size
         self.hidden_size = hidden_size
         self.num_hidden_layers = num_hidden_layers

src/transformers/configuration_camembert.py

@@ -29,4 +29,35 @@ CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 
 class CamembertConfig(RobertaConfig):
+    r"""
+        This is the configuration class to store the configuration of an :class:`~transformers.CamembertModel`.
+        It is used to instantiate an Camembert model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the BERT `bert-base-uncased <https://huggingface.co/bert-base-uncased>`__ architecture.
+
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
+
+        The :class:`~transformers.CamembertConfig` class directly inherits :class:`~transformers.BertConfig`.
+        It reuses the same defaults. Please check the parent class for more information.
+
+        Example::
+
+            from transformers import CamembertModel, CamembertConfig
+
+            # Initializing a CamemBERT configuration
+            configuration = CamembertConfig()
+
+            # Initializing a model from the configuration
+            model = CamembertModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
+    """
     pretrained_config_archive_map = CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "camembert"

src/transformers/configuration_ctrl.py

@@ -26,28 +26,64 @@ CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP = {"ctrl": "https://storage.googleapis.com/sf
 
 
 class CTRLConfig(PretrainedConfig):
-    """Configuration class to store the configuration of a `CTRLModel`.
+    """
+        This is the configuration class to store the configuration of an :class:`~transformers.CTRLModel`.
+        It is used to instantiate an CTRL model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the `ctrl <https://huggingface.co/ctrl>`__ architecture from SalesForce.
 
-    Args:
-        vocab_size: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file.
-        n_positions: Number of positional embeddings.
-        n_ctx: Size of the causal mask (usually same as n_positions).
-        dff: Size of the inner dimension of the FFN.
-        n_embd: Dimensionality of the embeddings and hidden states.
-        n_layer: Number of hidden layers in the Transformer encoder.
-        n_head: Number of attention heads for each attention layer in
-            the Transformer encoder.
-        layer_norm_epsilon: epsilon to use in the layer norm layers
-        resid_pdrop: The dropout probabilitiy for all fully connected
-            layers in the embeddings, encoder, and pooler.
-        attn_pdrop: The dropout ratio for the attention
-            probabilities.
-        embd_pdrop: The dropout ratio for the embeddings.
-        initializer_range: The sttdev of the truncated_normal_initializer for
-            initializing all weight matrices.
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
+
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 246534):
+                Vocabulary size of the CTRL model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.CTRLModel`.
+            n_positions (:obj:`int`, optional, defaults to 256):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            n_ctx (:obj:`int`, optional, defaults to 256):
+                Dimensionality of the causal mask (usually same as n_positions).
+            n_embd (:obj:`int`, optional, defaults to 1280):
+                Dimensionality of the embeddings and hidden states.
+            dff (:obj:`int`, optional, defaults to 8192):
+                Dimensionality of the inner dimension of the FFN.
+            n_layer (:obj:`int`, optional, defaults to 48):
+                Number of hidden layers in the Transformer encoder.
+            n_head (:obj:`int`, optional, defaults to 16):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            resid_pdrop (:obj:`float`, optional, defaults to 0.1):
+                The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+            embd_pdrop (:obj:`int`, optional, defaults to 0.1):
+                The dropout ratio for the embeddings.
+            attn_pdrop (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention.
+            layer_norm_epsilon (:obj:`float`, optional, defaults to 1e-6):
+                The epsilon to use in the layer normalization layers
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+
+        Example::
+
+            from transformers import CTRLModel, CTRLConfig
+
+            # Initializing a CTRL configuration
+            configuration = CTRLConfig()
+
+            # Initializing a model from the configuration
+            model = CTRLModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
     """
 
     pretrained_config_archive_map = CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "ctrl"
 
     def __init__(
         self,
@@ -70,27 +106,7 @@ class CTRLConfig(PretrainedConfig):
         summary_first_dropout=0.1,
         **kwargs
     ):
-        """Constructs CTRLConfig.
-
-        Args:
-            vocab_size: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file.
-            n_positions: Number of positional embeddings.
-            n_ctx: Size of the causal mask (usually same as n_positions).
-            dff: Size of the inner dimension of the FFN.
-            n_embd: Dimensionality of the embeddings and hidden states.
-            n_layer: Number of hidden layers in the Transformer encoder.
-            n_head: Number of attention heads for each attention layer in
-                the Transformer encoder.
-            layer_norm_epsilon: epsilon to use in the layer norm layers
-            resid_pdrop: The dropout probabilitiy for all fully connected
-                layers in the embeddings, encoder, and pooler.
-            attn_pdrop: The dropout ratio for the attention
-                probabilities.
-            embd_pdrop: The dropout ratio for the embeddings.
-            initializer_range: The sttdev of the truncated_normal_initializer for
-                initializing all weight matrices.
-        """
-        super(CTRLConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = vocab_size
         self.n_ctx = n_ctx
         self.n_positions = n_positions

src/transformers/configuration_distilbert.py

@@ -27,11 +27,74 @@ DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "distilbert-base-uncased-distilled-squad": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json",
     "distilbert-base-german-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-config.json",
     "distilbert-base-multilingual-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-config.json",
+    "distilbert-base-uncased-finetuned-sst-2-english": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json",
 }
 
 
 class DistilBertConfig(PretrainedConfig):
+    r"""
+        This is the configuration class to store the configuration of a :class:`~transformers.DistilBertModel`.
+        It is used to instantiate a DistilBERT model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the DistilBERT `distilbert-base-uncased <https://huggingface.co/distilbert-base-uncased>`__ architecture.
+
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
+
+
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 30522):
+                Vocabulary size of the DistilBERT model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.BertModel`.
+            max_position_embeddings (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            sinusoidal_pos_embds (:obj:`boolean`, optional, defaults to :obj:`False`):
+                Whether to use sinusoidal positional embeddings.
+            n_layers (:obj:`int`, optional, defaults to 6):
+                Number of hidden layers in the Transformer encoder.
+            n_heads (:obj:`int`, optional, defaults to 12):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            dim (:obj:`int`, optional, defaults to 768):
+                Dimensionality of the encoder layers and the pooler layer.
+            intermediate_size (:obj:`int`, optional, defaults to 3072):
+                The size of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+            dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
+            attention_dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention probabilities.
+            activation (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
+                The non-linear activation function (function or string) in the encoder and pooler.
+                If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            qa_dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout probabilities used in the question answering model
+                :class:`~tranformers.DistilBertForQuestionAnswering`.
+            seq_classif_dropout (:obj:`float`, optional, defaults to 0.2):
+                The dropout probabilities used in the sequence classification model
+                :class:`~tranformers.DistilBertForSequenceClassification`.
+
+        Example::
+
+            from transformers import DistilBertModel, DistilBertConfig
+
+            # Initializing a DistilBERT configuration
+            configuration = DistilBertConfig()
+
+            # Initializing a model from the configuration
+            model = DistilBertModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
+    """
     pretrained_config_archive_map = DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "distilbert"
 
     def __init__(
         self,
@@ -46,12 +109,11 @@ class DistilBertConfig(PretrainedConfig):
         attention_dropout=0.1,
         activation="gelu",
         initializer_range=0.02,
-        tie_weights_=True,
         qa_dropout=0.1,
         seq_classif_dropout=0.2,
         **kwargs
     ):
-        super(DistilBertConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = vocab_size
         self.max_position_embeddings = max_position_embeddings
         self.sinusoidal_pos_embds = sinusoidal_pos_embds
@@ -63,7 +125,6 @@ class DistilBertConfig(PretrainedConfig):
         self.attention_dropout = attention_dropout
         self.activation = activation
         self.initializer_range = initializer_range
-        self.tie_weights_ = tie_weights_
         self.qa_dropout = qa_dropout
         self.seq_classif_dropout = seq_classif_dropout
 

src/transformers/configuration_gpt2.py

@@ -33,27 +33,88 @@ GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 
 class GPT2Config(PretrainedConfig):
-    """Configuration class to store the configuration of a `GPT2Model`.
+    """
+        This is the configuration class to store the configuration of a :class:`~transformers.GPT2Model`.
+        It is used to instantiate an GPT-2 model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the GPT-2 `small <https://huggingface.co/gpt2>`__ architecture.
 
-    Args:
-        vocab_size: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file.
-        n_positions: Number of positional embeddings.
-        n_ctx: Size of the causal mask (usually same as n_positions).
-        n_embd: Dimensionality of the embeddings and hidden states.
-        n_layer: Number of hidden layers in the Transformer encoder.
-        n_head: Number of attention heads for each attention layer in
-            the Transformer encoder.
-        layer_norm_epsilon: epsilon to use in the layer norm layers
-        resid_pdrop: The dropout probabilitiy for all fully connected
-            layers in the embeddings, encoder, and pooler.
-        attn_pdrop: The dropout ratio for the attention
-            probabilities.
-        embd_pdrop: The dropout ratio for the embeddings.
-        initializer_range: The sttdev of the truncated_normal_initializer for
-            initializing all weight matrices.
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
+
+
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 50257):
+                Vocabulary size of the GPT-2 model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.GPT2Model`.
+            n_positions (:obj:`int`, optional, defaults to 1024):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            n_ctx (:obj:`int`, optional, defaults to 1024):
+                Dimensionality of the causal mask (usually same as n_positions).
+            n_embd (:obj:`int`, optional, defaults to 768):
+                Dimensionality of the embeddings and hidden states.
+            n_layer (:obj:`int`, optional, defaults to 12):
+                Number of hidden layers in the Transformer encoder.
+            n_head (:obj:`int`, optional, defaults to 12):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            resid_pdrop (:obj:`float`, optional, defaults to 0.1):
+                The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+            embd_pdrop (:obj:`int`, optional, defaults to 0.1):
+                The dropout ratio for the embeddings.
+            attn_pdrop (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention.
+            layer_norm_epsilon (:obj:`float`, optional, defaults to 1e-5):
+                The epsilon to use in the layer normalization layers
+            initializer_range (:obj:`float`, optional, defaults to 16):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            summary_type (:obj:`string`, optional, defaults to "cls_index"):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.GPT2DoubleHeadsModel`.
+                Is one of the following options:
+                    - 'last' => take the last token hidden state (like XLNet)
+                    - 'first' => take the first token hidden state (like Bert)
+                    - 'mean' => take the mean of all tokens hidden states
+                    - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2)
+                    - 'attn' => Not implemented now, use multi-head attention
+            summary_use_proj (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.GPT2DoubleHeadsModel`.
+                Add a projection after the vector extraction
+            summary_activation (:obj:`string` or :obj:`None`, optional, defaults to :obj:`None`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.GPT2DoubleHeadsModel`.
+                'tanh' => add a tanh activation to the output, Other => no activation.
+            summary_proj_to_labels (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.GPT2DoubleHeadsModel`.
+                If True, the projection outputs to config.num_labels classes (otherwise to hidden_size). Default: False.
+            summary_first_dropout (:obj:`float`, optional, defaults to 0.1):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.GPT2DoubleHeadsModel`.
+                Add a dropout before the projection and activation
+
+        Example::
+
+            from transformers import GPT2Model, GPT2Config
+
+            # Initializing a GPT2 configuration
+            configuration = GPT2Config()
+
+            # Initializing a model from the configuration
+            model = GPT2Model(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
     """
 
     pretrained_config_archive_map = GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "gpt2"
 
     def __init__(
         self,
@@ -75,26 +136,8 @@ class GPT2Config(PretrainedConfig):
         summary_first_dropout=0.1,
         **kwargs
     ):
-        """Constructs GPT2Config.
+        super().__init__(**kwargs)
 
-        Args:
-            vocab_size: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file.
-            n_positions: Number of positional embeddings.
-            n_ctx: Size of the causal mask (usually same as n_positions).
-            n_embd: Dimensionality of the embeddings and hidden states.
-            n_layer: Number of hidden layers in the Transformer encoder.
-            n_head: Number of attention heads for each attention layer in
-                the Transformer encoder.
-            layer_norm_epsilon: epsilon to use in the layer norm layers
-            resid_pdrop: The dropout probabilitiy for all fully connected
-                layers in the embeddings, encoder, and pooler.
-            attn_pdrop: The dropout ratio for the attention
-                probabilities.
-            embd_pdrop: The dropout ratio for the embeddings.
-            initializer_range: The sttdev of the truncated_normal_initializer for
-                initializing all weight matrices.
-        """
-        super(GPT2Config, self).__init__(**kwargs)
         self.vocab_size = vocab_size
         self.n_ctx = n_ctx
         self.n_positions = n_positions

src/transformers/configuration_mmbt.py

@@ -26,9 +26,13 @@ class MMBTConfig(object):
     """Configuration class to store the configuration of a `MMBT Model`.
 
     Args:
-        config: config of the underlying Transformer models. It's values are copied over to use a single config.
-        num_labels: Size of final Linear layer for classification.
-        modal_hidden_size: Embedding dimension of the non-text modality encoder.
+        config (:obj:`~transformers.PreTrainedConfig`):
+            Config of the underlying Transformer models. Its values are
+            copied over to use a single config.
+        num_labels (:obj:`int` or :obj:`None`, optional, defaults to `None`):
+            Size of final Linear layer for classification.
+        modal_hidden_size (:obj:`int`, optional, defautls to 2048):
+            Embedding dimension of the non-text modality encoder.
     """
 
     def __init__(self, config, num_labels=None, modal_hidden_size=2048):

src/transformers/configuration_openai.py

@@ -30,30 +30,91 @@ OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 class OpenAIGPTConfig(PretrainedConfig):
     """
-    Configuration class to store the configuration of a `OpenAIGPTModel`.
+        This is the configuration class to store the configuration of an :class:`~transformers.OpenAIGPTModel`.
+        It is used to instantiate an GPT model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the `GPT <https://huggingface.co/openai-gpt>`__ architecture from OpenAI.
 
-    Args:
-        vocab_size: Vocabulary size of `inputs_ids` in `OpenAIGPTModel` or a configuration json file.
-        n_positions: Number of positional embeddings.
-        n_ctx: Size of the causal mask (usually same as n_positions).
-        n_embd: Dimensionality of the embeddings and hidden states.
-        n_layer: Number of hidden layers in the Transformer encoder.
-        n_head: Number of attention heads for each attention layer in
-            the Transformer encoder.
-        afn: The non-linear activation function (function or string) in the
-            encoder and pooler. If string, "gelu", "relu" and "swish" are supported.
-        resid_pdrop: The dropout probabilitiy for all fully connected
-            layers in the embeddings, encoder, and pooler.
-        attn_pdrop: The dropout ratio for the attention
-            probabilities.
-        embd_pdrop: The dropout ratio for the embeddings.
-        layer_norm_epsilon: epsilon to use in the layer norm layers
-        initializer_range: The sttdev of the truncated_normal_initializer for
-            initializing all weight matrices.
-        predict_special_tokens: should we predict special tokens (when the model has a LM head)
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
+
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 40478):
+                Vocabulary size of the GPT model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.CTRLModel`.
+            n_positions (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            n_ctx (:obj:`int`, optional, defaults to 512):
+                Dimensionality of the causal mask (usually same as n_positions).
+            n_embd (:obj:`int`, optional, defaults to 768):
+                Dimensionality of the embeddings and hidden states.
+            n_layer (:obj:`int`, optional, defaults to 12):
+                Number of hidden layers in the Transformer encoder.
+            n_head (:obj:`int`, optional, defaults to 12):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            afn (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
+                The non-linear activation function (function or string) in the encoder and pooler.
+                If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            resid_pdrop (:obj:`float`, optional, defaults to 0.1):
+                The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+            embd_pdrop (:obj:`int`, optional, defaults to 0.1):
+                The dropout ratio for the embeddings.
+            attn_pdrop (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention.
+            layer_norm_epsilon (:obj:`float`, optional, defaults to 1e-5):
+                The epsilon to use in the layer normalization layers
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            predict_special_tokens (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Whether special tokens should be predicted when the model is has a language modeling head.
+            summary_type (:obj:`string`, optional, defaults to "cls_index"):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.OpenAIGPTDoubleHeadsModel`.
+                Is one of the following options:
+                    - 'last' => take the last token hidden state (like XLNet)
+                    - 'first' => take the first token hidden state (like Bert)
+                    - 'mean' => take the mean of all tokens hidden states
+                    - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2)
+                    - 'attn' => Not implemented now, use multi-head attention
+            summary_use_proj (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.OpenAIGPTDoubleHeadsModel`.
+                Add a projection after the vector extraction
+            summary_activation (:obj:`string` or :obj:`None`, optional, defaults to :obj:`None`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.OpenAIGPTDoubleHeadsModel`.
+                'tanh' => add a tanh activation to the output, Other => no activation.
+            summary_proj_to_labels (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.OpenAIGPTDoubleHeadsModel`.
+                If True, the projection outputs to config.num_labels classes (otherwise to hidden_size). Default: False.
+            summary_first_dropout (:obj:`float`, optional, defaults to 0.1):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.OpenAIGPTDoubleHeadsModel`.
+                Add a dropout before the projection and activation
+
+        Example::
+
+            from transformers import OpenAIGPTConfig, OpenAIGPTModel
+
+            # Initializing a GPT configuration
+            configuration = OpenAIGPTConfig()
+
+            # Initializing a model from the configuration
+            model = OpenAIGPTModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
     """
 
     pretrained_config_archive_map = OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "openai-gpt"
 
     def __init__(
         self,
@@ -77,9 +138,8 @@ class OpenAIGPTConfig(PretrainedConfig):
         summary_first_dropout=0.1,
         **kwargs
     ):
-        """Constructs OpenAIGPTConfig.
-        """
-        super(OpenAIGPTConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
+
         self.vocab_size = vocab_size
         self.n_ctx = n_ctx
         self.n_positions = n_positions

src/transformers/configuration_roberta.py

@@ -34,4 +34,35 @@ ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 
 class RobertaConfig(BertConfig):
+    r"""
+        This is the configuration class to store the configuration of an :class:`~transformers.RobertaModel`.
+        It is used to instantiate an RoBERTa model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the BERT `bert-base-uncased <https://huggingface.co/bert-base-uncased>`__ architecture.
+
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
+
+        The :class:`~transformers.RobertaConfig` class directly inherits :class:`~transformers.BertConfig`.
+        It reuses the same defaults. Please check the parent class for more information.
+
+        Example::
+
+            from transformers import RobertaConfig, RobertaModel
+
+            # Initializing a RoBERTa configuration
+            configuration = RobertaConfig()
+
+            # Initializing a model from the configuration
+            model = RobertaModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
+    """
     pretrained_config_archive_map = ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "roberta"

src/transformers/configuration_t5.py

@@ -60,6 +60,7 @@ class T5Config(PretrainedConfig):
             layer_norm_eps: The epsilon used by LayerNorm.
     """
     pretrained_config_archive_map = T5_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "t5"
 
     def __init__(
         self,
@@ -76,7 +77,7 @@ class T5Config(PretrainedConfig):
         initializer_factor=1.0,
         **kwargs
     ):
-        super(T5Config, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = vocab_size
         self.n_positions = n_positions
         self.d_model = d_model

src/transformers/configuration_transfo_xl.py

@@ -29,42 +29,95 @@ TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 
 class TransfoXLConfig(PretrainedConfig):
-    """Configuration class to store the configuration of a `TransfoXLModel`.
+    """
+        This is the configuration class to store the configuration of an :class:`~transformers.TransfoXLModel`.
+        It is used to instantiate a Transformer XL model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the `Transformer XL <https://huggingface.co/transfo-xl-wt103>`__ architecture.
+
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
 
         Args:
-            vocab_size: Vocabulary size of `inputs_ids` in `TransfoXLModel` or a configuration json file.
-            cutoffs: cutoffs for the adaptive softmax
-            d_model: Dimensionality of the model's hidden states.
-            d_embed: Dimensionality of the embeddings
-            d_head: Dimensionality of the model's heads.
-            div_val: divident value for adapative input and softmax
-            pre_lnorm: apply LayerNorm to the input instead of the output
-            d_inner: Inner dimension in FF
-            n_layer: Number of hidden layers in the Transformer encoder.
-            n_head: Number of attention heads for each attention layer in
-                the Transformer encoder.
-            tgt_len: number of tokens to predict
-            ext_len: length of the extended context
-            mem_len: length of the retained previous heads
-            same_length: use the same attn length for all tokens
-            proj_share_all_but_first: True to share all but first projs, False not to share.
-            attn_type: attention type. 0 for Transformer-XL, 1 for Shaw et al, 2 for Vaswani et al, 3 for Al Rfou et al.
-            clamp_len: use the same pos embeddings after clamp_len
-            sample_softmax: number of samples in sampled softmax
-            adaptive: use adaptive softmax
-            tie_weight: tie the word embedding and softmax weights
-            dropout: The dropout probabilitiy for all fully connected
-                layers in the embeddings, encoder, and pooler.
-            dropatt: The dropout ratio for the attention probabilities.
-            untie_r: untie relative position biases
-            embd_pdrop: The dropout ratio for the embeddings.
-            init: parameter initializer to use
-            init_range: parameters initialized by U(-init_range, init_range).
-            proj_init_std: parameters initialized by N(0, init_std)
-            init_std: parameters initialized by N(0, init_std)
+            vocab_size (:obj:`int`, optional, defaults to 267735):
+                Vocabulary size of the Transformer XL model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.TransfoXLModel`.
+            cutoffs (:obj:`List[int]`, optional, defaults to :obj:`[20000, 40000, 200000]`):
+                Cutoffs for the adaptive softmax
+            d_model (:obj:`int`, optional, defaults to 1024):
+                Dimensionality of the model's hidden states.
+            d_embed (:obj:`int`, optional, defaults to 1024):
+                Dimensionality of the embeddings
+            n_head (:obj:`int`, optional, defaults to 16):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            d_head (:obj:`int`, optional, defaults to 64):
+                Dimensionality of the model's heads.
+            d_inner (:obj:`int`, optional, defaults to 4096):
+                Inner dimension in FF
+            div_val (:obj:`int`, optional, defaults to 4):
+                Divident value for adapative input and softmax
+            pre_lnorm (:obj:`boolean`, optional, defaults to :obj:`False`):
+                Apply LayerNorm to the input instead of the output
+            n_layer (:obj:`int`, optional, defaults to 18):
+                Number of hidden layers in the Transformer encoder.
+            tgt_len (:obj:`int`, optional, defaults to 128):
+                Number of tokens to predict
+            ext_len (:obj:`int`, optional, defaults to 0):
+                Length of the extended context
+            mem_len (:obj:`int`, optional, defaults to 1600):
+                Length of the retained previous heads
+            clamp_len (:obj:`int`, optional, defaults to 1000):
+                use the same pos embeddings after clamp_len
+            same_length (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Use the same attn length for all tokens
+            proj_share_all_but_first (:obj:`boolean`, optional, defaults to :obj:`True`):
+                True to share all but first projs, False not to share.
+            attn_type (:obj:`int`, optional, defaults to 0):
+                Attention type. 0 for Transformer-XL, 1 for Shaw et al, 2 for Vaswani et al, 3 for Al Rfou et al.
+            sample_softmax (:obj:`int`, optional, defaults to -1):
+                number of samples in sampled softmax
+            adaptive (:obj:`boolean`, optional, defaults to :obj:`True`):
+                use adaptive softmax
+            tie_weight (:obj:`boolean`, optional, defaults to :obj:`True`):
+                tie the word embedding and softmax weights
+            dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
+            dropatt (:obj:`float`, optional, defaults to 0):
+                The dropout ratio for the attention probabilities.
+            untie_r (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Untie relative position biases
+            init (:obj:`string`, optional, defaults to `normal`):
+                Parameter initializer to use
+            init_range (:obj:`float`, optional, defaults to 0.01):
+                Parameters initialized by U(-init_range, init_range).
+            proj_init_std (:obj:`float`, optional, defaults to 0.01):
+                Parameters initialized by N(0, init_std)
+            init_std (:obj:`float`, optional, defaults to 0.02):
+                Parameters initialized by N(0, init_std)
+            layer_norm_epsilon (:obj:`float`, optional, defaults to 1e-5):
+                The epsilon to use in the layer normalization layers
+
+        Example::
+
+            from transformers import TransfoXLConfig, TransfoXLModel
+
+            # Initializing a Transformer XL configuration
+            configuration = TransfoXLConfig()
+
+            # Initializing a model from the configuration
+            model = TransfoXLModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
     """
 
     pretrained_config_archive_map = TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "transfo-xl"
 
     def __init__(
         self,
@@ -98,9 +151,8 @@ class TransfoXLConfig(PretrainedConfig):
         layer_norm_epsilon=1e-5,
         **kwargs
     ):
-        """Constructs TransfoXLConfig.
-        """
-        super(TransfoXLConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
+
         self.vocab_size = vocab_size
         self.cutoffs = []
         self.cutoffs.extend(cutoffs)

src/transformers/configuration_utils.py

@@ -20,6 +20,7 @@ import copy
 import json
 import logging
 import os
+from typing import Dict, Optional, Tuple
 
 from .file_utils import CONFIG_NAME, cached_path, hf_bucket_url, is_remote_url
 
@@ -36,7 +37,8 @@ class PretrainedConfig(object):
             It only affects the model's configuration.
 
         Class attributes (overridden by derived classes):
-            - ``pretrained_config_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained model configurations as values.
+            - ``pretrained_config_archive_map``: a python ``dict`` with `shortcut names` (string) as keys and `url` (string) of associated pretrained model configurations as values.
+            - ``model_type``: a string that identifies the model type, that we serialize into the JSON file, and that we use to recreate the correct object in :class:`~transformers.AutoConfig`.
 
         Parameters:
             ``finetuning_task``: string, default `None`. Name of the task used to fine-tune the model. This can be used when converting from an original (TensorFlow or PyTorch) checkpoint.
@@ -45,7 +47,8 @@ class PretrainedConfig(object):
             ``output_hidden_states``: string, default `False`. Should the model returns all hidden-states.
             ``torchscript``: string, default `False`. Is the model used with Torchscript.
     """
-    pretrained_config_archive_map = {}
+    pretrained_config_archive_map = {}  # type: Dict[str, str]
+    model_type = ""  # type: str
 
     def __init__(self, **kwargs):
         # Attributes with defaults
@@ -154,14 +157,32 @@ class PretrainedConfig(object):
             assert unused_kwargs == {'foo': False}
 
         """
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+        return cls.from_dict(config_dict, **kwargs)
+
+    @classmethod
+    def get_config_dict(
+        cls, pretrained_model_name_or_path: str, pretrained_config_archive_map: Optional[Dict] = None, **kwargs
+    ) -> Tuple[Dict, Dict]:
+        """
+        From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used
+        for instantiating a Config using `from_dict`.
+
+        Parameters:
+            pretrained_config_archive_map: (`optional`) Dict:
+                A map of `shortcut names` to `url`.
+                By default, will use the current class attribute.
+        """
         cache_dir = kwargs.pop("cache_dir", None)
         force_download = kwargs.pop("force_download", False)
         resume_download = kwargs.pop("resume_download", False)
         proxies = kwargs.pop("proxies", None)
-        return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)
 
-        if pretrained_model_name_or_path in cls.pretrained_config_archive_map:
-            config_file = cls.pretrained_config_archive_map[pretrained_model_name_or_path]
+        if pretrained_config_archive_map is None:
+            pretrained_config_archive_map = cls.pretrained_config_archive_map
+
+        if pretrained_model_name_or_path in pretrained_config_archive_map:
+            config_file = pretrained_config_archive_map[pretrained_model_name_or_path]
         elif os.path.isdir(pretrained_model_name_or_path):
             config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME)
         elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
@@ -178,23 +199,22 @@ class PretrainedConfig(object):
                 proxies=proxies,
                 resume_download=resume_download,
             )
-            # Load config
-            config = cls.from_json_file(resolved_config_file)
+            # Load config dict
+            if resolved_config_file is None:
+                raise EnvironmentError
+            config_dict = cls._dict_from_json_file(resolved_config_file)
 
         except EnvironmentError:
-            if pretrained_model_name_or_path in cls.pretrained_config_archive_map:
+            if pretrained_model_name_or_path in pretrained_config_archive_map:
                 msg = "Couldn't reach server at '{}' to download pretrained model configuration file.".format(
                     config_file
                 )
             else:
                 msg = (
-                    "Model name '{}' was not found in model name list ({}). "
-                    "We assumed '{}' was a path or url to a configuration file named {} or "
+                    "Model name '{}' was not found in model name list. "
+                    "We assumed '{}' was a path, a model identifier, or url to a configuration file named {} or "
                     "a directory containing such a file but couldn't find any such file at this path or url.".format(
-                        pretrained_model_name_or_path,
-                        ", ".join(cls.pretrained_config_archive_map.keys()),
-                        config_file,
-                        CONFIG_NAME,
+                        pretrained_model_name_or_path, config_file, CONFIG_NAME,
                     )
                 )
             raise EnvironmentError(msg)
@@ -212,6 +232,15 @@ class PretrainedConfig(object):
         else:
             logger.info("loading configuration file {} from cache at {}".format(config_file, resolved_config_file))
 
+        return config_dict, kwargs
+
+    @classmethod
+    def from_dict(cls, config_dict: Dict, **kwargs):
+        """Constructs a `Config` from a Python dictionary of parameters."""
+        return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)
+
+        config = cls(**config_dict)
+
         if hasattr(config, "pruned_heads"):
             config.pruned_heads = dict((int(key), value) for key, value in config.pruned_heads.items())
 
@@ -231,27 +260,28 @@ class PretrainedConfig(object):
             return config
 
     @classmethod
-    def from_dict(cls, json_object):
-        """Constructs a `Config` from a Python dictionary of parameters."""
-        return cls(**json_object)
+    def from_json_file(cls, json_file: str):
+        """Constructs a `Config` from the path to a json file of parameters."""
+        config_dict = cls._dict_from_json_file(json_file)
+        return cls(**config_dict)
 
     @classmethod
-    def from_json_file(cls, json_file):
-        """Constructs a `Config` from a json file of parameters."""
+    def _dict_from_json_file(cls, json_file: str):
         with open(json_file, "r", encoding="utf-8") as reader:
             text = reader.read()
-        dict_obj = json.loads(text)
-        return cls(**dict_obj)
+        return json.loads(text)
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__
 
     def __repr__(self):
-        return str(self.to_json_string())
+        return "{} {}".format(self.__class__.__name__, self.to_json_string())
 
     def to_dict(self):
         """Serializes this instance to a Python dictionary."""
         output = copy.deepcopy(self.__dict__)
+        if hasattr(self.__class__, "model_type"):
+            output["model_type"] = self.__class__.model_type
         return output
 
     def to_json_string(self):

src/transformers/configuration_xlm.py

@@ -37,47 +37,128 @@ XLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 
 class XLMConfig(PretrainedConfig):
-    """Configuration class to store the configuration of a `XLMModel`.
+    """
+        This is the configuration class to store the configuration of a :class:`~transformers.XLMModel`.
+        It is used to instantiate an XLM model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the `xlm-mlm-en-2048 <https://huggingface.co/xlm-mlm-en-2048>`__ architecture.
 
-    Args:
-        vocab_size: Vocabulary size of `inputs_ids` in `XLMModel`.
-        d_model: Size of the encoder layers and the pooler layer.
-        n_layer: Number of hidden layers in the Transformer encoder.
-        n_head: Number of attention heads for each attention layer in
-            the Transformer encoder.
-        d_inner: The size of the "intermediate" (i.e., feed-forward)
-            layer in the Transformer encoder.
-        ff_activation: The non-linear activation function (function or string) in the
-            encoder and pooler. If string, "gelu", "relu" and "swish" are supported.
-        untie_r: untie relative position biases
-        attn_type: 'bi' for XLM, 'uni' for Transformer-XL
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
 
-        dropout: The dropout probabilitiy for all fully connected
-            layers in the embeddings, encoder, and pooler.
-        max_position_embeddings: The maximum sequence length that this model might
-            ever be used with. Typically set this to something large just in case
-            (e.g., 512 or 1024 or 2048).
-        initializer_range: The sttdev of the truncated_normal_initializer for
-            initializing all weight matrices.
-        layer_norm_eps: The epsilon used by LayerNorm.
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 30145):
+                Vocabulary size of the XLM model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.XLMModel`.
+            emb_dim (:obj:`int`, optional, defaults to 2048):
+                Dimensionality of the encoder layers and the pooler layer.
+            n_layer (:obj:`int`, optional, defaults to 12):
+                Number of hidden layers in the Transformer encoder.
+            n_head (:obj:`int`, optional, defaults to 16):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout probability for all fully connected
+                layers in the embeddings, encoder, and pooler.
+            attention_dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout probability for the attention mechanism
+            gelu_activation (:obj:`boolean`, optional, defaults to :obj:`True`):
+                The non-linear activation function (function or string) in the
+                encoder and pooler. If set to `True`, "gelu" will be used instead of "relu".
+            sinusoidal_embeddings (:obj:`boolean`, optional, defaults to :obj:`False`):
+                Whether to use sinusoidal positional embeddings instead of absolute positional embeddings.
+            causal (:obj:`boolean`, optional, defaults to :obj:`False`):
+                Set this to `True` for the model to behave in a causal manner.
+                Causal models use a triangular attention mask in order to only attend to the left-side context instead
+                if a bidirectional context.
+            asm (:obj:`boolean`, optional, defaults to :obj:`False`):
+                Whether to use an adaptive log softmax projection layer instead of a linear layer for the prediction
+                layer.
+            n_langs (:obj:`int`, optional, defaults to 1):
+                The number of languages the model handles. Set to 1 for monolingual models.
+            use_lang_emb (:obj:`boolean`, optional, defaults to :obj:`True`)
+                Whether to use language embeddings. Some models use additional language embeddings, see
+                `the multilingual models page <http://huggingface.co/transformers/multilingual.html#xlm-language-embeddings>`__
+                for information on how to use them.
+            max_position_embeddings (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might
+                ever be used with. Typically set this to something large just in case
+                (e.g., 512 or 1024 or 2048).
+            embed_init_std (:obj:`float`, optional, defaults to 2048^-0.5):
+                The standard deviation of the truncated_normal_initializer for
+                initializing the embedding matrices.
+            init_std (:obj:`int`, optional, defaults to 50257):
+                The standard deviation of the truncated_normal_initializer for
+                initializing all weight matrices except the embedding matrices.
+            layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
+                The epsilon used by the layer normalization layers.
+            bos_index (:obj:`int`, optional, defaults to 0):
+                The index of the beginning of sentence token in the vocabulary.
+            eos_index (:obj:`int`, optional, defaults to 1):
+                The index of the end of sentence token in the vocabulary.
+            pad_index (:obj:`int`, optional, defaults to 2):
+                The index of the padding token in the vocabulary.
+            unk_index (:obj:`int`, optional, defaults to 3):
+                The index of the unknown token in the vocabulary.
+            mask_index (:obj:`int`, optional, defaults to 5):
+                The index of the masking token in the vocabulary.
+            is_encoder(:obj:`boolean`, optional, defaults to :obj:`True`):
+                Whether the initialized model should be a transformer encoder or decoder as seen in Vaswani et al.
+            summary_type (:obj:`string`, optional, defaults to "first"):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.XLMForSequenceClassification`.
+                Is one of the following options:
+                    - 'last' => take the last token hidden state (like XLNet)
+                    - 'first' => take the first token hidden state (like Bert)
+                    - 'mean' => take the mean of all tokens hidden states
+                    - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2)
+                    - 'attn' => Not implemented now, use multi-head attention
+            summary_use_proj (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.XLMForSequenceClassification`.
+                Add a projection after the vector extraction
+            summary_activation (:obj:`string` or :obj:`None`, optional, defaults to :obj:`None`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.XLMForSequenceClassification`.
+                'tanh' => add a tanh activation to the output, Other => no activation.
+            summary_proj_to_labels (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.XLMForSequenceClassification`.
+                If True, the projection outputs to config.num_labels classes (otherwise to hidden_size). Default: False.
+            summary_first_dropout (:obj:`float`, optional, defaults to 0.1):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.XLMForSequenceClassification`.
+                Add a dropout before the projection and activation
+            start_n_top (:obj:`int`, optional, defaults to 5):
+                Used in the SQuAD evaluation script for XLM and XLNet.
+            end_n_top (:obj:`int`, optional, defaults to 5):
+                Used in the SQuAD evaluation script for XLM and XLNet.
+            mask_token_id (:obj:`int`, optional, defaults to 0):
+                Model agnostic parameter to identify masked tokens when generating text in an MLM context.
+            lang_id (:obj:`int`, optional, defaults to 1):
+                The ID of the language used by the model. This parameter is used when generating
+                text in a given language.
 
-        dropout: float, dropout rate.
-        init: str, the initialization scheme, either "normal" or "uniform".
-        init_range: float, initialize the parameters with a uniform distribution
-            in [-init_range, init_range]. Only effective when init="uniform".
-        init_std: float, initialize the parameters with a normal distribution
-            with mean 0 and stddev init_std. Only effective when init="normal".
-        mem_len: int, the number of tokens to cache.
-        reuse_len: int, the number of tokens in the currect batch to be cached
-            and reused in the future.
-        bi_data: bool, whether to use bidirectional input pipeline.
-            Usually set to True during pretraining and False during finetuning.
-        clamp_len: int, clamp all relative distances larger than clamp_len.
-            -1 means no clamping.
-        same_length: bool, whether to use the same attention length for each token.
+        Example::
+
+            from transformers import XLMConfig, XLMModel
+
+            # Initializing a XLM configuration
+            configuration = XLMConfig()
+
+            # Initializing a model from the configuration
+            model = XLMModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
     """
 
     pretrained_config_archive_map = XLM_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "xlm"
 
     def __init__(
         self,
@@ -116,7 +197,7 @@ class XLMConfig(PretrainedConfig):
     ):
         """Constructs XLMConfig.
         """
-        super(XLMConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = vocab_size
         self.emb_dim = emb_dim
         self.n_layers = n_layers

src/transformers/configuration_xlm_roberta.py

@@ -35,3 +35,4 @@ XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 class XLMRobertaConfig(RobertaConfig):
     pretrained_config_archive_map = XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "xlm-roberta"

src/transformers/configuration_xlnet.py

@@ -30,45 +30,106 @@ XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 
 class XLNetConfig(PretrainedConfig):
-    """Configuration class to store the configuration of a ``XLNetModel``.
+    """
+        This is the configuration class to store the configuration of a :class:`~transformers.XLNetModel`.
+        It is used to instantiate an XLNet model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the `xlnet-large-cased <https://huggingface.co/xlnet-large-cased>`__ architecture.
 
-    Args:
-        vocab_size: Vocabulary size of ``inputs_ids`` in ``XLNetModel``.
-        d_model: Size of the encoder layers and the pooler layer.
-        n_layer: Number of hidden layers in the Transformer encoder.
-        n_head: Number of attention heads for each attention layer in
-            the Transformer encoder.
-        d_inner: The size of the "intermediate" (i.e., feed-forward)
-            layer in the Transformer encoder.
-        ff_activation: The non-linear activation function (function or string) in the
-            encoder and pooler. If string, "gelu", "relu" and "swish" are supported.
-        untie_r: untie relative position biases
-        attn_type: 'bi' for XLNet, 'uni' for Transformer-XL
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
 
-        dropout: The dropout probabilitiy for all fully connected
-            layers in the embeddings, encoder, and pooler.
-        initializer_range: The sttdev of the truncated_normal_initializer for
-            initializing all weight matrices.
-        layer_norm_eps: The epsilon used by LayerNorm.
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 32000):
+                Vocabulary size of the XLNet model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.XLNetModel`.
+            d_model (:obj:`int`, optional, defaults to 1024):
+                Dimensionality of the encoder layers and the pooler layer.
+            n_layer (:obj:`int`, optional, defaults to 24):
+                Number of hidden layers in the Transformer encoder.
+            n_head (:obj:`int`, optional, defaults to 16):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            d_inner (:obj:`int`, optional, defaults to 4096):
+                Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+            ff_activation (:obj:`string`, optional, defaults to "gelu"):
+                The non-linear activation function (function or string) in the
+                encoder and pooler. If string, "gelu", "relu" and "swish" are supported.
+            untie_r (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Untie relative position biases
+            attn_type (:obj:`string`, optional, defaults to "bi"):
+                The attention type used by the model. Set 'bi' for XLNet, 'uni' for Transformer-XL.
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
+                The epsilon used by the layer normalization layers.
+            dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+            mem_len (:obj:`int` or :obj:`None`, optional, defaults to :obj:`None`):
+                The number of tokens to cache. The key/value pairs that have already been pre-computed
+                in a previous forward pass won't be re-computed. See the
+                `quickstart <https://huggingface.co/transformers/quickstart.html#using-the-past>`__
+                for more information.
+            reuse_len (:obj:`int` or :obj:`None`, optional, defaults to :obj:`None`):
+                The number of tokens in the current batch to be cached and reused in the future.
+            bi_data (:obj:`boolean`, optional, defaults to :obj:`False`):
+                Whether to use bidirectional input pipeline. Usually set to `True` during
+                pretraining and `False` during finetuning.
+            clamp_len (:obj:`int`, optional, defaults to -1):
+                Clamp all relative distances larger than clamp_len.
+                Setting this attribute to -1 means no clamping.
+            same_length (:obj:`boolean`, optional, defaults to :obj:`False`):
+                Whether to use the same attention length for each token.
+            summary_type (:obj:`string`, optional, defaults to "last"):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:transformers.XLNetForSequenceClassification` and :class:`~transformers.XLNetForMultipleChoice`.
+                Is one of the following options:
+                    - 'last' => take the last token hidden state (like XLNet)
+                    - 'first' => take the first token hidden state (like Bert)
+                    - 'mean' => take the mean of all tokens hidden states
+                    - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2)
+                    - 'attn' => Not implemented now, use multi-head attention
+            summary_use_proj (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.XLNetForSequenceClassification` and :class:`~transformers.XLNetForMultipleChoice`.
+                Add a projection after the vector extraction
+            summary_activation (:obj:`string` or :obj:`None`, optional, defaults to :obj:`None`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.XLNetForSequenceClassification` and :class:`~transformers.XLNetForMultipleChoice`.
+                'tanh' => add a tanh activation to the output, Other => no activation.
+            summary_proj_to_labels (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.XLNetForSequenceClassification` and :class:`~transformers.XLNetForMultipleChoice`.
+                If True, the projection outputs to config.num_labels classes (otherwise to hidden_size). Default: False.
+            summary_last_dropout (:obj:`float`, optional, defaults to 0.1):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.XLNetForSequenceClassification` and :class:`~transformers.XLNetForMultipleChoice`.
+                Add a dropout after the projection and activation
+            start_n_top (:obj:`int`, optional, defaults to 5):
+                Used in the SQuAD evaluation script for XLM and XLNet.
+            end_n_top (:obj:`int`, optional, defaults to 5):
+                Used in the SQuAD evaluation script for XLM and XLNet.
 
-        dropout: float, dropout rate.
-        init: str, the initialization scheme, either "normal" or "uniform".
-        init_range: float, initialize the parameters with a uniform distribution
-            in [-init_range, init_range]. Only effective when init="uniform".
-        init_std: float, initialize the parameters with a normal distribution
-            with mean 0 and stddev init_std. Only effective when init="normal".
-        mem_len: int, the number of tokens to cache.
-        reuse_len: int, the number of tokens in the currect batch to be cached
-            and reused in the future.
-        bi_data: bool, whether to use bidirectional input pipeline.
-            Usually set to True during pretraining and False during finetuning.
-        clamp_len: int, clamp all relative distances larger than clamp_len.
-            -1 means no clamping.
-        same_length: bool, whether to use the same attention length for each token.
-        finetuning_task: name of the glue task on which the model was fine-tuned if any
+        Example::
+
+            from transformers import XLNetConfig, XLNetModel
+
+            # Initializing a XLNet configuration
+            configuration = XLNetConfig()
+
+            # Initializing a model from the configuration
+            model = XLNetModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
     """
 
     pretrained_config_archive_map = XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "xlnet"
 
     def __init__(
         self,
@@ -98,7 +159,7 @@ class XLNetConfig(PretrainedConfig):
     ):
         """Constructs XLNetConfig.
         """
-        super(XLNetConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = vocab_size
         self.d_model = d_model
         self.n_layer = n_layer

src/transformers/data/metrics/__init__.py

@@ -72,6 +72,8 @@ if _has_sklearn:
             return {"acc": simple_accuracy(preds, labels)}
         elif task_name == "wnli":
             return {"acc": simple_accuracy(preds, labels)}
+        elif task_name == "hans":
+            return {"acc": simple_accuracy(preds, labels)}
         else:
             raise KeyError(task_name)
 

src/transformers/file_utils.py

@@ -4,7 +4,6 @@ This file is adapted from the AllenNLP library at https://github.com/allenai/all
 Copyright by the AllenNLP authors.
 """
 
-
 import fnmatch
 import json
 import logging
@@ -14,6 +13,7 @@ import tempfile
 from contextlib import contextmanager
 from functools import partial, wraps
 from hashlib import sha256
+from typing import Optional
 from urllib.parse import urlparse
 
 import boto3
@@ -29,28 +29,31 @@ from . import __version__
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 try:
-    os.environ.setdefault("USE_TORCH", "YES")
-    if os.environ["USE_TORCH"].upper() in ("1", "ON", "YES"):
+    USE_TF = os.environ.get("USE_TF", "AUTO").upper()
+    USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper()
+    if USE_TORCH in ("1", "ON", "YES", "AUTO") and USE_TF not in ("1", "ON", "YES"):
         import torch
 
         _torch_available = True  # pylint: disable=invalid-name
         logger.info("PyTorch version {} available.".format(torch.__version__))
     else:
-        logger.info("USE_TORCH override through env variable, disabling PyTorch")
+        logger.info("Disabling PyTorch because USE_TF is set")
         _torch_available = False
 except ImportError:
     _torch_available = False  # pylint: disable=invalid-name
 
 try:
-    os.environ.setdefault("USE_TF", "YES")
-    if os.environ["USE_TF"].upper() in ("1", "ON", "YES"):
+    USE_TF = os.environ.get("USE_TF", "AUTO").upper()
+    USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper()
+
+    if USE_TF in ("1", "ON", "YES", "AUTO") and USE_TORCH not in ("1", "ON", "YES"):
         import tensorflow as tf
 
         assert hasattr(tf, "__version__") and int(tf.__version__[0]) >= 2
         _tf_available = True  # pylint: disable=invalid-name
         logger.info("TensorFlow version {} available.".format(tf.__version__))
     else:
-        logger.info("USE_TF override through env variable, disabling Tensorflow")
+        logger.info("Disabling Tensorflow because USE_TORCH is set")
         _tf_available = False
 except (ImportError, AssertionError):
     _tf_available = False  # pylint: disable=invalid-name
@@ -65,7 +68,6 @@ except ImportError:
     )
 default_cache_path = os.path.join(torch_cache_home, "transformers")
 
-
 try:
     from pathlib import Path
 
@@ -122,7 +124,7 @@ def is_remote_url(url_or_filename):
     return parsed.scheme in ("http", "https", "s3")
 
 
-def hf_bucket_url(identifier, postfix=None, cdn=False):
+def hf_bucket_url(identifier, postfix=None, cdn=False) -> str:
     endpoint = CLOUDFRONT_DISTRIB_PREFIX if cdn else S3_BUCKET_PREFIX
     if postfix is None:
         return "/".join((endpoint, identifier))
@@ -182,7 +184,7 @@ def filename_to_url(filename, cache_dir=None):
 
 def cached_path(
     url_or_filename, cache_dir=None, force_download=False, proxies=None, resume_download=False, user_agent=None
-):
+) -> Optional[str]:
     """
     Given something that might be a URL (or might be a local path),
     determine which. If it's a URL, download the file and cache it, and
@@ -193,6 +195,10 @@ def cached_path(
         force_download: if True, re-dowload the file even if it's already cached in the cache dir.
         resume_download: if True, resume the download if incompletly recieved file is found.
         user_agent: Optional string or dict that will be appended to the user-agent on remote requests.
+
+    Return:
+        None in case of non-recoverable file (non-existent or inaccessible url + no cache on disk).
+        Local path (string) otherwise
     """
     if cache_dir is None:
         cache_dir = TRANSFORMERS_CACHE
@@ -306,18 +312,21 @@ def http_get(url, temp_file, proxies=None, resume_size=0, user_agent=None):
 
 def get_from_cache(
     url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10, resume_download=False, user_agent=None
-):
+) -> Optional[str]:
     """
-    Given a URL, look for the corresponding dataset in the local cache.
+    Given a URL, look for the corresponding file in the local cache.
     If it's not there, download it. Then return the path to the cached file.
+
+    Return:
+        None in case of non-recoverable file (non-existent or inaccessible url + no cache on disk).
+        Local path (string) otherwise
     """
     if cache_dir is None:
         cache_dir = TRANSFORMERS_CACHE
     if isinstance(cache_dir, Path):
         cache_dir = str(cache_dir)
 
-    if not os.path.exists(cache_dir):
-        os.makedirs(cache_dir)
+    os.makedirs(cache_dir, exist_ok=True)
 
     # Get eTag to add to filename, if it exists.
     if url.startswith("s3://"):
@@ -337,16 +346,25 @@ def get_from_cache(
     # get cache path to put the file
     cache_path = os.path.join(cache_dir, filename)
 
-    # If we don't have a connection (etag is None) and can't identify the file
+    # etag is None = we don't have a connection, or url doesn't exist, or is otherwise inaccessible.
     # try to get the last downloaded one
-    if not os.path.exists(cache_path) and etag is None:
-        matching_files = [
-            file
-            for file in fnmatch.filter(os.listdir(cache_dir), filename + ".*")
-            if not file.endswith(".json") and not file.endswith(".lock")
-        ]
-        if matching_files:
-            cache_path = os.path.join(cache_dir, matching_files[-1])
+    if etag is None:
+        if os.path.exists(cache_path):
+            return cache_path
+        else:
+            matching_files = [
+                file
+                for file in fnmatch.filter(os.listdir(cache_dir), filename + ".*")
+                if not file.endswith(".json") and not file.endswith(".lock")
+            ]
+            if len(matching_files) > 0:
+                return os.path.join(cache_dir, matching_files[-1])
+            else:
+                return None
+
+    # From now on, etag is not None.
+    if os.path.exists(cache_path) and not force_download:
+        return cache_path
 
     # Prevent parallel downloads of the same file with a lock.
     lock_path = cache_path + ".lock"
@@ -369,32 +387,26 @@ def get_from_cache(
             temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False)
             resume_size = 0
 
-        if etag is not None and (not os.path.exists(cache_path) or force_download):
-            # Download to temporary file, then copy to cache dir once finished.
-            # Otherwise you get corrupt cache entries if the download gets interrupted.
-            with temp_file_manager() as temp_file:
-                logger.info(
-                    "%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name
-                )
+        # Download to temporary file, then copy to cache dir once finished.
+        # Otherwise you get corrupt cache entries if the download gets interrupted.
+        with temp_file_manager() as temp_file:
+            logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name)
 
-                # GET file object
-                if url.startswith("s3://"):
-                    if resume_download:
-                        logger.warn('Warning: resumable downloads are not implemented for "s3://" urls')
-                    s3_get(url, temp_file, proxies=proxies)
-                else:
-                    http_get(url, temp_file, proxies=proxies, resume_size=resume_size, user_agent=user_agent)
+            # GET file object
+            if url.startswith("s3://"):
+                if resume_download:
+                    logger.warn('Warning: resumable downloads are not implemented for "s3://" urls')
+                s3_get(url, temp_file, proxies=proxies)
+            else:
+                http_get(url, temp_file, proxies=proxies, resume_size=resume_size, user_agent=user_agent)
 
-                # we are copying the file before closing it, so flush to avoid truncation
-                temp_file.flush()
+        logger.info("storing %s in cache at %s", url, cache_path)
+        os.rename(temp_file.name, cache_path)
 
-                logger.info("storing %s in cache at %s", url, cache_path)
-                os.rename(temp_file.name, cache_path)
-
-                logger.info("creating metadata file for %s", cache_path)
-                meta = {"url": url, "etag": etag}
-                meta_path = cache_path + ".json"
-                with open(meta_path, "w") as meta_file:
-                    json.dump(meta, meta_file)
+        logger.info("creating metadata file for %s", cache_path)
+        meta = {"url": url, "etag": etag}
+        meta_path = cache_path + ".json"
+        with open(meta_path, "w") as meta_file:
+            json.dump(meta, meta_file)
 
     return cache_path

src/transformers/modeling_albert.py

@@ -167,7 +167,7 @@ class AlbertEmbeddings(BertEmbeddings):
     """
 
     def __init__(self, config):
-        super(AlbertEmbeddings, self).__init__(config)
+        super().__init__(config)
 
         self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0)
         self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size)
@@ -177,7 +177,7 @@ class AlbertEmbeddings(BertEmbeddings):
 
 class AlbertAttention(BertSelfAttention):
     def __init__(self, config):
-        super(AlbertAttention, self).__init__(config)
+        super().__init__(config)
 
         self.output_attentions = config.output_attentions
         self.num_attention_heads = config.num_attention_heads
@@ -258,7 +258,7 @@ class AlbertAttention(BertSelfAttention):
 
 class AlbertLayer(nn.Module):
     def __init__(self, config):
-        super(AlbertLayer, self).__init__()
+        super().__init__()
 
         self.config = config
         self.full_layer_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
@@ -279,7 +279,7 @@ class AlbertLayer(nn.Module):
 
 class AlbertLayerGroup(nn.Module):
     def __init__(self, config):
-        super(AlbertLayerGroup, self).__init__()
+        super().__init__()
 
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
@@ -309,7 +309,7 @@ class AlbertLayerGroup(nn.Module):
 
 class AlbertTransformer(nn.Module):
     def __init__(self, config):
-        super(AlbertTransformer, self).__init__()
+        super().__init__()
 
         self.config = config
         self.output_attentions = config.output_attentions
@@ -471,7 +471,7 @@ class AlbertModel(AlbertPreTrainedModel):
     base_model_prefix = "albert"
 
     def __init__(self, config):
-        super(AlbertModel, self).__init__(config)
+        super().__init__(config)
 
         self.config = config
         self.embeddings = AlbertEmbeddings(config)
@@ -571,7 +571,7 @@ class AlbertModel(AlbertPreTrainedModel):
 
 class AlbertMLMHead(nn.Module):
     def __init__(self, config):
-        super(AlbertMLMHead, self).__init__()
+        super().__init__()
 
         self.LayerNorm = nn.LayerNorm(config.embedding_size)
         self.bias = nn.Parameter(torch.zeros(config.vocab_size))
@@ -579,6 +579,9 @@ class AlbertMLMHead(nn.Module):
         self.decoder = nn.Linear(config.embedding_size, config.vocab_size)
         self.activation = ACT2FN[config.hidden_act]
 
+        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
+        self.decoder.bias = self.bias
+
     def forward(self, hidden_states):
         hidden_states = self.dense(hidden_states)
         hidden_states = self.activation(hidden_states)
@@ -616,7 +619,7 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(AlbertForMaskedLM, self).__init__(config)
+        super().__init__(config)
 
         self.albert = AlbertModel(config)
         self.predictions = AlbertMLMHead(config)
@@ -703,7 +706,7 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(AlbertForSequenceClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.albert = AlbertModel(config)
@@ -801,7 +804,7 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(AlbertForQuestionAnswering, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.albert = AlbertModel(config)

src/transformers/modeling_auto.py

@@ -16,9 +16,11 @@
 
 
 import logging
+from collections import OrderedDict
 
 from .configuration_auto import (
     AlbertConfig,
+    AutoConfig,
     BertConfig,
     CamembertConfig,
     CTRLConfig,
@@ -26,11 +28,13 @@ from .configuration_auto import (
     GPT2Config,
     OpenAIGPTConfig,
     RobertaConfig,
+    T5Config,
     TransfoXLConfig,
     XLMConfig,
     XLMRobertaConfig,
     XLNetConfig,
 )
+from .configuration_utils import PretrainedConfig
 from .modeling_albert import (
     ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP,
     AlbertForMaskedLM,
@@ -120,6 +124,76 @@ ALL_PRETRAINED_MODEL_ARCHIVE_MAP = dict(
     for key, value, in pretrained_map.items()
 )
 
+MODEL_MAPPING = OrderedDict(
+    [
+        (T5Config, T5Model),
+        (DistilBertConfig, DistilBertModel),
+        (AlbertConfig, AlbertModel),
+        (CamembertConfig, CamembertModel),
+        (RobertaConfig, RobertaModel),
+        (XLMRobertaConfig, XLMRobertaModel),
+        (BertConfig, BertModel),
+        (OpenAIGPTConfig, OpenAIGPTModel),
+        (GPT2Config, GPT2Model),
+        (TransfoXLConfig, TransfoXLModel),
+        (XLNetConfig, XLNetModel),
+        (XLMConfig, XLMModel),
+        (CTRLConfig, CTRLModel),
+    ]
+)
+
+MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
+    [
+        (T5Config, T5WithLMHeadModel),
+        (DistilBertConfig, DistilBertForMaskedLM),
+        (AlbertConfig, AlbertForMaskedLM),
+        (CamembertConfig, CamembertForMaskedLM),
+        (RobertaConfig, RobertaForMaskedLM),
+        (XLMRobertaConfig, XLMRobertaForMaskedLM),
+        (BertConfig, BertForMaskedLM),
+        (OpenAIGPTConfig, OpenAIGPTLMHeadModel),
+        (GPT2Config, GPT2LMHeadModel),
+        (TransfoXLConfig, TransfoXLLMHeadModel),
+        (XLNetConfig, XLNetLMHeadModel),
+        (XLMConfig, XLMWithLMHeadModel),
+        (CTRLConfig, CTRLLMHeadModel),
+    ]
+)
+
+MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
+    [
+        (DistilBertConfig, DistilBertForSequenceClassification),
+        (AlbertConfig, AlbertForSequenceClassification),
+        (CamembertConfig, CamembertForSequenceClassification),
+        (RobertaConfig, RobertaForSequenceClassification),
+        (XLMRobertaConfig, XLMRobertaForSequenceClassification),
+        (BertConfig, BertForSequenceClassification),
+        (XLNetConfig, XLNetForSequenceClassification),
+        (XLMConfig, XLMForSequenceClassification),
+    ]
+)
+
+MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
+    [
+        (DistilBertConfig, DistilBertForQuestionAnswering),
+        (AlbertConfig, AlbertForQuestionAnswering),
+        (BertConfig, BertForQuestionAnswering),
+        (XLNetConfig, XLNetForQuestionAnswering),
+        (XLMConfig, XLMForQuestionAnswering),
+    ]
+)
+
+MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = OrderedDict(
+    [
+        (DistilBertConfig, DistilBertForTokenClassification),
+        (CamembertConfig, CamembertForTokenClassification),
+        (RobertaConfig, RobertaForTokenClassification),
+        (XLMRobertaConfig, XLMRobertaForTokenClassification),
+        (BertConfig, BertForTokenClassification),
+        (XLNetConfig, XLNetForTokenClassification),
+    ]
+)
+
 
 class AutoModel(object):
     r"""
@@ -129,7 +203,8 @@ class AutoModel(object):
         or the `AutoModel.from_config(config)` class methods.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The base model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -179,31 +254,15 @@ class AutoModel(object):
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = AutoModel.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, DistilBertConfig):
-            return DistilBertModel(config)
-        elif isinstance(config, RobertaConfig):
-            return RobertaModel(config)
-        elif isinstance(config, BertConfig):
-            return BertModel(config)
-        elif isinstance(config, OpenAIGPTConfig):
-            return OpenAIGPTModel(config)
-        elif isinstance(config, GPT2Config):
-            return GPT2Model(config)
-        elif isinstance(config, TransfoXLConfig):
-            return TransfoXLModel(config)
-        elif isinstance(config, XLNetConfig):
-            return XLNetModel(config)
-        elif isinstance(config, XLMConfig):
-            return XLMModel(config)
-        elif isinstance(config, CTRLConfig):
-            return CTRLModel(config)
-        elif isinstance(config, AlbertConfig):
-            return AlbertModel(config)
-        elif isinstance(config, CamembertConfig):
-            return CamembertModel(config)
-        elif isinstance(config, XLMRobertaConfig):
-            return XLMRobertaModel(config)
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in MODEL_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_MAPPING.keys())
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -286,37 +345,17 @@ class AutoModel(object):
             model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
 
         """
-        if "t5" in pretrained_model_name_or_path:
-            return T5Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "distilbert" in pretrained_model_name_or_path:
-            return DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "albert" in pretrained_model_name_or_path:
-            return AlbertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "camembert" in pretrained_model_name_or_path:
-            return CamembertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlm-roberta" in pretrained_model_name_or_path:
-            return XLMRobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "roberta" in pretrained_model_name_or_path:
-            return RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "bert" in pretrained_model_name_or_path:
-            return BertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "openai-gpt" in pretrained_model_name_or_path:
-            return OpenAIGPTModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "gpt2" in pretrained_model_name_or_path:
-            return GPT2Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "transfo-xl" in pretrained_model_name_or_path:
-            return TransfoXLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return XLNetModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlm" in pretrained_model_name_or_path:
-            return XLMModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "ctrl" in pretrained_model_name_or_path:
-            return CTRLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+        for config_class, model_class in MODEL_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-            "'xlm-roberta', 'xlm', 'roberta, 'ctrl', 'distilbert', 'camembert', 'albert'".format(
-                pretrained_model_name_or_path
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_MAPPING.keys())
             )
         )
 
@@ -329,7 +368,8 @@ class AutoModelWithLMHead(object):
         class method.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -379,27 +419,15 @@ class AutoModelWithLMHead(object):
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = AutoModelWithLMHead.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, DistilBertConfig):
-            return DistilBertForMaskedLM(config)
-        elif isinstance(config, RobertaConfig):
-            return RobertaForMaskedLM(config)
-        elif isinstance(config, BertConfig):
-            return BertForMaskedLM(config)
-        elif isinstance(config, OpenAIGPTConfig):
-            return OpenAIGPTLMHeadModel(config)
-        elif isinstance(config, GPT2Config):
-            return GPT2LMHeadModel(config)
-        elif isinstance(config, TransfoXLConfig):
-            return TransfoXLLMHeadModel(config)
-        elif isinstance(config, XLNetConfig):
-            return XLNetLMHeadModel(config)
-        elif isinstance(config, XLMConfig):
-            return XLMWithLMHeadModel(config)
-        elif isinstance(config, CTRLConfig):
-            return CTRLLMHeadModel(config)
-        elif isinstance(config, XLMRobertaConfig):
-            return XLMRobertaForMaskedLM(config)
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in MODEL_WITH_LM_HEAD_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_WITH_LM_HEAD_MAPPING.keys())
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -407,7 +435,8 @@ class AutoModelWithLMHead(object):
         from a pre-trained model configuration.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -484,37 +513,17 @@ class AutoModelWithLMHead(object):
             model = AutoModelWithLMHead.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
 
         """
-        if "t5" in pretrained_model_name_or_path:
-            return T5WithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "distilbert" in pretrained_model_name_or_path:
-            return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "albert" in pretrained_model_name_or_path:
-            return AlbertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "camembert" in pretrained_model_name_or_path:
-            return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlm-roberta" in pretrained_model_name_or_path:
-            return XLMRobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "roberta" in pretrained_model_name_or_path:
-            return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "bert" in pretrained_model_name_or_path:
-            return BertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "openai-gpt" in pretrained_model_name_or_path:
-            return OpenAIGPTLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "gpt2" in pretrained_model_name_or_path:
-            return GPT2LMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "transfo-xl" in pretrained_model_name_or_path:
-            return TransfoXLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return XLNetLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlm" in pretrained_model_name_or_path:
-            return XLMWithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "ctrl" in pretrained_model_name_or_path:
-            return CTRLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+        for config_class, model_class in MODEL_WITH_LM_HEAD_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-            "'xlm-roberta', 'xlm', 'roberta','ctrl', 'distilbert', 'camembert', 'albert'".format(
-                pretrained_model_name_or_path
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__, cls.__name__, ", ".join(c.__name__ for c in MODEL_WITH_LM_HEAD_MAPPING.keys())
             )
         )
 
@@ -527,7 +536,8 @@ class AutoModelForSequenceClassification(object):
         class method.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -568,23 +578,17 @@ class AutoModelForSequenceClassification(object):
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = AutoModelForSequenceClassification.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, AlbertConfig):
-            return AlbertForSequenceClassification(config)
-        elif isinstance(config, CamembertConfig):
-            return CamembertForSequenceClassification(config)
-        elif isinstance(config, DistilBertConfig):
-            return DistilBertForSequenceClassification(config)
-        elif isinstance(config, RobertaConfig):
-            return RobertaForSequenceClassification(config)
-        elif isinstance(config, BertConfig):
-            return BertForSequenceClassification(config)
-        elif isinstance(config, XLNetConfig):
-            return XLNetForSequenceClassification(config)
-        elif isinstance(config, XLMConfig):
-            return XLMForSequenceClassification(config)
-        elif isinstance(config, XLMRobertaConfig):
-            return XLMRobertaForSequenceClassification(config)
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys()),
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -592,7 +596,8 @@ class AutoModelForSequenceClassification(object):
         from a pre-trained model configuration.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -665,37 +670,19 @@ class AutoModelForSequenceClassification(object):
             model = AutoModelForSequenceClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
 
         """
-        if "distilbert" in pretrained_model_name_or_path:
-            return DistilBertForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "albert" in pretrained_model_name_or_path:
-            return AlbertForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "camembert" in pretrained_model_name_or_path:
-            return CamembertForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "xlm-roberta" in pretrained_model_name_or_path:
-            return XLMRobertaForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "roberta" in pretrained_model_name_or_path:
-            return RobertaForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "bert" in pretrained_model_name_or_path:
-            return BertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return XLNetForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlm" in pretrained_model_name_or_path:
-            return XLMForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
 
+        for config_class, model_class in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'bert', 'xlnet', 'xlm-roberta', 'xlm', 'roberta', 'distilbert', 'camembert', 'albert'".format(
-                pretrained_model_name_or_path
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys()),
             )
         )
 
@@ -708,7 +695,8 @@ class AutoModelForQuestionAnswering(object):
         class method.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -745,17 +733,18 @@ class AutoModelForQuestionAnswering(object):
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = AutoModelForSequenceClassification.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, AlbertConfig):
-            return AlbertForQuestionAnswering(config)
-        elif isinstance(config, DistilBertConfig):
-            return DistilBertForQuestionAnswering(config)
-        elif isinstance(config, BertConfig):
-            return BertForQuestionAnswering(config)
-        elif isinstance(config, XLNetConfig):
-            return XLNetForQuestionAnswering(config)
-        elif isinstance(config, XLMConfig):
-            return XLMForQuestionAnswering(config)
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in MODEL_FOR_QUESTION_ANSWERING_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys()),
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -763,7 +752,8 @@ class AutoModelForQuestionAnswering(object):
         from a pre-trained model configuration.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -830,20 +820,21 @@ class AutoModelForQuestionAnswering(object):
             model = AutoModelForQuestionAnswering.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
 
         """
-        if "distilbert" in pretrained_model_name_or_path:
-            return DistilBertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "albert" in pretrained_model_name_or_path:
-            return AlbertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "bert" in pretrained_model_name_or_path:
-            return BertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return XLNetForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlm" in pretrained_model_name_or_path:
-            return XLMForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+        for config_class, model_class in MODEL_FOR_QUESTION_ANSWERING_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
 
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'bert', 'xlnet', 'xlm', 'distilbert', 'albert'".format(pretrained_model_name_or_path)
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys()),
+            )
         )
 
 
@@ -873,19 +864,18 @@ class AutoModelForTokenClassification:
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = AutoModelForTokenClassification.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, CamembertConfig):
-            return CamembertForTokenClassification(config)
-        elif isinstance(config, DistilBertConfig):
-            return DistilBertForTokenClassification(config)
-        elif isinstance(config, BertConfig):
-            return BertForTokenClassification(config)
-        elif isinstance(config, XLNetConfig):
-            return XLNetForTokenClassification(config)
-        elif isinstance(config, RobertaConfig):
-            return RobertaForTokenClassification(config)
-        elif isinstance(config, XLMRobertaConfig):
-            return XLMRobertaForTokenClassification(config)
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys()),
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -893,7 +883,8 @@ class AutoModelForTokenClassification:
         from a pre-trained model configuration.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -959,28 +950,19 @@ class AutoModelForTokenClassification:
             model = AutoModelForTokenClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
 
         """
-        if "camembert" in pretrained_model_name_or_path:
-            return CamembertForTokenClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "distilbert" in pretrained_model_name_or_path:
-            return DistilBertForTokenClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "xlm-roberta" in pretrained_model_name_or_path:
-            return XLMRobertaForTokenClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "roberta" in pretrained_model_name_or_path:
-            return RobertaForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "bert" in pretrained_model_name_or_path:
-            return BertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return XLNetForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+        for config_class, model_class in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
 
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'bert', 'xlnet', 'camembert', 'distilbert', 'xlm-roberta', 'roberta'".format(
-                pretrained_model_name_or_path
+            "Unrecognized configuration class {} for this kind of AutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys()),
             )
         )

src/transformers/modeling_bert.py

@@ -160,7 +160,7 @@ class BertEmbeddings(nn.Module):
     """
 
     def __init__(self, config):
-        super(BertEmbeddings, self).__init__()
+        super().__init__()
         self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=0)
         self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
         self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
@@ -197,7 +197,7 @@ class BertEmbeddings(nn.Module):
 
 class BertSelfAttention(nn.Module):
     def __init__(self, config):
-        super(BertSelfAttention, self).__init__()
+        super().__init__()
         if config.hidden_size % config.num_attention_heads != 0:
             raise ValueError(
                 "The hidden size (%d) is not a multiple of the number of attention "
@@ -275,7 +275,7 @@ class BertSelfAttention(nn.Module):
 
 class BertSelfOutput(nn.Module):
     def __init__(self, config):
-        super(BertSelfOutput, self).__init__()
+        super().__init__()
         self.dense = nn.Linear(config.hidden_size, config.hidden_size)
         self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps)
         self.dropout = nn.Dropout(config.hidden_dropout_prob)
@@ -289,7 +289,7 @@ class BertSelfOutput(nn.Module):
 
 class BertAttention(nn.Module):
     def __init__(self, config):
-        super(BertAttention, self).__init__()
+        super().__init__()
         self.self = BertSelfAttention(config)
         self.output = BertSelfOutput(config)
         self.pruned_heads = set()
@@ -335,7 +335,7 @@ class BertAttention(nn.Module):
 
 class BertIntermediate(nn.Module):
     def __init__(self, config):
-        super(BertIntermediate, self).__init__()
+        super().__init__()
         self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
         if isinstance(config.hidden_act, str):
             self.intermediate_act_fn = ACT2FN[config.hidden_act]
@@ -350,7 +350,7 @@ class BertIntermediate(nn.Module):
 
 class BertOutput(nn.Module):
     def __init__(self, config):
-        super(BertOutput, self).__init__()
+        super().__init__()
         self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
         self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps)
         self.dropout = nn.Dropout(config.hidden_dropout_prob)
@@ -364,7 +364,7 @@ class BertOutput(nn.Module):
 
 class BertLayer(nn.Module):
     def __init__(self, config):
-        super(BertLayer, self).__init__()
+        super().__init__()
         self.attention = BertAttention(config)
         self.is_decoder = config.is_decoder
         if self.is_decoder:
@@ -399,7 +399,7 @@ class BertLayer(nn.Module):
 
 class BertEncoder(nn.Module):
     def __init__(self, config):
-        super(BertEncoder, self).__init__()
+        super().__init__()
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
         self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
@@ -440,7 +440,7 @@ class BertEncoder(nn.Module):
 
 class BertPooler(nn.Module):
     def __init__(self, config):
-        super(BertPooler, self).__init__()
+        super().__init__()
         self.dense = nn.Linear(config.hidden_size, config.hidden_size)
         self.activation = nn.Tanh()
 
@@ -455,7 +455,7 @@ class BertPooler(nn.Module):
 
 class BertPredictionHeadTransform(nn.Module):
     def __init__(self, config):
-        super(BertPredictionHeadTransform, self).__init__()
+        super().__init__()
         self.dense = nn.Linear(config.hidden_size, config.hidden_size)
         if isinstance(config.hidden_act, str):
             self.transform_act_fn = ACT2FN[config.hidden_act]
@@ -472,7 +472,7 @@ class BertPredictionHeadTransform(nn.Module):
 
 class BertLMPredictionHead(nn.Module):
     def __init__(self, config):
-        super(BertLMPredictionHead, self).__init__()
+        super().__init__()
         self.transform = BertPredictionHeadTransform(config)
 
         # The output weights are the same as the input embeddings, but there is
@@ -481,6 +481,9 @@ class BertLMPredictionHead(nn.Module):
 
         self.bias = nn.Parameter(torch.zeros(config.vocab_size))
 
+        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
+        self.decoder.bias = self.bias
+
     def forward(self, hidden_states):
         hidden_states = self.transform(hidden_states)
         hidden_states = self.decoder(hidden_states) + self.bias
@@ -489,7 +492,7 @@ class BertLMPredictionHead(nn.Module):
 
 class BertOnlyMLMHead(nn.Module):
     def __init__(self, config):
-        super(BertOnlyMLMHead, self).__init__()
+        super().__init__()
         self.predictions = BertLMPredictionHead(config)
 
     def forward(self, sequence_output):
@@ -499,7 +502,7 @@ class BertOnlyMLMHead(nn.Module):
 
 class BertOnlyNSPHead(nn.Module):
     def __init__(self, config):
-        super(BertOnlyNSPHead, self).__init__()
+        super().__init__()
         self.seq_relationship = nn.Linear(config.hidden_size, 2)
 
     def forward(self, pooled_output):
@@ -509,7 +512,7 @@ class BertOnlyNSPHead(nn.Module):
 
 class BertPreTrainingHeads(nn.Module):
     def __init__(self, config):
-        super(BertPreTrainingHeads, self).__init__()
+        super().__init__()
         self.predictions = BertLMPredictionHead(config)
         self.seq_relationship = nn.Linear(config.hidden_size, 2)
 
@@ -654,7 +657,7 @@ class BertModel(BertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(BertModel, self).__init__(config)
+        super().__init__(config)
         self.config = config
 
         self.embeddings = BertEmbeddings(config)
@@ -861,7 +864,7 @@ class BertForPreTraining(BertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(BertForPreTraining, self).__init__(config)
+        super().__init__(config)
 
         self.bert = BertModel(config)
         self.cls = BertPreTrainingHeads(config)
@@ -951,7 +954,7 @@ class BertForMaskedLM(BertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(BertForMaskedLM, self).__init__(config)
+        super().__init__(config)
 
         self.bert = BertModel(config)
         self.cls = BertOnlyMLMHead(config)
@@ -1050,7 +1053,7 @@ class BertForNextSentencePrediction(BertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(BertForNextSentencePrediction, self).__init__(config)
+        super().__init__(config)
 
         self.bert = BertModel(config)
         self.cls = BertOnlyNSPHead(config)
@@ -1129,7 +1132,7 @@ class BertForSequenceClassification(BertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(BertForSequenceClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.bert = BertModel(config)
@@ -1218,7 +1221,7 @@ class BertForMultipleChoice(BertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(BertForMultipleChoice, self).__init__(config)
+        super().__init__(config)
 
         self.bert = BertModel(config)
         self.dropout = nn.Dropout(config.hidden_dropout_prob)
@@ -1305,7 +1308,7 @@ class BertForTokenClassification(BertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(BertForTokenClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.bert = BertModel(config)
@@ -1403,7 +1406,7 @@ class BertForQuestionAnswering(BertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(BertForQuestionAnswering, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.bert = BertModel(config)

src/transformers/modeling_ctrl.py

@@ -81,7 +81,7 @@ def scaled_dot_product_attention(q, k, v, mask, attention_mask=None, head_mask=N
 
 class MultiHeadAttention(torch.nn.Module):
     def __init__(self, d_model_size, num_heads, output_attentions=False):
-        super(MultiHeadAttention, self).__init__()
+        super().__init__()
         self.output_attentions = output_attentions
         self.num_heads = num_heads
         self.d_model_size = d_model_size
@@ -132,7 +132,7 @@ def point_wise_feed_forward_network(d_model_size, dff):
 
 class EncoderLayer(torch.nn.Module):
     def __init__(self, d_model_size, num_heads, dff, rate=0.1, output_attentions=False):
-        super(EncoderLayer, self).__init__()
+        super().__init__()
 
         self.multi_head_attention = MultiHeadAttention(d_model_size, num_heads, output_attentions)
         self.ffn = point_wise_feed_forward_network(d_model_size, dff)
@@ -274,7 +274,7 @@ class CTRLModel(CTRLPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(CTRLModel, self).__init__(config)
+        super().__init__(config)
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
         self.output_past = config.output_past
@@ -481,7 +481,7 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(CTRLLMHeadModel, self).__init__(config)
+        super().__init__(config)
         self.transformer = CTRLModel(config)
         self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=True)
 

src/transformers/modeling_distilbert.py

@@ -40,6 +40,7 @@ DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
     "distilbert-base-uncased-distilled-squad": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin",
     "distilbert-base-german-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-pytorch_model.bin",
     "distilbert-base-multilingual-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-pytorch_model.bin",
+    "distilbert-base-uncased-finetuned-sst-2-english": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin",
 }
 
 
@@ -58,7 +59,7 @@ def create_sinusoidal_embeddings(n_pos, dim, out):
 
 class Embeddings(nn.Module):
     def __init__(self, config):
-        super(Embeddings, self).__init__()
+        super().__init__()
         self.word_embeddings = nn.Embedding(config.vocab_size, config.dim, padding_idx=0)
         self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.dim)
         if config.sinusoidal_pos_embds:
@@ -96,7 +97,7 @@ class Embeddings(nn.Module):
 
 class MultiHeadSelfAttention(nn.Module):
     def __init__(self, config):
-        super(MultiHeadSelfAttention, self).__init__()
+        super().__init__()
 
         self.n_heads = config.n_heads
         self.dim = config.dim
@@ -194,7 +195,7 @@ class MultiHeadSelfAttention(nn.Module):
 
 class FFN(nn.Module):
     def __init__(self, config):
-        super(FFN, self).__init__()
+        super().__init__()
         self.dropout = nn.Dropout(p=config.dropout)
         self.lin1 = nn.Linear(in_features=config.dim, out_features=config.hidden_dim)
         self.lin2 = nn.Linear(in_features=config.hidden_dim, out_features=config.dim)
@@ -213,7 +214,7 @@ class FFN(nn.Module):
 
 class TransformerBlock(nn.Module):
     def __init__(self, config):
-        super(TransformerBlock, self).__init__()
+        super().__init__()
 
         self.n_heads = config.n_heads
         self.dim = config.dim
@@ -265,7 +266,7 @@ class TransformerBlock(nn.Module):
 
 class Transformer(nn.Module):
     def __init__(self, config):
-        super(Transformer, self).__init__()
+        super().__init__()
         self.n_layers = config.n_layers
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
@@ -423,7 +424,7 @@ class DistilBertModel(DistilBertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(DistilBertModel, self).__init__(config)
+        super().__init__(config)
 
         self.embeddings = Embeddings(config)  # Embeddings
         self.transformer = Transformer(config)  # Encoder
@@ -524,7 +525,7 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(DistilBertForMaskedLM, self).__init__(config)
+        super().__init__(config)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
 
@@ -599,7 +600,7 @@ class DistilBertForSequenceClassification(DistilBertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(DistilBertForSequenceClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.distilbert = DistilBertModel(config)
@@ -678,7 +679,7 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(DistilBertForQuestionAnswering, self).__init__(config)
+        super().__init__(config)
 
         self.distilbert = DistilBertModel(config)
         self.qa_outputs = nn.Linear(config.dim, config.num_labels)
@@ -765,7 +766,7 @@ class DistilBertForTokenClassification(DistilBertPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(DistilBertForTokenClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.distilbert = DistilBertModel(config)

src/transformers/modeling_encoder_decoder.py

@@ -37,7 +37,7 @@ class PreTrainedEncoderDecoder(nn.Module):
     """
 
     def __init__(self, encoder, decoder):
-        super(PreTrainedEncoderDecoder, self).__init__()
+        super().__init__()
         self.encoder = encoder
         self.decoder = decoder
 
@@ -290,7 +290,7 @@ class Model2Model(PreTrainedEncoderDecoder):
     """
 
     def __init__(self, *args, **kwargs):
-        super(Model2Model, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.tie_weights()
 
     def tie_weights(self):
@@ -321,7 +321,7 @@ class Model2Model(PreTrainedEncoderDecoder):
         ):
             raise ValueError("Only the Bert model is currently supported.")
 
-        model = super(Model2Model, cls).from_pretrained(
+        model = super().from_pretrained(
             encoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
             decoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
             *args,
@@ -345,5 +345,5 @@ class Model2LSTM(PreTrainedEncoderDecoder):
                     "      E.g. `decoder_config={'input_size': 768, 'hidden_size': 768, 'num_layers': 2}`"
                 )
             kwargs["decoder_model"] = torch.nn.LSTM(kwargs.pop("decoder_config"))
-        model = super(Model2LSTM, cls).from_pretrained(*args, **kwargs)
+        model = super().from_pretrained(*args, **kwargs)
         return model

src/transformers/modeling_gpt2.py

@@ -101,7 +101,7 @@ def gelu(x):
 
 class Attention(nn.Module):
     def __init__(self, nx, n_ctx, config, scale=False):
-        super(Attention, self).__init__()
+        super().__init__()
         self.output_attentions = config.output_attentions
 
         n_state = nx  # in Attention: n_state=768 (nx=n_embd)
@@ -202,7 +202,7 @@ class Attention(nn.Module):
 
 class MLP(nn.Module):
     def __init__(self, n_state, config):  # in MLP: n_state=3072 (4 * n_embd)
-        super(MLP, self).__init__()
+        super().__init__()
         nx = config.n_embd
         self.c_fc = Conv1D(n_state, nx)
         self.c_proj = Conv1D(nx, n_state)
@@ -217,7 +217,7 @@ class MLP(nn.Module):
 
 class Block(nn.Module):
     def __init__(self, n_ctx, config, scale=False):
-        super(Block, self).__init__()
+        super().__init__()
         nx = config.n_embd
         self.ln_1 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon)
         self.attn = Attention(nx, n_ctx, config, scale)
@@ -249,7 +249,7 @@ class GPT2PreTrainedModel(PreTrainedModel):
     base_model_prefix = "transformer"
 
     def __init__(self, *inputs, **kwargs):
-        super(GPT2PreTrainedModel, self).__init__(*inputs, **kwargs)
+        super().__init__(*inputs, **kwargs)
 
     def _init_weights(self, module):
         """ Initialize the weights.
@@ -355,7 +355,7 @@ class GPT2Model(GPT2PreTrainedModel):
     """
 
     def __init__(self, config):
-        super(GPT2Model, self).__init__(config)
+        super().__init__(config)
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
         self.output_past = config.output_past
@@ -550,7 +550,7 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
     """
 
     def __init__(self, config):
-        super(GPT2LMHeadModel, self).__init__(config)
+        super().__init__(config)
         self.transformer = GPT2Model(config)
         self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
 
@@ -678,7 +678,7 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
     """
 
     def __init__(self, config):
-        super(GPT2DoubleHeadsModel, self).__init__(config)
+        super().__init__(config)
         config.num_labels = 1
         self.transformer = GPT2Model(config)
         self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)

src/transformers/modeling_mmbt.py

@@ -33,7 +33,7 @@ class ModalEmbeddings(nn.Module):
     """
 
     def __init__(self, config, encoder, embeddings):
-        super(ModalEmbeddings, self).__init__()
+        super().__init__()
         self.config = config
         self.encoder = encoder
         self.proj_embeddings = nn.Linear(config.modal_hidden_size, config.hidden_size)
@@ -84,7 +84,7 @@ MMBT_START_DOCSTRING = r"""    MMBT model was proposed in
     refer to the PyTorch documentation for all matter related to general usage and behavior.
 
     .. _`Supervised Multimodal Bitransformers for Classifying Images and Text`:
-        https://www.github.com/salesforce/ctrl
+        https://github.com/facebookresearch/mmbt
 
     .. _`torch.nn.Module`:
         https://pytorch.org/docs/stable/nn.html#module
@@ -175,7 +175,7 @@ class MMBTModel(nn.Module):
         """
 
     def __init__(self, config, transformer, encoder):
-        super(MMBTModel, self).__init__()
+        super().__init__()
         self.config = config
         self.transformer = transformer
         self.modal_encoder = ModalEmbeddings(config, encoder, transformer.embeddings)
@@ -359,7 +359,7 @@ class MMBTForClassification(nn.Module):
         """
 
     def __init__(self, config, transformer, encoder):
-        super(MMBTForClassification, self).__init__()
+        super().__init__()
         self.num_labels = config.num_labels
 
         self.mmbt = MMBTModel(config, transformer, encoder)

src/transformers/modeling_openai.py

@@ -127,7 +127,7 @@ ACT_FNS = {"relu": nn.ReLU, "swish": swish, "gelu": gelu}
 
 class Attention(nn.Module):
     def __init__(self, nx, n_ctx, config, scale=False):
-        super(Attention, self).__init__()
+        super().__init__()
         n_state = nx  # in Attention: n_state=768 (nx=n_embd)
         # [switch nx => n_state from Block to Attention to keep identical to TF implem]
         assert n_state % config.n_head == 0
@@ -221,7 +221,7 @@ class Attention(nn.Module):
 
 class MLP(nn.Module):
     def __init__(self, n_state, config):  # in MLP: n_state=3072 (4 * n_embd)
-        super(MLP, self).__init__()
+        super().__init__()
         nx = config.n_embd
         self.c_fc = Conv1D(n_state, nx)
         self.c_proj = Conv1D(nx, n_state)
@@ -236,7 +236,7 @@ class MLP(nn.Module):
 
 class Block(nn.Module):
     def __init__(self, n_ctx, config, scale=False):
-        super(Block, self).__init__()
+        super().__init__()
         nx = config.n_embd
         self.attn = Attention(nx, n_ctx, config, scale)
         self.ln_1 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon)
@@ -359,7 +359,7 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(OpenAIGPTModel, self).__init__(config)
+        super().__init__(config)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
 
@@ -518,7 +518,7 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(OpenAIGPTLMHeadModel, self).__init__(config)
+        super().__init__(config)
         self.transformer = OpenAIGPTModel(config)
         self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
 
@@ -623,7 +623,7 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(OpenAIGPTDoubleHeadsModel, self).__init__(config)
+        super().__init__(config)
 
         config.num_labels = 1
         self.transformer = OpenAIGPTModel(config)

src/transformers/modeling_roberta.py

@@ -45,7 +45,7 @@ class RobertaEmbeddings(BertEmbeddings):
     """
 
     def __init__(self, config):
-        super(RobertaEmbeddings, self).__init__(config)
+        super().__init__(config)
         self.padding_idx = 1
         self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=self.padding_idx)
         self.position_embeddings = nn.Embedding(
@@ -60,7 +60,7 @@ class RobertaEmbeddings(BertEmbeddings):
             else:
                 position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds)
 
-        return super(RobertaEmbeddings, self).forward(
+        return super().forward(
             input_ids, token_type_ids=token_type_ids, position_ids=position_ids, inputs_embeds=inputs_embeds
         )
 
@@ -204,7 +204,7 @@ class RobertaModel(BertModel):
     base_model_prefix = "roberta"
 
     def __init__(self, config):
-        super(RobertaModel, self).__init__(config)
+        super().__init__(config)
 
         self.embeddings = RobertaEmbeddings(config)
         self.init_weights()
@@ -254,7 +254,7 @@ class RobertaForMaskedLM(BertPreTrainedModel):
     base_model_prefix = "roberta"
 
     def __init__(self, config):
-        super(RobertaForMaskedLM, self).__init__(config)
+        super().__init__(config)
 
         self.roberta = RobertaModel(config)
         self.lm_head = RobertaLMHead(config)
@@ -299,13 +299,16 @@ class RobertaLMHead(nn.Module):
     """Roberta Head for masked language modeling."""
 
     def __init__(self, config):
-        super(RobertaLMHead, self).__init__()
+        super().__init__()
         self.dense = nn.Linear(config.hidden_size, config.hidden_size)
         self.layer_norm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps)
 
         self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
         self.bias = nn.Parameter(torch.zeros(config.vocab_size))
 
+        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
+        self.decoder.bias = self.bias
+
     def forward(self, features, **kwargs):
         x = self.dense(features)
         x = gelu(x)
@@ -359,7 +362,7 @@ class RobertaForSequenceClassification(BertPreTrainedModel):
     base_model_prefix = "roberta"
 
     def __init__(self, config):
-        super(RobertaForSequenceClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.roberta = RobertaModel(config)
@@ -481,7 +484,7 @@ class RobertaForMultipleChoice(BertPreTrainedModel):
     base_model_prefix = "roberta"
 
     def __init__(self, config):
-        super(RobertaForMultipleChoice, self).__init__(config)
+        super().__init__(config)
 
         self.roberta = RobertaModel(config)
         self.dropout = nn.Dropout(config.hidden_dropout_prob)
@@ -568,7 +571,7 @@ class RobertaForTokenClassification(BertPreTrainedModel):
     base_model_prefix = "roberta"
 
     def __init__(self, config):
-        super(RobertaForTokenClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.roberta = RobertaModel(config)
@@ -622,7 +625,7 @@ class RobertaClassificationHead(nn.Module):
     """Head for sentence-level classification tasks."""
 
     def __init__(self, config):
-        super(RobertaClassificationHead, self).__init__()
+        super().__init__()
         self.dense = nn.Linear(config.hidden_size, config.hidden_size)
         self.dropout = nn.Dropout(config.hidden_dropout_prob)
         self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
@@ -681,7 +684,7 @@ class RobertaForQuestionAnswering(BertPreTrainedModel):
     base_model_prefix = "roberta"
 
     def __init__(self, config):
-        super(RobertaForQuestionAnswering, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.roberta = RobertaModel(config)

src/transformers/modeling_t5.py

@@ -142,7 +142,7 @@ class T5LayerNorm(nn.Module):
         """ Construct a layernorm module in the T5 style
             No bias and no substraction of mean.
         """
-        super(T5LayerNorm, self).__init__()
+        super().__init__()
         self.weight = nn.Parameter(torch.ones(hidden_size))
         self.variance_epsilon = eps
 
@@ -154,7 +154,7 @@ class T5LayerNorm(nn.Module):
 
 class T5DenseReluDense(nn.Module):
     def __init__(self, config):
-        super(T5DenseReluDense, self).__init__()
+        super().__init__()
         self.wi = nn.Linear(config.d_model, config.d_ff, bias=False)
         self.wo = nn.Linear(config.d_ff, config.d_model, bias=False)
         self.dropout = nn.Dropout(config.dropout_rate)
@@ -169,7 +169,7 @@ class T5DenseReluDense(nn.Module):
 
 class T5LayerFF(nn.Module):
     def __init__(self, config):
-        super(T5LayerFF, self).__init__()
+        super().__init__()
         self.DenseReluDense = T5DenseReluDense(config)
         self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
         self.dropout = nn.Dropout(config.dropout_rate)
@@ -185,7 +185,7 @@ class T5Attention(nn.Module):
     NEW_ID = itertools.count()
 
     def __init__(self, config, has_relative_attention_bias=False):
-        super(T5Attention, self).__init__()
+        super().__init__()
         self.layer_id = next(T5Attention.NEW_ID)
         self.is_decoder = config.is_decoder
         self.has_relative_attention_bias = has_relative_attention_bias
@@ -286,6 +286,7 @@ class T5Attention(nn.Module):
             bidirectional=not self.is_decoder,
             num_buckets=self.relative_attention_num_buckets,
         )
+        rp_bucket = rp_bucket.to(self.relative_attention_bias.weight.device)
         values = self.relative_attention_bias(rp_bucket)  # shape (qlen, klen, num_heads)
         values = values.permute([2, 0, 1]).unsqueeze(0)  # shape (1, num_heads, qlen, klen)
         return values
@@ -362,7 +363,7 @@ class T5Attention(nn.Module):
 
 class T5LayerSelfAttention(nn.Module):
     def __init__(self, config, has_relative_attention_bias=False):
-        super(T5LayerSelfAttention, self).__init__()
+        super().__init__()
         self.SelfAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias)
         self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
         self.dropout = nn.Dropout(config.dropout_rate)
@@ -380,7 +381,7 @@ class T5LayerSelfAttention(nn.Module):
 
 class T5LayerCrossAttention(nn.Module):
     def __init__(self, config, has_relative_attention_bias=False):
-        super(T5LayerCrossAttention, self).__init__()
+        super().__init__()
         self.EncDecAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias)
         self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
         self.dropout = nn.Dropout(config.dropout_rate)
@@ -398,7 +399,7 @@ class T5LayerCrossAttention(nn.Module):
 
 class T5Block(nn.Module):
     def __init__(self, config, has_relative_attention_bias=False):
-        super(T5Block, self).__init__()
+        super().__init__()
         self.is_decoder = config.is_decoder
         self.layer = nn.ModuleList()
         self.layer.append(T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias))
@@ -500,7 +501,7 @@ class T5PreTrainedModel(PreTrainedModel):
 
 class T5Stack(T5PreTrainedModel):
     def __init__(self, config):
-        super(T5Stack, self).__init__(config)
+        super().__init__(config)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
         self.is_decoder = config.is_decoder
@@ -723,7 +724,7 @@ class T5Model(T5PreTrainedModel):
     """
 
     def __init__(self, config):
-        super(T5Model, self).__init__(config)
+        super().__init__(config)
         self.shared = nn.Embedding(config.vocab_size, config.d_model)
 
         encoder_config = copy.deepcopy(config)
@@ -829,7 +830,7 @@ class T5WithLMHeadModel(T5PreTrainedModel):
     """
 
     def __init__(self, config):
-        super(T5WithLMHeadModel, self).__init__(config)
+        super().__init__(config)
         self.model_dim = config.d_model
 
         self.shared = nn.Embedding(config.vocab_size, config.d_model)

src/transformers/modeling_tf_albert.py

@@ -45,7 +45,7 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer):
     """
 
     def __init__(self, config, **kwargs):
-        super(TFAlbertEmbeddings, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.config = config
         self.position_embeddings = tf.keras.layers.Embedding(
@@ -76,7 +76,7 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer):
                 shape=[self.config.vocab_size, self.config.embedding_size],
                 initializer=get_initializer(self.config.initializer_range),
             )
-        super(TFAlbertEmbeddings, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, inputs, mode="embedding", training=False):
         """Get token embeddings of inputs.
@@ -141,7 +141,7 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer):
 
 class TFAlbertSelfAttention(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFAlbertSelfAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         if config.hidden_size % config.num_attention_heads != 0:
             raise ValueError(
                 "The hidden size (%d) is not a multiple of the number of attention "
@@ -217,7 +217,7 @@ class TFAlbertSelfAttention(tf.keras.layers.Layer):
 
 class TFAlbertSelfOutput(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFAlbertSelfOutput, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.dense = tf.keras.layers.Dense(
             config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="dense"
         )
@@ -235,7 +235,7 @@ class TFAlbertSelfOutput(tf.keras.layers.Layer):
 
 class TFAlbertAttention(TFBertSelfAttention):
     def __init__(self, config, **kwargs):
-        super(TFAlbertAttention, self).__init__(config, **kwargs)
+        super().__init__(config, **kwargs)
 
         self.hidden_size = config.hidden_size
         self.dense = tf.keras.layers.Dense(
@@ -303,7 +303,7 @@ class TFAlbertAttention(TFBertSelfAttention):
 
 class TFAlbertLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFAlbertLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.attention = TFAlbertAttention(config, name="attention")
 
         self.ffn = tf.keras.layers.Dense(
@@ -341,7 +341,7 @@ class TFAlbertLayer(tf.keras.layers.Layer):
 
 class TFAlbertLayerGroup(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFAlbertLayerGroup, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
@@ -376,7 +376,7 @@ class TFAlbertLayerGroup(tf.keras.layers.Layer):
 
 class TFAlbertTransformer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFAlbertTransformer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.config = config
         self.output_attentions = config.output_attentions
@@ -445,7 +445,7 @@ class TFAlbertPreTrainedModel(TFPreTrainedModel):
 
 class TFAlbertMLMHead(tf.keras.layers.Layer):
     def __init__(self, config, input_embeddings, **kwargs):
-        super(TFAlbertMLMHead, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = config.vocab_size
 
         self.dense = tf.keras.layers.Dense(
@@ -467,7 +467,7 @@ class TFAlbertMLMHead(tf.keras.layers.Layer):
         self.decoder_bias = self.add_weight(
             shape=(self.vocab_size,), initializer="zeros", trainable=True, name="decoder/bias"
         )
-        super(TFAlbertMLMHead, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, hidden_states):
         hidden_states = self.dense(hidden_states)
@@ -596,7 +596,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel):
     """
 
     def __init__(self, config, **kwargs):
-        super(TFAlbertModel, self).__init__(config, **kwargs)
+        super().__init__(config, **kwargs)
         self.num_hidden_layers = config.num_hidden_layers
 
         self.embeddings = TFAlbertEmbeddings(config, name="embeddings")
@@ -733,7 +733,7 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFAlbertForMaskedLM, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
 
         self.albert = TFAlbertModel(config, name="albert")
         self.predictions = TFAlbertMLMHead(config, self.albert.embeddings, name="predictions")
@@ -786,7 +786,7 @@ class TFAlbertForSequenceClassification(TFAlbertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFAlbertForSequenceClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.albert = TFAlbertModel(config, name="albert")

src/transformers/modeling_tf_auto.py

@@ -16,18 +16,23 @@
 
 
 import logging
+from collections import OrderedDict
 
 from .configuration_auto import (
+    AlbertConfig,
+    AutoConfig,
     BertConfig,
     CTRLConfig,
     DistilBertConfig,
     GPT2Config,
     OpenAIGPTConfig,
     RobertaConfig,
+    T5Config,
     TransfoXLConfig,
     XLMConfig,
     XLNetConfig,
 )
+from .configuration_utils import PretrainedConfig
 from .modeling_tf_albert import (
     TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP,
     TFAlbertForMaskedLM,
@@ -104,6 +109,67 @@ TF_ALL_PRETRAINED_MODEL_ARCHIVE_MAP = dict(
     for key, value, in pretrained_map.items()
 )
 
+TF_MODEL_MAPPING = OrderedDict(
+    [
+        (T5Config, TFT5Model),
+        (DistilBertConfig, TFDistilBertModel),
+        (AlbertConfig, TFAlbertModel),
+        (RobertaConfig, TFRobertaModel),
+        (BertConfig, TFBertModel),
+        (OpenAIGPTConfig, TFOpenAIGPTModel),
+        (GPT2Config, TFGPT2Model),
+        (TransfoXLConfig, TFTransfoXLModel),
+        (XLNetConfig, TFXLNetModel),
+        (XLMConfig, TFXLMModel),
+        (CTRLConfig, TFCTRLModel),
+    ]
+)
+
+TF_MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
+    [
+        (T5Config, TFT5WithLMHeadModel),
+        (DistilBertConfig, TFDistilBertForMaskedLM),
+        (AlbertConfig, TFAlbertForMaskedLM),
+        (RobertaConfig, TFRobertaForMaskedLM),
+        (BertConfig, TFBertForMaskedLM),
+        (OpenAIGPTConfig, TFOpenAIGPTLMHeadModel),
+        (GPT2Config, TFGPT2LMHeadModel),
+        (TransfoXLConfig, TFTransfoXLLMHeadModel),
+        (XLNetConfig, TFXLNetLMHeadModel),
+        (XLMConfig, TFXLMWithLMHeadModel),
+        (CTRLConfig, TFCTRLLMHeadModel),
+    ]
+)
+
+TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
+    [
+        (DistilBertConfig, TFDistilBertForSequenceClassification),
+        (AlbertConfig, TFAlbertForSequenceClassification),
+        (RobertaConfig, TFRobertaForSequenceClassification),
+        (BertConfig, TFBertForSequenceClassification),
+        (XLNetConfig, TFXLNetForSequenceClassification),
+        (XLMConfig, TFXLMForSequenceClassification),
+    ]
+)
+
+TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
+    [
+        (DistilBertConfig, TFDistilBertForQuestionAnswering),
+        (BertConfig, TFBertForQuestionAnswering),
+        (XLNetConfig, TFXLNetForQuestionAnsweringSimple),
+        (XLMConfig, TFXLMForQuestionAnsweringSimple),
+    ]
+)
+
+TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = OrderedDict(
+    [
+        (DistilBertConfig, TFDistilBertForTokenClassification),
+        (RobertaConfig, TFRobertaForTokenClassification),
+        (BertConfig, TFBertForTokenClassification),
+        (XLNetConfig, TFXLNetForTokenClassification),
+    ]
+)
+
 
 class TFAutoModel(object):
     r"""
@@ -113,7 +179,8 @@ class TFAutoModel(object):
         class method.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The base model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -160,25 +227,15 @@ class TFAutoModel(object):
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = TFAutoModel.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, DistilBertConfig):
-            return TFDistilBertModel(config)
-        elif isinstance(config, RobertaConfig):
-            return TFRobertaModel(config)
-        elif isinstance(config, BertConfig):
-            return TFBertModel(config)
-        elif isinstance(config, OpenAIGPTConfig):
-            return TFOpenAIGPTModel(config)
-        elif isinstance(config, GPT2Config):
-            return TFGPT2Model(config)
-        elif isinstance(config, TransfoXLConfig):
-            return TFTransfoXLModel(config)
-        elif isinstance(config, XLNetConfig):
-            return TFXLNetModel(config)
-        elif isinstance(config, XLMConfig):
-            return TFXLMModel(config)
-        elif isinstance(config, CTRLConfig):
-            return TFCTRLModel(config)
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in TF_MODEL_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__, cls.__name__, ", ".join(c.__name__ for c in TF_MODEL_MAPPING.keys())
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -257,33 +314,18 @@ class TFAutoModel(object):
             model = TFAutoModel.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config)
 
         """
-        if "t5" in pretrained_model_name_or_path:
-            return TFT5Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "distilbert" in pretrained_model_name_or_path:
-            return TFDistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "albert" in pretrained_model_name_or_path:
-            return TFAlbertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "roberta" in pretrained_model_name_or_path:
-            return TFRobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "bert" in pretrained_model_name_or_path:
-            return TFBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "openai-gpt" in pretrained_model_name_or_path:
-            return TFOpenAIGPTModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "gpt2" in pretrained_model_name_or_path:
-            return TFGPT2Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "transfo-xl" in pretrained_model_name_or_path:
-            return TFTransfoXLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return TFXLNetModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlm" in pretrained_model_name_or_path:
-            return TFXLMModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "ctrl" in pretrained_model_name_or_path:
-            return TFCTRLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
 
+        for config_class, model_class in TF_MODEL_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'distilbert', 'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-            "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__, cls.__name__, ", ".join(c.__name__ for c in TF_MODEL_MAPPING.keys())
+            )
         )
 
 
@@ -295,7 +337,8 @@ class TFAutoModelWithLMHead(object):
         class method.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -342,25 +385,15 @@ class TFAutoModelWithLMHead(object):
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = AutoModelWithLMHead.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, DistilBertConfig):
-            return TFDistilBertForMaskedLM(config)
-        elif isinstance(config, RobertaConfig):
-            return TFRobertaForMaskedLM(config)
-        elif isinstance(config, BertConfig):
-            return TFBertForMaskedLM(config)
-        elif isinstance(config, OpenAIGPTConfig):
-            return TFOpenAIGPTLMHeadModel(config)
-        elif isinstance(config, GPT2Config):
-            return TFGPT2LMHeadModel(config)
-        elif isinstance(config, TransfoXLConfig):
-            return TFTransfoXLLMHeadModel(config)
-        elif isinstance(config, XLNetConfig):
-            return TFXLNetLMHeadModel(config)
-        elif isinstance(config, XLMConfig):
-            return TFXLMWithLMHeadModel(config)
-        elif isinstance(config, CTRLConfig):
-            return TFCTRLLMHeadModel(config)
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in TF_MODEL_WITH_LM_HEAD_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__, cls.__name__, ", ".join(c.__name__ for c in TF_MODEL_WITH_LM_HEAD_MAPPING.keys())
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -368,7 +401,8 @@ class TFAutoModelWithLMHead(object):
         from a pre-trained model configuration.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -443,33 +477,18 @@ class TFAutoModelWithLMHead(object):
             model = TFAutoModelWithLMHead.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config)
 
         """
-        if "t5" in pretrained_model_name_or_path:
-            return TFT5WithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "distilbert" in pretrained_model_name_or_path:
-            return TFDistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "albert" in pretrained_model_name_or_path:
-            return TFAlbertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "roberta" in pretrained_model_name_or_path:
-            return TFRobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "bert" in pretrained_model_name_or_path:
-            return TFBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "openai-gpt" in pretrained_model_name_or_path:
-            return TFOpenAIGPTLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "gpt2" in pretrained_model_name_or_path:
-            return TFGPT2LMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "transfo-xl" in pretrained_model_name_or_path:
-            return TFTransfoXLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return TFXLNetLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlm" in pretrained_model_name_or_path:
-            return TFXLMWithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "ctrl" in pretrained_model_name_or_path:
-            return TFCTRLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
 
+        for config_class, model_class in TF_MODEL_WITH_LM_HEAD_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'distilbert', 'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-            "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__, cls.__name__, ", ".join(c.__name__ for c in TF_MODEL_WITH_LM_HEAD_MAPPING.keys())
+            )
         )
 
 
@@ -481,7 +500,8 @@ class TFAutoModelForSequenceClassification(object):
         class method.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -519,17 +539,17 @@ class TFAutoModelForSequenceClassification(object):
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = AutoModelForSequenceClassification.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, DistilBertConfig):
-            return TFDistilBertForSequenceClassification(config)
-        elif isinstance(config, RobertaConfig):
-            return TFRobertaForSequenceClassification(config)
-        elif isinstance(config, BertConfig):
-            return TFBertForSequenceClassification(config)
-        elif isinstance(config, XLNetConfig):
-            return TFXLNetForSequenceClassification(config)
-        elif isinstance(config, XLMConfig):
-            return TFXLMForSequenceClassification(config)
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys()),
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -537,7 +557,8 @@ class TFAutoModelForSequenceClassification(object):
         from a pre-trained model configuration.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -610,32 +631,20 @@ class TFAutoModelForSequenceClassification(object):
             model = TFAutoModelForSequenceClassification.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config)
 
         """
-        if "distilbert" in pretrained_model_name_or_path:
-            return TFDistilBertForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "albert" in pretrained_model_name_or_path:
-            return TFAlbertForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "roberta" in pretrained_model_name_or_path:
-            return TFRobertaForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "bert" in pretrained_model_name_or_path:
-            return TFBertForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "xlnet" in pretrained_model_name_or_path:
-            return TFXLNetForSequenceClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "xlm" in pretrained_model_name_or_path:
-            return TFXLMForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
 
+        for config_class, model_class in TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'distilbert', 'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys()),
+            )
         )
 
 
@@ -647,7 +656,8 @@ class TFAutoModelForQuestionAnswering(object):
         class method.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -683,15 +693,17 @@ class TFAutoModelForQuestionAnswering(object):
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = AutoModelForSequenceClassification.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, DistilBertConfig):
-            return TFDistilBertForQuestionAnswering(config)
-        elif isinstance(config, BertConfig):
-            return TFBertForQuestionAnswering(config)
-        elif isinstance(config, XLNetConfig):
-            raise NotImplementedError("TFXLNetForQuestionAnswering isn't implemented")
-        elif isinstance(config, XLMConfig):
-            raise NotImplementedError("TFXLMForQuestionAnswering isn't implemented")
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys()),
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -699,7 +711,8 @@ class TFAutoModelForQuestionAnswering(object):
         from a pre-trained model configuration.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -771,24 +784,20 @@ class TFAutoModelForQuestionAnswering(object):
             model = TFAutoModelForQuestionAnswering.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config)
 
         """
-        if "distilbert" in pretrained_model_name_or_path:
-            return TFDistilBertForQuestionAnswering.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "bert" in pretrained_model_name_or_path:
-            return TFBertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return TFXLNetForQuestionAnsweringSimple.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "xlm" in pretrained_model_name_or_path:
-            return TFXLMForQuestionAnsweringSimple.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
 
+        for config_class, model_class in TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'distilbert', 'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys()),
+            )
         )
 
 
@@ -817,15 +826,17 @@ class TFAutoModelForTokenClassification:
             config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
             model = TFAutoModelForTokenClassification.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
         """
-        if isinstance(config, BertConfig):
-            return TFBertForTokenClassification(config)
-        elif isinstance(config, XLNetConfig):
-            return TFXLNetForTokenClassification(config)
-        elif isinstance(config, DistilBertConfig):
-            return TFDistilBertForTokenClassification(config)
-        elif isinstance(config, RobertaConfig):
-            return TFRobertaForTokenClassification(config)
-        raise ValueError("Unrecognized configuration class {}".format(config))
+        for config_class, model_class in TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class(config)
+        raise ValueError(
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys()),
+            )
+        )
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
@@ -833,7 +844,8 @@ class TFAutoModelForTokenClassification:
         from a pre-trained model configuration.
 
         The `from_pretrained()` method takes care of returning the correct model class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The model class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -898,20 +910,18 @@ class TFAutoModelForTokenClassification:
             model = TFAutoModelForTokenClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
 
         """
-        if "bert" in pretrained_model_name_or_path:
-            return TFBertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return TFXLNetForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
-        elif "distilbert" in pretrained_model_name_or_path:
-            return TFDistilBertForTokenClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
-        elif "roberta" in pretrained_model_name_or_path:
-            return TFRobertaForTokenClassification.from_pretrained(
-                pretrained_model_name_or_path, *model_args, **kwargs
-            )
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
 
+        for config_class, model_class in TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.items():
+            if isinstance(config, config_class):
+                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'bert', 'xlnet', 'distilbert', 'roberta'".format(pretrained_model_name_or_path)
+            "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"
+            "Model type should be one of {}.".format(
+                config.__class__,
+                cls.__name__,
+                ", ".join(c.__name__ for c in TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys()),
+            )
         )

src/transformers/modeling_tf_bert.py

@@ -93,7 +93,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer):
     """
 
     def __init__(self, config, **kwargs):
-        super(TFBertEmbeddings, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = config.vocab_size
         self.hidden_size = config.hidden_size
         self.initializer_range = config.initializer_range
@@ -126,7 +126,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer):
                 shape=[self.vocab_size, self.hidden_size],
                 initializer=get_initializer(self.initializer_range),
             )
-        super(TFBertEmbeddings, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, inputs, mode="embedding", training=False):
         """Get token embeddings of inputs.
@@ -193,7 +193,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer):
 
 class TFBertSelfAttention(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertSelfAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         if config.hidden_size % config.num_attention_heads != 0:
             raise ValueError(
                 "The hidden size (%d) is not a multiple of the number of attention "
@@ -269,7 +269,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer):
 
 class TFBertSelfOutput(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertSelfOutput, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.dense = tf.keras.layers.Dense(
             config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="dense"
         )
@@ -287,7 +287,7 @@ class TFBertSelfOutput(tf.keras.layers.Layer):
 
 class TFBertAttention(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.self_attention = TFBertSelfAttention(config, name="self")
         self.dense_output = TFBertSelfOutput(config, name="output")
 
@@ -305,7 +305,7 @@ class TFBertAttention(tf.keras.layers.Layer):
 
 class TFBertIntermediate(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertIntermediate, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.dense = tf.keras.layers.Dense(
             config.intermediate_size, kernel_initializer=get_initializer(config.initializer_range), name="dense"
         )
@@ -322,7 +322,7 @@ class TFBertIntermediate(tf.keras.layers.Layer):
 
 class TFBertOutput(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertOutput, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.dense = tf.keras.layers.Dense(
             config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="dense"
         )
@@ -340,7 +340,7 @@ class TFBertOutput(tf.keras.layers.Layer):
 
 class TFBertLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.attention = TFBertAttention(config, name="attention")
         self.intermediate = TFBertIntermediate(config, name="intermediate")
         self.bert_output = TFBertOutput(config, name="output")
@@ -358,7 +358,7 @@ class TFBertLayer(tf.keras.layers.Layer):
 
 class TFBertEncoder(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertEncoder, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
         self.layer = [TFBertLayer(config, name="layer_._{}".format(i)) for i in range(config.num_hidden_layers)]
@@ -392,7 +392,7 @@ class TFBertEncoder(tf.keras.layers.Layer):
 
 class TFBertPooler(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertPooler, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.dense = tf.keras.layers.Dense(
             config.hidden_size,
             kernel_initializer=get_initializer(config.initializer_range),
@@ -410,7 +410,7 @@ class TFBertPooler(tf.keras.layers.Layer):
 
 class TFBertPredictionHeadTransform(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertPredictionHeadTransform, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.dense = tf.keras.layers.Dense(
             config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="dense"
         )
@@ -429,7 +429,7 @@ class TFBertPredictionHeadTransform(tf.keras.layers.Layer):
 
 class TFBertLMPredictionHead(tf.keras.layers.Layer):
     def __init__(self, config, input_embeddings, **kwargs):
-        super(TFBertLMPredictionHead, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = config.vocab_size
         self.transform = TFBertPredictionHeadTransform(config, name="transform")
 
@@ -439,7 +439,7 @@ class TFBertLMPredictionHead(tf.keras.layers.Layer):
 
     def build(self, input_shape):
         self.bias = self.add_weight(shape=(self.vocab_size,), initializer="zeros", trainable=True, name="bias")
-        super(TFBertLMPredictionHead, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, hidden_states):
         hidden_states = self.transform(hidden_states)
@@ -450,7 +450,7 @@ class TFBertLMPredictionHead(tf.keras.layers.Layer):
 
 class TFBertMLMHead(tf.keras.layers.Layer):
     def __init__(self, config, input_embeddings, **kwargs):
-        super(TFBertMLMHead, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.predictions = TFBertLMPredictionHead(config, input_embeddings, name="predictions")
 
     def call(self, sequence_output):
@@ -460,7 +460,7 @@ class TFBertMLMHead(tf.keras.layers.Layer):
 
 class TFBertNSPHead(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertNSPHead, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.seq_relationship = tf.keras.layers.Dense(
             2, kernel_initializer=get_initializer(config.initializer_range), name="seq_relationship"
         )
@@ -472,7 +472,7 @@ class TFBertNSPHead(tf.keras.layers.Layer):
 
 class TFBertMainLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFBertMainLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.num_hidden_layers = config.num_hidden_layers
 
         self.embeddings = TFBertEmbeddings(config, name="embeddings")
@@ -707,7 +707,7 @@ class TFBertModel(TFBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFBertModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.bert = TFBertMainLayer(config, name="bert")
 
     def call(self, inputs, **kwargs):
@@ -750,7 +750,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFBertForPreTraining, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
 
         self.bert = TFBertMainLayer(config, name="bert")
         self.nsp = TFBertNSPHead(config, name="nsp___cls")
@@ -803,7 +803,7 @@ class TFBertForMaskedLM(TFBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFBertForMaskedLM, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
 
         self.bert = TFBertMainLayer(config, name="bert")
         self.mlm = TFBertMLMHead(config, self.bert.embeddings, name="mlm___cls")
@@ -854,7 +854,7 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFBertForNextSentencePrediction, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
 
         self.bert = TFBertMainLayer(config, name="bert")
         self.nsp = TFBertNSPHead(config, name="nsp___cls")
@@ -903,7 +903,7 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFBertForSequenceClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.bert = TFBertMainLayer(config, name="bert")
@@ -960,7 +960,7 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFBertForMultipleChoice, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
 
         self.bert = TFBertMainLayer(config, name="bert")
         self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
@@ -1064,7 +1064,7 @@ class TFBertForTokenClassification(TFBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFBertForTokenClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.bert = TFBertMainLayer(config, name="bert")
@@ -1121,7 +1121,7 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFBertForQuestionAnswering, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.bert = TFBertMainLayer(config, name="bert")

src/transformers/modeling_tf_ctrl.py

@@ -75,7 +75,7 @@ def scaled_dot_product_attention(q, k, v, mask, attention_mask=None, head_mask=N
 
 class TFMultiHeadAttention(tf.keras.layers.Layer):
     def __init__(self, d_model_size, num_heads, output_attentions=False, **kwargs):
-        super(TFMultiHeadAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_attentions = output_attentions
         self.num_heads = num_heads
         self.d_model_size = d_model_size
@@ -132,7 +132,7 @@ class TFEncoderLayer(tf.keras.layers.Layer):
     def __init__(
         self, d_model_size, num_heads, dff, rate=0.1, layer_norm_epsilon=1e-6, output_attentions=False, **kwargs
     ):
-        super(TFEncoderLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.multi_head_attention = TFMultiHeadAttention(
             d_model_size, num_heads, output_attentions, name="multi_head_attention"
@@ -166,7 +166,7 @@ class TFEncoderLayer(tf.keras.layers.Layer):
 
 class TFCTRLMainLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFCTRLMainLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
         self.output_past = config.output_past
@@ -443,7 +443,7 @@ class TFCTRLModel(TFCTRLPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFCTRLModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFCTRLMainLayer(config, name="transformer")
 
     def call(self, inputs, **kwargs):
@@ -453,7 +453,7 @@ class TFCTRLModel(TFCTRLPreTrainedModel):
 
 class TFCTRLLMHead(tf.keras.layers.Layer):
     def __init__(self, config, input_embeddings, **kwargs):
-        super(TFCTRLLMHead, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = config.vocab_size
 
         # The output weights are the same as the input embeddings, but there is
@@ -462,7 +462,7 @@ class TFCTRLLMHead(tf.keras.layers.Layer):
 
     def build(self, input_shape):
         self.bias = self.add_weight(shape=(self.vocab_size,), initializer="zeros", trainable=True, name="bias")
-        super(TFCTRLLMHead, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, hidden_states):
         hidden_states = self.input_embeddings(hidden_states, mode="linear")
@@ -508,7 +508,7 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFCTRLLMHeadModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFCTRLMainLayer(config, name="transformer")
 
         self.lm_head = TFCTRLLMHead(config, self.transformer.w, name="lm_head")

src/transformers/modeling_tf_distilbert.py

@@ -34,6 +34,7 @@ TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
     "distilbert-base-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-tf_model.h5",
     "distilbert-base-uncased-distilled-squad": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-tf_model.h5",
     "distilbert-base-multilingual-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-tf_model.h5",
+    "distilbert-base-uncased-finetuned-sst-2-english": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-tf_model.h5",
 }
 
 
@@ -64,7 +65,7 @@ def gelu_new(x):
 
 class TFEmbeddings(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFEmbeddings, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = config.vocab_size
         self.dim = config.dim
         self.initializer_range = config.initializer_range
@@ -91,7 +92,7 @@ class TFEmbeddings(tf.keras.layers.Layer):
             self.word_embeddings = self.add_weight(
                 "weight", shape=[self.vocab_size, self.dim], initializer=get_initializer(self.initializer_range)
             )
-        super(TFEmbeddings, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, inputs, inputs_embeds=None, mode="embedding", training=False):
         """Get token embeddings of inputs.
@@ -168,7 +169,7 @@ class TFEmbeddings(tf.keras.layers.Layer):
 
 class TFMultiHeadSelfAttention(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFMultiHeadSelfAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.n_heads = config.n_heads
         self.dim = config.dim
@@ -258,7 +259,7 @@ class TFMultiHeadSelfAttention(tf.keras.layers.Layer):
 
 class TFFFN(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFFFN, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.dropout = tf.keras.layers.Dropout(config.dropout)
         self.lin1 = tf.keras.layers.Dense(
             config.hidden_dim, kernel_initializer=get_initializer(config.initializer_range), name="lin1"
@@ -283,7 +284,7 @@ class TFFFN(tf.keras.layers.Layer):
 
 class TFTransformerBlock(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFTransformerBlock, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.n_heads = config.n_heads
         self.dim = config.dim
@@ -337,7 +338,7 @@ class TFTransformerBlock(tf.keras.layers.Layer):
 
 class TFTransformer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFTransformer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.n_layers = config.n_layers
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
@@ -398,7 +399,7 @@ class TFTransformer(tf.keras.layers.Layer):
 
 class TFDistilBertMainLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFDistilBertMainLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.num_hidden_layers = config.num_hidden_layers
 
         self.embeddings = TFEmbeddings(config, name="embeddings")  # Embeddings
@@ -568,7 +569,7 @@ class TFDistilBertModel(TFDistilBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFDistilBertModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.distilbert = TFDistilBertMainLayer(config, name="distilbert")  # Embeddings
 
     def call(self, inputs, **kwargs):
@@ -578,7 +579,7 @@ class TFDistilBertModel(TFDistilBertPreTrainedModel):
 
 class TFDistilBertLMHead(tf.keras.layers.Layer):
     def __init__(self, config, input_embeddings, **kwargs):
-        super(TFDistilBertLMHead, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = config.vocab_size
 
         # The output weights are the same as the input embeddings, but there is
@@ -587,7 +588,7 @@ class TFDistilBertLMHead(tf.keras.layers.Layer):
 
     def build(self, input_shape):
         self.bias = self.add_weight(shape=(self.vocab_size,), initializer="zeros", trainable=True, name="bias")
-        super(TFDistilBertLMHead, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, hidden_states):
         hidden_states = self.input_embeddings(hidden_states, mode="linear")
@@ -627,7 +628,7 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFDistilBertForMaskedLM, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
         self.vocab_size = config.vocab_size
@@ -689,7 +690,7 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFDistilBertForSequenceClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.distilbert = TFDistilBertMainLayer(config, name="distilbert")
@@ -746,7 +747,7 @@ class TFDistilBertForTokenClassification(TFDistilBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFDistilBertForTokenClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.distilbert = TFDistilBertMainLayer(config, name="distilbert")
@@ -803,7 +804,7 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFDistilBertForQuestionAnswering, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
 
         self.distilbert = TFDistilBertMainLayer(config, name="distilbert")
         self.qa_outputs = tf.keras.layers.Dense(

src/transformers/modeling_tf_gpt2.py

@@ -58,7 +58,7 @@ def gelu(x):
 
 class TFAttention(tf.keras.layers.Layer):
     def __init__(self, nx, n_ctx, config, scale=False, **kwargs):
-        super(TFAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_attentions = config.output_attentions
 
         n_state = nx  # in Attention: n_state=768 (nx=n_embd)
@@ -157,7 +157,7 @@ class TFAttention(tf.keras.layers.Layer):
 
 class TFMLP(tf.keras.layers.Layer):
     def __init__(self, n_state, config, **kwargs):
-        super(TFMLP, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         nx = config.n_embd
         self.c_fc = TFConv1D(n_state, nx, initializer_range=config.initializer_range, name="c_fc")
         self.c_proj = TFConv1D(nx, n_state, initializer_range=config.initializer_range, name="c_proj")
@@ -173,7 +173,7 @@ class TFMLP(tf.keras.layers.Layer):
 
 class TFBlock(tf.keras.layers.Layer):
     def __init__(self, n_ctx, config, scale=False, **kwargs):
-        super(TFBlock, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         nx = config.n_embd
         self.ln_1 = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name="ln_1")
         self.attn = TFAttention(nx, n_ctx, config, scale, name="attn")
@@ -198,7 +198,7 @@ class TFBlock(tf.keras.layers.Layer):
 
 class TFGPT2MainLayer(tf.keras.layers.Layer):
     def __init__(self, config, *inputs, **kwargs):
-        super(TFGPT2MainLayer, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
         self.num_hidden_layers = config.n_layer
@@ -475,7 +475,7 @@ class TFGPT2Model(TFGPT2PreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFGPT2Model, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFGPT2MainLayer(config, name="transformer")
 
     def call(self, inputs, **kwargs):
@@ -521,7 +521,7 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFGPT2LMHeadModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFGPT2MainLayer(config, name="transformer")
 
     def get_output_embeddings(self):
@@ -598,7 +598,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFGPT2DoubleHeadsModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         config.num_labels = 1
         self.transformer = TFGPT2MainLayer(config, name="transformer")
         self.multiple_choice_head = TFSequenceSummary(

src/transformers/modeling_tf_openai.py

@@ -66,7 +66,7 @@ ACT_FNS = {
 
 class TFAttention(tf.keras.layers.Layer):
     def __init__(self, nx, n_ctx, config, scale=False, **kwargs):
-        super(TFAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_attentions = config.output_attentions
 
         n_state = nx  # in Attention: n_state=768 (nx=n_embd)
@@ -160,7 +160,7 @@ class TFAttention(tf.keras.layers.Layer):
 
 class TFMLP(tf.keras.layers.Layer):
     def __init__(self, n_state, config, **kwargs):
-        super(TFMLP, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         nx = config.n_embd
         self.c_fc = TFConv1D(n_state, nx, initializer_range=config.initializer_range, name="c_fc")
         self.c_proj = TFConv1D(nx, n_state, initializer_range=config.initializer_range, name="c_proj")
@@ -176,7 +176,7 @@ class TFMLP(tf.keras.layers.Layer):
 
 class TFBlock(tf.keras.layers.Layer):
     def __init__(self, n_ctx, config, scale=False, **kwargs):
-        super(TFBlock, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         nx = config.n_embd
         self.attn = TFAttention(nx, n_ctx, config, scale, name="attn")
         self.ln_1 = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name="ln_1")
@@ -199,7 +199,7 @@ class TFBlock(tf.keras.layers.Layer):
 
 class TFOpenAIGPTMainLayer(tf.keras.layers.Layer):
     def __init__(self, config, *inputs, **kwargs):
-        super(TFOpenAIGPTMainLayer, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
         self.num_hidden_layers = config.n_layer
@@ -453,7 +453,7 @@ class TFOpenAIGPTModel(TFOpenAIGPTPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFOpenAIGPTModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFOpenAIGPTMainLayer(config, name="transformer")
 
     def call(self, inputs, **kwargs):
@@ -494,7 +494,7 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFOpenAIGPTLMHeadModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFOpenAIGPTMainLayer(config, name="transformer")
 
     def get_output_embeddings(self):
@@ -563,7 +563,7 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFOpenAIGPTDoubleHeadsModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         config.num_labels = 1
         self.transformer = TFOpenAIGPTMainLayer(config, name="transformer")
         self.multiple_choice_head = TFSequenceSummary(

src/transformers/modeling_tf_roberta.py

@@ -42,7 +42,7 @@ class TFRobertaEmbeddings(TFBertEmbeddings):
     """
 
     def __init__(self, config, **kwargs):
-        super(TFRobertaEmbeddings, self).__init__(config, **kwargs)
+        super().__init__(config, **kwargs)
         self.padding_idx = 1
 
     def create_position_ids_from_input_ids(self, x):
@@ -78,9 +78,7 @@ class TFRobertaEmbeddings(TFBertEmbeddings):
             else:
                 position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds)
 
-        return super(TFRobertaEmbeddings, self)._embedding(
-            [input_ids, position_ids, token_type_ids, inputs_embeds], training=training
-        )
+        return super()._embedding([input_ids, position_ids, token_type_ids, inputs_embeds], training=training)
 
 
 class TFRobertaMainLayer(TFBertMainLayer):
@@ -89,7 +87,7 @@ class TFRobertaMainLayer(TFBertMainLayer):
     """
 
     def __init__(self, config, **kwargs):
-        super(TFRobertaMainLayer, self).__init__(config, **kwargs)
+        super().__init__(config, **kwargs)
         self.embeddings = TFRobertaEmbeddings(config, name="embeddings")
 
     def get_input_embeddings(self):
@@ -234,7 +232,7 @@ class TFRobertaModel(TFRobertaPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFRobertaModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.roberta = TFRobertaMainLayer(config, name="roberta")
 
     def call(self, inputs, **kwargs):
@@ -246,7 +244,7 @@ class TFRobertaLMHead(tf.keras.layers.Layer):
     """Roberta Head for masked language modeling."""
 
     def __init__(self, config, input_embeddings, **kwargs):
-        super(TFRobertaLMHead, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = config.vocab_size
         self.dense = tf.keras.layers.Dense(
             config.hidden_size, kernel_initializer=get_initializer(config.initializer_range), name="dense"
@@ -260,7 +258,7 @@ class TFRobertaLMHead(tf.keras.layers.Layer):
 
     def build(self, input_shape):
         self.bias = self.add_weight(shape=(self.vocab_size,), initializer="zeros", trainable=True, name="bias")
-        super(TFRobertaLMHead, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, features):
         x = self.dense(features)
@@ -305,7 +303,7 @@ class TFRobertaForMaskedLM(TFRobertaPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFRobertaForMaskedLM, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
 
         self.roberta = TFRobertaMainLayer(config, name="roberta")
         self.lm_head = TFRobertaLMHead(config, self.roberta.embeddings, name="lm_head")
@@ -328,7 +326,7 @@ class TFRobertaClassificationHead(tf.keras.layers.Layer):
     """Head for sentence-level classification tasks."""
 
     def __init__(self, config, **kwargs):
-        super(TFRobertaClassificationHead, self).__init__(config, **kwargs)
+        super().__init__(config, **kwargs)
         self.dense = tf.keras.layers.Dense(
             config.hidden_size,
             kernel_initializer=get_initializer(config.initializer_range),
@@ -383,7 +381,7 @@ class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFRobertaForSequenceClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.roberta = TFRobertaMainLayer(config, name="roberta")
@@ -433,7 +431,7 @@ class TFRobertaForTokenClassification(TFRobertaPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFRobertaForTokenClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.roberta = TFRobertaMainLayer(config, name="roberta")

src/transformers/modeling_tf_t5.py

@@ -50,13 +50,13 @@ class TFT5LayerNorm(tf.keras.layers.Layer):
         """ Construct a layernorm module in the T5 style
             No bias and no substraction of mean.
         """
-        super(TFT5LayerNorm, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.variance_epsilon = epsilon
 
     def build(self, input_shape):
         """Build shared word embedding layer """
         self.weight = self.add_weight("weight", shape=(input_shape[-1],), initializer="ones")
-        super(TFT5LayerNorm, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, x):
         variance = tf.math.reduce_mean(tf.math.square(x), axis=-1, keepdims=True)
@@ -66,7 +66,7 @@ class TFT5LayerNorm(tf.keras.layers.Layer):
 
 class TFT5DenseReluDense(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFT5DenseReluDense, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.wi = tf.keras.layers.Dense(config.d_ff, use_bias=False, name="wi")
         self.wo = tf.keras.layers.Dense(config.d_model, use_bias=False, name="wo")
         self.dropout = tf.keras.layers.Dropout(config.dropout_rate)
@@ -82,7 +82,7 @@ class TFT5DenseReluDense(tf.keras.layers.Layer):
 
 class TFT5LayerFF(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFT5LayerFF, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.DenseReluDense = TFT5DenseReluDense(config, name="DenseReluDense")
         self.layer_norm = TFT5LayerNorm(epsilon=config.layer_norm_epsilon, name="layer_norm")
         self.dropout = tf.keras.layers.Dropout(config.dropout_rate)
@@ -98,7 +98,7 @@ class TFT5Attention(tf.keras.layers.Layer):
     NEW_ID = itertools.count()
 
     def __init__(self, config, has_relative_attention_bias=False, **kwargs):
-        super(TFT5Attention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.layer_id = next(TFT5Attention.NEW_ID)
         self.is_decoder = config.is_decoder
         self.has_relative_attention_bias = has_relative_attention_bias
@@ -259,7 +259,7 @@ class TFT5Attention(tf.keras.layers.Layer):
 
 class TFT5LayerSelfAttention(tf.keras.layers.Layer):
     def __init__(self, config, has_relative_attention_bias=False, **kwargs):
-        super(TFT5LayerSelfAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.SelfAttention = TFT5Attention(
             config, has_relative_attention_bias=has_relative_attention_bias, name="SelfAttention"
         )
@@ -279,7 +279,7 @@ class TFT5LayerSelfAttention(tf.keras.layers.Layer):
 
 class TFT5LayerCrossAttention(tf.keras.layers.Layer):
     def __init__(self, config, has_relative_attention_bias=False, **kwargs):
-        super(TFT5LayerCrossAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.EncDecAttention = TFT5Attention(
             config, has_relative_attention_bias=has_relative_attention_bias, name="EncDecAttention"
         )
@@ -299,7 +299,7 @@ class TFT5LayerCrossAttention(tf.keras.layers.Layer):
 
 class TFT5Block(tf.keras.layers.Layer):
     def __init__(self, config, has_relative_attention_bias=False, **kwargs):
-        super(TFT5Block, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.is_decoder = config.is_decoder
         self.layer = []
         self.layer.append(
@@ -361,7 +361,7 @@ class TFT5Block(tf.keras.layers.Layer):
 ####################################################
 class TFT5MainLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFT5MainLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
         self.is_decoder = config.is_decoder
@@ -633,7 +633,7 @@ class TFT5Model(TFT5PreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFT5Model, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.shared = TFSharedEmbeddings(config.vocab_size, config.d_model, name="shared")
 
         encoder_config = copy.deepcopy(config)
@@ -724,7 +724,7 @@ class TFT5WithLMHeadModel(TFT5PreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFT5WithLMHeadModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.model_dim = config.d_model
 
         self.shared = TFSharedEmbeddings(config.vocab_size, config.d_model, name="shared")

src/transformers/modeling_tf_transfo_xl.py

@@ -36,7 +36,7 @@ TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP = {
 
 class TFPositionalEmbedding(tf.keras.layers.Layer):
     def __init__(self, demb, **kwargs):
-        super(TFPositionalEmbedding, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.inv_freq = 1 / (10000 ** (tf.range(0, demb, 2.0) / demb))
 
@@ -52,7 +52,7 @@ class TFPositionalEmbedding(tf.keras.layers.Layer):
 
 class TFPositionwiseFF(tf.keras.layers.Layer):
     def __init__(self, d_model, d_inner, dropout, pre_lnorm=False, layer_norm_epsilon=1e-5, init_std=0.02, **kwargs):
-        super(TFPositionwiseFF, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.d_model = d_model
         self.d_inner = d_inner
@@ -112,7 +112,7 @@ class TFRelPartialLearnableMultiHeadAttn(tf.keras.layers.Layer):
         init_std=0.02,
         **kwargs
     ):
-        super(TFRelPartialLearnableMultiHeadAttn, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.output_attentions = output_attentions
         self.n_head = n_head
@@ -155,7 +155,7 @@ class TFRelPartialLearnableMultiHeadAttn(tf.keras.layers.Layer):
             self.r_w_bias = self.add_weight(
                 shape=(self.n_head, self.d_head), initializer="zeros", trainable=True, name="r_w_bias"
             )
-        super(TFRelPartialLearnableMultiHeadAttn, self).build(input_shape)
+        super().build(input_shape)
 
     def _rel_shift(self, x):
         x_size = shape_list(x)
@@ -267,7 +267,7 @@ class TFRelPartialLearnableDecoderLayer(tf.keras.layers.Layer):
         init_std=0.02,
         **kwargs
     ):
-        super(TFRelPartialLearnableDecoderLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.dec_attn = TFRelPartialLearnableMultiHeadAttn(
             n_head,
@@ -308,7 +308,7 @@ class TFRelPartialLearnableDecoderLayer(tf.keras.layers.Layer):
 
 class TFAdaptiveEmbedding(tf.keras.layers.Layer):
     def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, init_std=0.02, sample_softmax=False, **kwargs):
-        super(TFAdaptiveEmbedding, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.n_token = n_token
         self.d_embed = d_embed
@@ -350,7 +350,7 @@ class TFAdaptiveEmbedding(tf.keras.layers.Layer):
                     name="emb_projs_._{}".format(i),
                 )
             )
-        super(TFAdaptiveEmbedding, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, inp):
         if self.div_val == 1:
@@ -380,7 +380,7 @@ class TFAdaptiveEmbedding(tf.keras.layers.Layer):
 
 class TFTransfoXLMainLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFTransfoXLMainLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
 
@@ -455,7 +455,7 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer):
             self.r_r_bias = self.add_weight(
                 shape=(self.n_head, self.d_head), initializer="zeros", trainable=True, name="r_r_bias"
             )
-        super(TFTransfoXLMainLayer, self).build(input_shape)
+        super().build(input_shape)
 
     def get_input_embeddings(self):
         return self.word_emb
@@ -728,7 +728,7 @@ class TFTransfoXLModel(TFTransfoXLPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFTransfoXLModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFTransfoXLMainLayer(config, name="transformer")
 
     def call(self, inputs, **kwargs):
@@ -774,7 +774,7 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(TFTransfoXLLMHeadModel, self).__init__(config)
+        super().__init__(config)
         self.transformer = TFTransfoXLMainLayer(config, name="transformer")
         self.sample_softmax = config.sample_softmax
         # use sampled softmax

src/transformers/modeling_tf_transfo_xl_utilities.py

@@ -24,7 +24,7 @@ from .modeling_tf_utils import shape_list
 
 class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer):
     def __init__(self, vocab_size, d_embed, d_proj, cutoffs, div_val=1, keep_order=False, **kwargs):
-        super(TFAdaptiveSoftmaxMask, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.vocab_size = vocab_size
         self.d_embed = d_embed
@@ -98,7 +98,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer):
                     name="out_layers_._{}_._bias".format(i),
                 )
                 self.out_layers.append((weight, bias))
-        super(TFAdaptiveSoftmaxMask, self).build(input_shape)
+        super().build(input_shape)
 
     @staticmethod
     def _logit(x, W, b, proj=None):

src/transformers/modeling_tf_utils.py

@@ -20,6 +20,7 @@ import logging
 import os
 
 import h5py
+import numpy as np
 import tensorflow as tf
 from tensorflow.python.keras.saving import hdf5_format
 
@@ -31,7 +32,22 @@ from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model
 logger = logging.getLogger(__name__)
 
 
-class TFPreTrainedModel(tf.keras.Model):
+class TFModelUtilsMixin:
+    """
+    A few utilities for `tf.keras.Model`s, to be used as a mixin.
+    """
+
+    def num_parameters(self, only_trainable: bool = False) -> int:
+        """
+        Get number of (optionally, trainable) parameters in the model.
+        """
+        if only_trainable:
+            return int(sum(np.prod(w.shape.as_list()) for w in self.trainable_variables))
+        else:
+            return self.count_params()
+
+
+class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
     r""" Base class for all TF models.
 
         :class:`~transformers.TFPreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models
@@ -62,7 +78,7 @@ class TFPreTrainedModel(tf.keras.Model):
         return {"input_ids": tf.constant(DUMMY_INPUTS)}
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFPreTrainedModel, self).__init__(*inputs, **kwargs)
+        super().__init__(*inputs, **kwargs)
         if not isinstance(config, PretrainedConfig):
             raise ValueError(
                 "Parameter config in `{}(config)` should be an instance of class `PretrainedConfig`. "
@@ -369,7 +385,7 @@ class TFConv1D(tf.keras.layers.Layer):
         """ TFConv1D layer as defined by Radford et al. for OpenAI GPT (and also used in GPT-2)
             Basically works like a Linear layer but the weights are transposed
         """
-        super(TFConv1D, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.nf = nf
         self.nx = nx
         self.initializer_range = initializer_range
@@ -396,7 +412,7 @@ class TFSharedEmbeddings(tf.keras.layers.Layer):
     """
 
     def __init__(self, vocab_size, hidden_size, initializer_range=None, **kwargs):
-        super(TFSharedEmbeddings, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = vocab_size
         self.hidden_size = hidden_size
         self.initializer_range = hidden_size ** -0.5 if initializer_range is None else initializer_range
@@ -409,7 +425,7 @@ class TFSharedEmbeddings(tf.keras.layers.Layer):
         self.weight = self.add_weight(
             "weight", shape=[self.vocab_size, self.hidden_size], initializer=get_initializer(self.initializer_range)
         )
-        super(TFSharedEmbeddings, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, inputs, mode="embedding"):
         """Get token embeddings of inputs.
@@ -469,7 +485,7 @@ class TFSequenceSummary(tf.keras.layers.Layer):
     """
 
     def __init__(self, config, initializer_range=0.02, **kwargs):
-        super(TFSequenceSummary, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
         self.summary_type = config.summary_type if hasattr(config, "summary_use_proj") else "last"
         if self.summary_type == "attn":

src/transformers/modeling_tf_xlm.py

@@ -97,7 +97,7 @@ class TFMultiHeadAttention(tf.keras.layers.Layer):
     NEW_ID = itertools.count()
 
     def __init__(self, n_heads, dim, config, **kwargs):
-        super(TFMultiHeadAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.layer_id = next(TFMultiHeadAttention.NEW_ID)
         self.output_attentions = config.output_attentions
         self.dim = dim
@@ -182,7 +182,7 @@ class TFMultiHeadAttention(tf.keras.layers.Layer):
 
 class TFTransformerFFN(tf.keras.layers.Layer):
     def __init__(self, in_dim, dim_hidden, out_dim, config, **kwargs):
-        super(TFTransformerFFN, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.lin1 = tf.keras.layers.Dense(dim_hidden, kernel_initializer=get_initializer(config.init_std), name="lin1")
         self.lin2 = tf.keras.layers.Dense(out_dim, kernel_initializer=get_initializer(config.init_std), name="lin2")
         self.act = tf.keras.layers.Activation(gelu) if config.gelu_activation else tf.keras.activations.relu
@@ -198,7 +198,7 @@ class TFTransformerFFN(tf.keras.layers.Layer):
 
 class TFXLMMainLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFXLMMainLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
 
@@ -608,7 +608,7 @@ class TFXLMModel(TFXLMPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXLMModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFXLMMainLayer(config, name="transformer")
 
     def call(self, inputs, **kwargs):
@@ -622,7 +622,7 @@ class TFXLMPredLayer(tf.keras.layers.Layer):
     """
 
     def __init__(self, config, input_embeddings, **kwargs):
-        super(TFXLMPredLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.asm = config.asm
         self.n_words = config.n_words
         self.pad_index = config.pad_index
@@ -641,7 +641,7 @@ class TFXLMPredLayer(tf.keras.layers.Layer):
     def build(self, input_shape):
         # The output weights are the same as the input embeddings, but there is an output-only bias for each token.
         self.bias = self.add_weight(shape=(self.n_words,), initializer="zeros", trainable=True, name="bias")
-        super(TFXLMPredLayer, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, hidden_states):
         hidden_states = self.input_embeddings(hidden_states, mode="linear")
@@ -682,7 +682,7 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXLMWithLMHeadModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFXLMMainLayer(config, name="transformer")
         self.pred_layer = TFXLMPredLayer(config, self.transformer.embeddings, name="pred_layer_._proj")
 
@@ -733,7 +733,7 @@ class TFXLMForSequenceClassification(TFXLMPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXLMForSequenceClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.transformer = TFXLMMainLayer(config, name="transformer")
@@ -784,7 +784,7 @@ class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXLMForQuestionAnsweringSimple, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFXLMMainLayer(config, name="transformer")
         self.qa_outputs = tf.keras.layers.Dense(
             config.num_labels, kernel_initializer=get_initializer(config.init_std), name="qa_outputs"

src/transformers/modeling_tf_xlnet.py

@@ -57,7 +57,7 @@ ACT2FN = {
 
 class TFXLNetRelativeAttention(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFXLNetRelativeAttention, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_attentions = config.output_attentions
 
         if config.d_model % config.n_head != 0:
@@ -104,7 +104,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer):
         self.seg_embed = self.add_weight(
             shape=(2, self.n_head, self.d_head), initializer=initializer, trainable=True, name="seg_embed"
         )
-        super(TFXLNetRelativeAttention, self).build(input_shape)
+        super().build(input_shape)
 
     def prune_heads(self, heads):
         raise NotImplementedError
@@ -280,7 +280,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer):
 
 class TFXLNetFeedForward(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFXLNetFeedForward, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="layer_norm")
         self.layer_1 = tf.keras.layers.Dense(
             config.d_inner, kernel_initializer=get_initializer(config.initializer_range), name="layer_1"
@@ -307,7 +307,7 @@ class TFXLNetFeedForward(tf.keras.layers.Layer):
 
 class TFXLNetLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFXLNetLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.rel_attn = TFXLNetRelativeAttention(config, name="rel_attn")
         self.ff = TFXLNetFeedForward(config, name="ff")
         self.dropout = tf.keras.layers.Dropout(config.dropout)
@@ -326,7 +326,7 @@ class TFXLNetLayer(tf.keras.layers.Layer):
 
 class TFXLNetLMHead(tf.keras.layers.Layer):
     def __init__(self, config, input_embeddings, **kwargs):
-        super(TFXLNetLMHead, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = config.vocab_size
         # The output weights are the same as the input embeddings, but there is
         # an output-only bias for each token.
@@ -334,7 +334,7 @@ class TFXLNetLMHead(tf.keras.layers.Layer):
 
     def build(self, input_shape):
         self.bias = self.add_weight(shape=(self.vocab_size,), initializer="zeros", trainable=True, name="bias")
-        super(TFXLNetLMHead, self).build(input_shape)
+        super().build(input_shape)
 
     def call(self, hidden_states):
         hidden_states = self.input_embeddings(hidden_states, mode="linear")
@@ -344,7 +344,7 @@ class TFXLNetLMHead(tf.keras.layers.Layer):
 
 class TFXLNetMainLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFXLNetMainLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
         self.output_past = config.output_past
@@ -832,7 +832,7 @@ class TFXLNetModel(TFXLNetPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXLNetModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFXLNetMainLayer(config, name="transformer")
 
     def call(self, inputs, **kwargs):
@@ -885,7 +885,7 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXLNetLMHeadModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFXLNetMainLayer(config, name="transformer")
         self.lm_loss = TFXLNetLMHead(config, self.transformer.word_embedding, name="lm_loss")
 
@@ -940,7 +940,7 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXLNetForSequenceClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.transformer = TFXLNetMainLayer(config, name="transformer")
@@ -1001,7 +1001,7 @@ class TFXLNetForTokenClassification(TFXLNetPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXLNetForTokenClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.transformer = TFXLNetMainLayer(config, name="transformer")
@@ -1058,7 +1058,7 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXLNetForQuestionAnsweringSimple, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFXLNetMainLayer(config, name="transformer")
         self.qa_outputs = tf.keras.layers.Dense(
             config.num_labels, kernel_initializer=get_initializer(config.initializer_range), name="qa_outputs"
@@ -1127,7 +1127,7 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel):
 
 #     """
 #     def __init__(self, config, *inputs, **kwargs):
-#         super(TFXLNetForQuestionAnswering, self).__init__(config, *inputs, **kwargs)
+#         super().__init__(config, *inputs, **kwargs)
 #         self.start_n_top = config.start_n_top
 #         self.end_n_top = config.end_n_top
 

src/transformers/modeling_transfo_xl.py

@@ -165,7 +165,7 @@ def load_tf_weights_in_transfo_xl(model, config, tf_path):
 
 class PositionalEmbedding(nn.Module):
     def __init__(self, demb):
-        super(PositionalEmbedding, self).__init__()
+        super().__init__()
 
         self.demb = demb
 
@@ -184,7 +184,7 @@ class PositionalEmbedding(nn.Module):
 
 class PositionwiseFF(nn.Module):
     def __init__(self, d_model, d_inner, dropout, pre_lnorm=False, layer_norm_epsilon=1e-5):
-        super(PositionwiseFF, self).__init__()
+        super().__init__()
 
         self.d_model = d_model
         self.d_inner = d_inner
@@ -236,7 +236,7 @@ class RelPartialLearnableMultiHeadAttn(nn.Module):
         output_attentions=False,
         layer_norm_epsilon=1e-5,
     ):
-        super(RelPartialLearnableMultiHeadAttn, self).__init__()
+        super().__init__()
 
         self.output_attentions = output_attentions
         self.n_head = n_head
@@ -368,7 +368,7 @@ class RelPartialLearnableMultiHeadAttn(nn.Module):
 
 class RelPartialLearnableDecoderLayer(nn.Module):
     def __init__(self, n_head, d_model, d_head, d_inner, dropout, layer_norm_epsilon=1e-5, **kwargs):
-        super(RelPartialLearnableDecoderLayer, self).__init__()
+        super().__init__()
 
         self.dec_attn = RelPartialLearnableMultiHeadAttn(
             n_head, d_model, d_head, dropout, layer_norm_epsilon=layer_norm_epsilon, **kwargs
@@ -389,7 +389,7 @@ class RelPartialLearnableDecoderLayer(nn.Module):
 
 class AdaptiveEmbedding(nn.Module):
     def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, sample_softmax=False):
-        super(AdaptiveEmbedding, self).__init__()
+        super().__init__()
 
         self.n_token = n_token
         self.d_embed = d_embed
@@ -587,7 +587,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(TransfoXLModel, self).__init__(config)
+        super().__init__(config)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
 
@@ -845,7 +845,7 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(TransfoXLLMHeadModel, self).__init__(config)
+        super().__init__(config)
         self.transformer = TransfoXLModel(config)
         self.sample_softmax = config.sample_softmax
         # use sampled softmax

src/transformers/modeling_transfo_xl_utilities.py

@@ -29,7 +29,7 @@ import torch.nn.functional as F
 
 class ProjectedAdaptiveLogSoftmax(nn.Module):
     def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, keep_order=False):
-        super(ProjectedAdaptiveLogSoftmax, self).__init__()
+        super().__init__()
 
         self.n_token = n_token
         self.d_embed = d_embed

src/transformers/modeling_utils.py

@@ -47,13 +47,26 @@ except ImportError:
         """
 
         def __init__(self, *args, **kwargs):
-            super(Identity, self).__init__()
+            super().__init__()
 
         def forward(self, input):
             return input
 
 
-class PreTrainedModel(nn.Module):
+class ModuleUtilsMixin:
+    """
+    A few utilities for torch.nn.Modules, to be used as a mixin.
+    """
+
+    def num_parameters(self, only_trainable: bool = False) -> int:
+        """
+        Get number of (optionally, trainable) parameters in the module.
+        """
+        params = filter(lambda x: x.requires_grad, self.parameters()) if only_trainable else self.parameters()
+        return sum(p.numel() for p in params)
+
+
+class PreTrainedModel(nn.Module, ModuleUtilsMixin):
     r""" Base class for all models.
 
         :class:`~transformers.PreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models
@@ -84,7 +97,7 @@ class PreTrainedModel(nn.Module):
         return {"input_ids": torch.tensor(DUMMY_INPUTS)}
 
     def __init__(self, config, *inputs, **kwargs):
-        super(PreTrainedModel, self).__init__()
+        super().__init__()
         if not isinstance(config, PretrainedConfig):
             raise ValueError(
                 "Parameter config in `{}(config)` should be an instance of class `PretrainedConfig`. "
@@ -1089,7 +1102,7 @@ class Conv1D(nn.Module):
         """ Conv1D layer as defined by Radford et al. for OpenAI GPT (and also used in GPT-2)
             Basically works like a Linear layer but the weights are transposed
         """
-        super(Conv1D, self).__init__()
+        super().__init__()
         self.nf = nf
         w = torch.empty(nx, nf)
         nn.init.normal_(w, std=0.02)
@@ -1107,7 +1120,7 @@ class PoolerStartLogits(nn.Module):
     """ Compute SQuAD start_logits from sequence hidden states. """
 
     def __init__(self, config):
-        super(PoolerStartLogits, self).__init__()
+        super().__init__()
         self.dense = nn.Linear(config.hidden_size, 1)
 
     def forward(self, hidden_states, p_mask=None):
@@ -1132,7 +1145,7 @@ class PoolerEndLogits(nn.Module):
     """
 
     def __init__(self, config):
-        super(PoolerEndLogits, self).__init__()
+        super().__init__()
         self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
         self.activation = nn.Tanh()
         self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
@@ -1178,7 +1191,7 @@ class PoolerAnswerClass(nn.Module):
     """ Compute SQuAD 2.0 answer class from classification and start tokens hidden states. """
 
     def __init__(self, config):
-        super(PoolerAnswerClass, self).__init__()
+        super().__init__()
         self.dense_0 = nn.Linear(config.hidden_size * 2, config.hidden_size)
         self.activation = nn.Tanh()
         self.dense_1 = nn.Linear(config.hidden_size, 1, bias=False)
@@ -1263,7 +1276,7 @@ class SQuADHead(nn.Module):
     """
 
     def __init__(self, config):
-        super(SQuADHead, self).__init__()
+        super().__init__()
         self.start_n_top = config.start_n_top
         self.end_n_top = config.end_n_top
 
@@ -1355,7 +1368,7 @@ class SequenceSummary(nn.Module):
     """
 
     def __init__(self, config):
-        super(SequenceSummary, self).__init__()
+        super().__init__()
 
         self.summary_type = config.summary_type if hasattr(config, "summary_type") else "last"
         if self.summary_type == "attn":

src/transformers/modeling_xlm.py

@@ -96,7 +96,7 @@ class MultiHeadAttention(nn.Module):
     NEW_ID = itertools.count()
 
     def __init__(self, n_heads, dim, config):
-        super(MultiHeadAttention, self).__init__()
+        super().__init__()
         self.layer_id = next(MultiHeadAttention.NEW_ID)
         self.output_attentions = config.output_attentions
         self.dim = dim
@@ -197,7 +197,7 @@ class MultiHeadAttention(nn.Module):
 
 class TransformerFFN(nn.Module):
     def __init__(self, in_dim, dim_hidden, out_dim, config):
-        super(TransformerFFN, self).__init__()
+        super().__init__()
         self.dropout = config.dropout
         self.lin1 = nn.Linear(in_dim, dim_hidden)
         self.lin2 = nn.Linear(dim_hidden, out_dim)
@@ -222,7 +222,7 @@ class XLMPreTrainedModel(PreTrainedModel):
     base_model_prefix = "transformer"
 
     def __init__(self, *inputs, **kwargs):
-        super(XLMPreTrainedModel, self).__init__(*inputs, **kwargs)
+        super().__init__(*inputs, **kwargs)
 
     @property
     def dummy_inputs(self):
@@ -354,7 +354,7 @@ class XLMModel(XLMPreTrainedModel):
     """
 
     def __init__(self, config):  # , dico, is_encoder, with_output):
-        super(XLMModel, self).__init__(config)
+        super().__init__(config)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
 
@@ -585,7 +585,7 @@ class XLMPredLayer(nn.Module):
     """
 
     def __init__(self, config):
-        super(XLMPredLayer, self).__init__()
+        super().__init__()
         self.asm = config.asm
         self.n_words = config.n_words
         self.pad_index = config.pad_index
@@ -661,7 +661,7 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLMWithLMHeadModel, self).__init__(config)
+        super().__init__(config)
         self.transformer = XLMModel(config)
         self.pred_layer = XLMPredLayer(config)
 
@@ -754,7 +754,7 @@ class XLMForSequenceClassification(XLMPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLMForSequenceClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.transformer = XLMModel(config)
@@ -856,7 +856,7 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLMForQuestionAnsweringSimple, self).__init__(config)
+        super().__init__(config)
 
         self.transformer = XLMModel(config)
         self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
@@ -973,7 +973,7 @@ class XLMForQuestionAnswering(XLMPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLMForQuestionAnswering, self).__init__(config)
+        super().__init__(config)
 
         self.transformer = XLMModel(config)
         self.qa_outputs = SQuADHead(config)

src/transformers/modeling_xlnet.py

@@ -204,7 +204,7 @@ XLNetLayerNorm = nn.LayerNorm
 
 class XLNetRelativeAttention(nn.Module):
     def __init__(self, config):
-        super(XLNetRelativeAttention, self).__init__()
+        super().__init__()
         self.output_attentions = config.output_attentions
 
         if config.d_model % config.n_head != 0:
@@ -414,7 +414,7 @@ class XLNetRelativeAttention(nn.Module):
 
 class XLNetFeedForward(nn.Module):
     def __init__(self, config):
-        super(XLNetFeedForward, self).__init__()
+        super().__init__()
         self.layer_norm = XLNetLayerNorm(config.d_model, eps=config.layer_norm_eps)
         self.layer_1 = nn.Linear(config.d_model, config.d_inner)
         self.layer_2 = nn.Linear(config.d_inner, config.d_model)
@@ -437,7 +437,7 @@ class XLNetFeedForward(nn.Module):
 
 class XLNetLayer(nn.Module):
     def __init__(self, config):
-        super(XLNetLayer, self).__init__()
+        super().__init__()
         self.rel_attn = XLNetRelativeAttention(config)
         self.ff = XLNetFeedForward(config)
         self.dropout = nn.Dropout(config.dropout)
@@ -631,7 +631,7 @@ class XLNetModel(XLNetPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLNetModel, self).__init__(config)
+        super().__init__(config)
         self.output_attentions = config.output_attentions
         self.output_hidden_states = config.output_hidden_states
         self.output_past = config.output_past
@@ -996,7 +996,7 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLNetLMHeadModel, self).__init__(config)
+        super().__init__(config)
         self.attn_type = config.attn_type
         self.same_length = config.same_length
 
@@ -1119,7 +1119,7 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLNetForSequenceClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.transformer = XLNetModel(config)
@@ -1234,7 +1234,7 @@ class XLNetForTokenClassification(XLNetPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLNetForTokenClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.transformer = XLNetModel(config)
@@ -1355,7 +1355,7 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLNetForMultipleChoice, self).__init__(config)
+        super().__init__(config)
 
         self.transformer = XLNetModel(config)
         self.sequence_summary = SequenceSummary(config)
@@ -1463,7 +1463,7 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLNetForQuestionAnsweringSimple, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.transformer = XLNetModel(config)
@@ -1595,7 +1595,7 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XLNetForQuestionAnswering, self).__init__(config)
+        super().__init__(config)
         self.start_n_top = config.start_n_top
         self.end_n_top = config.end_n_top
 

src/transformers/optimization.py

@@ -114,7 +114,7 @@ class AdamW(Optimizer):
         if not 0.0 <= eps:
             raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps))
         defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, correct_bias=correct_bias)
-        super(AdamW, self).__init__(params, defaults)
+        super().__init__(params, defaults)
 
     def step(self, closure=None):
         """Performs a single optimization step.

src/transformers/optimization_tf.py

@@ -24,7 +24,7 @@ class WarmUp(tf.keras.optimizers.schedules.LearningRateSchedule):
     """Applys a warmup schedule on a given learning rate decay schedule."""
 
     def __init__(self, initial_learning_rate, decay_schedule_fn, warmup_steps, power=1.0, name=None):
-        super(WarmUp, self).__init__()
+        super().__init__()
         self.initial_learning_rate = initial_learning_rate
         self.warmup_steps = warmup_steps
         self.power = power
@@ -102,7 +102,7 @@ class AdamWeightDecay(tf.keras.optimizers.Adam):
         name="AdamWeightDecay",
         **kwargs
     ):
-        super(AdamWeightDecay, self).__init__(learning_rate, beta_1, beta_2, epsilon, amsgrad, name, **kwargs)
+        super().__init__(learning_rate, beta_1, beta_2, epsilon, amsgrad, name, **kwargs)
         self.weight_decay_rate = weight_decay_rate
         self._include_in_weight_decay = include_in_weight_decay
         self._exclude_from_weight_decay = exclude_from_weight_decay
@@ -111,10 +111,10 @@ class AdamWeightDecay(tf.keras.optimizers.Adam):
     def from_config(cls, config):
         """Creates an optimizer from its config with WarmUp custom object."""
         custom_objects = {"WarmUp": WarmUp}
-        return super(AdamWeightDecay, cls).from_config(config, custom_objects=custom_objects)
+        return super().from_config(config, custom_objects=custom_objects)
 
     def _prepare_local(self, var_device, var_dtype, apply_state):
-        super(AdamWeightDecay, self)._prepare_local(var_device, var_dtype, apply_state)
+        super()._prepare_local(var_device, var_dtype, apply_state)
         apply_state["weight_decay_rate"] = tf.constant(self.weight_decay_rate, name="adam_weight_decay_rate")
 
     def _decay_weights_op(self, var, learning_rate, apply_state):
@@ -128,7 +128,7 @@ class AdamWeightDecay(tf.keras.optimizers.Adam):
     def apply_gradients(self, grads_and_vars, clip_norm, name=None):
         grads, tvars = list(zip(*grads_and_vars))
         (grads, _) = tf.clip_by_global_norm(grads, clip_norm=clip_norm)
-        return super(AdamWeightDecay, self).apply_gradients(zip(grads, tvars))
+        return super().apply_gradients(zip(grads, tvars))
 
     def _get_lr(self, var_device, var_dtype, apply_state):
         """Retrieves the learning rate with the given state."""
@@ -147,16 +147,16 @@ class AdamWeightDecay(tf.keras.optimizers.Adam):
         lr_t, kwargs = self._get_lr(var.device, var.dtype.base_dtype, apply_state)
         decay = self._decay_weights_op(var, lr_t, apply_state)
         with tf.control_dependencies([decay]):
-            return super(AdamWeightDecay, self)._resource_apply_dense(grad, var, **kwargs)
+            return super()._resource_apply_dense(grad, var, **kwargs)
 
     def _resource_apply_sparse(self, grad, var, indices, apply_state=None):
         lr_t, kwargs = self._get_lr(var.device, var.dtype.base_dtype, apply_state)
         decay = self._decay_weights_op(var, lr_t, apply_state)
         with tf.control_dependencies([decay]):
-            return super(AdamWeightDecay, self)._resource_apply_sparse(grad, var, indices, **kwargs)
+            return super()._resource_apply_sparse(grad, var, indices, **kwargs)
 
     def get_config(self):
-        config = super(AdamWeightDecay, self).get_config()
+        config = super().get_config()
         config.update({"weight_decay_rate": self.weight_decay_rate})
         return config
 

src/transformers/pipelines.py

@@ -705,55 +705,71 @@ class QuestionAnsweringPipeline(Pipeline):
 
         # Convert inputs to features
         examples = self._args_parser(*texts, **kwargs)
-        features = squad_convert_examples_to_features(
-            examples, self.tokenizer, kwargs["max_seq_len"], kwargs["doc_stride"], kwargs["max_question_len"], False
-        )
-        fw_args = self.inputs_for_model([f.__dict__ for f in features])
+        features_list = [
+            squad_convert_examples_to_features(
+                [example],
+                self.tokenizer,
+                kwargs["max_seq_len"],
+                kwargs["doc_stride"],
+                kwargs["max_question_len"],
+                False,
+            )
+            for example in examples
+        ]
+        all_answers = []
+        for features, example in zip(features_list, examples):
+            fw_args = self.inputs_for_model([f.__dict__ for f in features])
 
-        # Manage tensor allocation on correct device
-        with self.device_placement():
-            if self.framework == "tf":
-                fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()}
-                start, end = self.model(fw_args)
-                start, end = start.numpy(), end.numpy()
-            else:
-                with torch.no_grad():
-                    # Retrieve the score for the context tokens only (removing question tokens)
-                    fw_args = {k: torch.tensor(v, device=self.device) for (k, v) in fw_args.items()}
-                    start, end = self.model(**fw_args)
-                    start, end = start.cpu().numpy(), end.cpu().numpy()
+            # Manage tensor allocation on correct device
+            with self.device_placement():
+                if self.framework == "tf":
+                    fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()}
+                    start, end = self.model(fw_args)
+                    start, end = start.numpy(), end.numpy()
+                else:
+                    with torch.no_grad():
+                        # Retrieve the score for the context tokens only (removing question tokens)
+                        fw_args = {k: torch.tensor(v, device=self.device) for (k, v) in fw_args.items()}
+                        start, end = self.model(**fw_args)
+                        start, end = start.cpu().numpy(), end.cpu().numpy()
 
-        answers = []
-        for (example, feature, start_, end_) in zip(examples, features, start, end):
-            # Normalize logits and spans to retrieve the answer
-            start_ = np.exp(start_) / np.sum(np.exp(start_))
-            end_ = np.exp(end_) / np.sum(np.exp(end_))
+            answers = []
+            for (feature, start_, end_) in zip(features, start, end):
+                # Normalize logits and spans to retrieve the answer
+                start_ = np.exp(start_) / np.sum(np.exp(start_))
+                end_ = np.exp(end_) / np.sum(np.exp(end_))
 
-            # Mask padding and question
-            start_, end_ = start_ * np.abs(np.array(feature.p_mask) - 1), end_ * np.abs(np.array(feature.p_mask) - 1)
+                # Mask padding and question
+                start_, end_ = (
+                    start_ * np.abs(np.array(feature.p_mask) - 1),
+                    end_ * np.abs(np.array(feature.p_mask) - 1),
+                )
 
-            # TODO : What happens if not possible
-            # Mask CLS
-            start_[0] = end_[0] = 0
+                # TODO : What happens if not possible
+                # Mask CLS
+                start_[0] = end_[0] = 0
 
-            starts, ends, scores = self.decode(start_, end_, kwargs["topk"], kwargs["max_answer_len"])
-            char_to_word = np.array(example.char_to_word_offset)
+                starts, ends, scores = self.decode(start_, end_, kwargs["topk"], kwargs["max_answer_len"])
+                char_to_word = np.array(example.char_to_word_offset)
 
-            # Convert the answer (tokens) back to the original text
-            answers += [
-                {
-                    "score": score.item(),
-                    "start": np.where(char_to_word == feature.token_to_orig_map[s])[0][0].item(),
-                    "end": np.where(char_to_word == feature.token_to_orig_map[e])[0][-1].item(),
-                    "answer": " ".join(
-                        example.doc_tokens[feature.token_to_orig_map[s] : feature.token_to_orig_map[e] + 1]
-                    ),
-                }
-                for s, e, score in zip(starts, ends, scores)
-            ]
-        if len(answers) == 1:
-            return answers[0]
-        return answers
+                # Convert the answer (tokens) back to the original text
+                answers += [
+                    {
+                        "score": score.item(),
+                        "start": np.where(char_to_word == feature.token_to_orig_map[s])[0][0].item(),
+                        "end": np.where(char_to_word == feature.token_to_orig_map[e])[0][-1].item(),
+                        "answer": " ".join(
+                            example.doc_tokens[feature.token_to_orig_map[s] : feature.token_to_orig_map[e] + 1]
+                        ),
+                    }
+                    for s, e, score in zip(starts, ends, scores)
+                ]
+            answers = sorted(answers, key=lambda x: x["score"], reverse=True)[: kwargs["topk"]]
+            all_answers += answers
+
+        if len(all_answers) == 1:
+            return all_answers[0]
+        return all_answers
 
     def decode(self, start: np.ndarray, end: np.ndarray, topk: int, max_answer_len: int) -> Tuple:
         """
@@ -854,10 +870,10 @@ SUPPORTED_TASKS = {
         "pt": AutoModelForSequenceClassification if is_torch_available() else None,
         "default": {
             "model": {
-                "pt": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin",
-                "tf": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-tf_model.h5",
+                "pt": "distilbert-base-uncased-finetuned-sst-2-english",
+                "tf": "distilbert-base-uncased-finetuned-sst-2-english",
             },
-            "config": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json",
+            "config": "distilbert-base-uncased-finetuned-sst-2-english",
             "tokenizer": "distilbert-base-uncased",
         },
     },
@@ -867,10 +883,10 @@ SUPPORTED_TASKS = {
         "pt": AutoModelForTokenClassification if is_torch_available() else None,
         "default": {
             "model": {
-                "pt": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-pytorch_model.bin",
-                "tf": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-tf_model.h5",
+                "pt": "dbmdz/bert-large-cased-finetuned-conll03-english",
+                "tf": "dbmdz/bert-large-cased-finetuned-conll03-english",
             },
-            "config": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json",
+            "config": "dbmdz/bert-large-cased-finetuned-conll03-english",
             "tokenizer": "bert-large-cased",
         },
     },
@@ -909,6 +925,7 @@ def pipeline(
         pipeline('sentiment-analysis')
         pipeline('question-answering', model='distilbert-base-uncased-distilled-squad', tokenizer='bert-base-cased')
         pipeline('ner', model=AutoModel.from_pretrained(...), tokenizer=AutoTokenizer.from_pretrained(...)
+        pipeline('ner', model='dbmdz/bert-large-cased-finetuned-conll03-english', tokenizer='bert-base-cased')
         pipeline('ner', model='https://...pytorch-model.bin', config='https://...config.json', tokenizer='bert-base-cased')
     """
     # Retrieve the task

src/transformers/tokenization_albert.py

@@ -79,7 +79,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
         mask_token="[MASK]",
         **kwargs
     ):
-        super(AlbertTokenizer, self).__init__(
+        super().__init__(
             bos_token=bos_token,
             eos_token=eos_token,
             unk_token=unk_token,

src/transformers/tokenization_auto.py

@@ -16,7 +16,25 @@
 
 
 import logging
+from collections import OrderedDict
 
+from .configuration_auto import (
+    AlbertConfig,
+    AutoConfig,
+    BertConfig,
+    CamembertConfig,
+    CTRLConfig,
+    DistilBertConfig,
+    GPT2Config,
+    OpenAIGPTConfig,
+    RobertaConfig,
+    T5Config,
+    TransfoXLConfig,
+    XLMConfig,
+    XLMRobertaConfig,
+    XLNetConfig,
+)
+from .configuration_utils import PretrainedConfig
 from .tokenization_albert import AlbertTokenizer
 from .tokenization_bert import BertTokenizer
 from .tokenization_bert_japanese import BertJapaneseTokenizer
@@ -36,6 +54,25 @@ from .tokenization_xlnet import XLNetTokenizer
 logger = logging.getLogger(__name__)
 
 
+TOKENIZER_MAPPING = OrderedDict(
+    [
+        (T5Config, T5Tokenizer),
+        (DistilBertConfig, DistilBertTokenizer),
+        (AlbertConfig, AlbertTokenizer),
+        (CamembertConfig, CamembertTokenizer),
+        (RobertaConfig, RobertaTokenizer),
+        (XLMRobertaConfig, XLMRobertaTokenizer),
+        (BertConfig, BertTokenizer),
+        (OpenAIGPTConfig, OpenAIGPTTokenizer),
+        (GPT2Config, GPT2Tokenizer),
+        (TransfoXLConfig, TransfoXLTokenizer),
+        (XLNetConfig, XLNetTokenizer),
+        (XLMConfig, XLMTokenizer),
+        (CTRLConfig, CTRLTokenizer),
+    ]
+)
+
+
 class AutoTokenizer(object):
     r""":class:`~transformers.AutoTokenizer` is a generic tokenizer class
         that will be instantiated as one of the tokenizer classes of the library
@@ -43,7 +80,8 @@ class AutoTokenizer(object):
         class method.
 
         The `from_pretrained()` method take care of returning the correct tokenizer class instance
-        using pattern matching on the `pretrained_model_name_or_path` string.
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
 
         The tokenizer class to instantiate is selected as the first pattern matching
         in the `pretrained_model_name_or_path` string (in the following order):
@@ -72,7 +110,7 @@ class AutoTokenizer(object):
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
-        r""" Instantiate a one of the tokenizer classes of the library
+        r""" Instantiate one of the tokenizer classes of the library
         from a pre-trained model vocabulary.
 
         The tokenizer class to instantiate is selected as the first pattern matching
@@ -129,38 +167,20 @@ class AutoTokenizer(object):
             tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/')
 
         """
-        if "t5" in pretrained_model_name_or_path:
-            return T5Tokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "distilbert" in pretrained_model_name_or_path:
-            return DistilBertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "albert" in pretrained_model_name_or_path:
-            return AlbertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "camembert" in pretrained_model_name_or_path:
-            return CamembertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "xlm-roberta" in pretrained_model_name_or_path:
-            return XLMRobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "roberta" in pretrained_model_name_or_path:
-            return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "bert-base-japanese" in pretrained_model_name_or_path:
+        config = kwargs.pop("config", None)
+        if not isinstance(config, PretrainedConfig):
+            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+        if "bert-base-japanese" in pretrained_model_name_or_path:
             return BertJapaneseTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "bert" in pretrained_model_name_or_path:
-            return BertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "openai-gpt" in pretrained_model_name_or_path:
-            return OpenAIGPTTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "gpt2" in pretrained_model_name_or_path:
-            return GPT2Tokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "transfo-xl" in pretrained_model_name_or_path:
-            return TransfoXLTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "xlnet" in pretrained_model_name_or_path:
-            return XLNetTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "xlm" in pretrained_model_name_or_path:
-            return XLMTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        elif "ctrl" in pretrained_model_name_or_path:
-            return CTRLTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+        for config_class, tokenizer_class in TOKENIZER_MAPPING.items():
+            if isinstance(config, config_class):
+                return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
         raise ValueError(
-            "Unrecognized model identifier in {}. Should contains one of "
-            "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', "
-            "'xlm-roberta', 'xlm', 'roberta', 'distilbert,' 'camembert', 'ctrl', 'albert'".format(
-                pretrained_model_name_or_path
+            "Unrecognized configuration class {} to build an AutoTokenizer.\n"
+            "Model type should be one of {}.".format(
+                config.__class__, ", ".join(c.__name__ for c in TOKENIZER_MAPPING.keys())
             )
         )

src/transformers/tokenization_bert.py

@@ -163,7 +163,7 @@ class BertTokenizer(PreTrainedTokenizer):
                 This should likely be deactivated for Japanese:
                 see: https://github.com/huggingface/pytorch-pretrained-BERT/issues/328
         """
-        super(BertTokenizer, self).__init__(
+        super().__init__(
             unk_token=unk_token,
             sep_token=sep_token,
             pad_token=pad_token,
@@ -341,7 +341,7 @@ class BasicTokenizer(object):
             if self.do_lower_case and token not in never_split:
                 token = token.lower()
                 token = self._run_strip_accents(token)
-            split_tokens.extend(self._run_split_on_punc(token))
+            split_tokens.extend(self._run_split_on_punc(token, never_split))
 
         output_tokens = whitespace_tokenize(" ".join(split_tokens))
         return output_tokens
@@ -554,7 +554,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
         add_special_tokens=True,
         **kwargs
     ):
-        super(BertTokenizerFast, self).__init__(
+        super().__init__(
             unk_token=unk_token,
             sep_token=sep_token,
             pad_token=pad_token,

src/transformers/tokenization_bert_japanese.py

@@ -115,6 +115,7 @@ class BertJapaneseTokenizer(BertTokenizer):
             mask_token=mask_token,
             **kwargs,
         )
+        # ^^ We call the grandparent's init, not the parent's.
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens
 

src/transformers/tokenization_camembert.py

@@ -66,7 +66,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
         additional_special_tokens=["<s>NOTUSED", "</s>NOTUSED"],
         **kwargs
     ):
-        super(CamembertTokenizer, self).__init__(
+        super().__init__(
             max_len=512,
             bos_token=bos_token,
             eos_token=eos_token,

src/transformers/tokenization_ctrl.py

@@ -126,7 +126,7 @@ class CTRLTokenizer(PreTrainedTokenizer):
     control_codes = CONTROL_CODES
 
     def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
-        super(CTRLTokenizer, self).__init__(unk_token=unk_token, **kwargs)
+        super().__init__(unk_token=unk_token, **kwargs)
         self.max_len_single_sentence = (
             self.max_len
         )  # no default special tokens - you can update this value if you add special tokens

src/transformers/tokenization_gpt2.py

@@ -122,7 +122,7 @@ class GPT2Tokenizer(PreTrainedTokenizer):
         eos_token="<|endoftext|>",
         **kwargs
     ):
-        super(GPT2Tokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
+        super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
         self.max_len_single_sentence = (
             self.max_len
         )  # no default special tokens - you can update this value if you add special tokens
@@ -268,9 +268,7 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
         truncation_strategy="longest_first",
         **kwargs
     ):
-        super(GPT2TokenizerFast, self).__init__(
-            bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs
-        )
+        super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
 
         self._tokenizer = tk.Tokenizer(tk.models.BPE.from_files(vocab_file, merges_file))
         self._update_special_tokens()

src/transformers/tokenization_openai.py

@@ -82,7 +82,7 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer):
     max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 
     def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
-        super(OpenAIGPTTokenizer, self).__init__(unk_token=unk_token, **kwargs)
+        super().__init__(unk_token=unk_token, **kwargs)
 
         self.max_len_single_sentence = (
             self.max_len

src/transformers/tokenization_roberta.py

@@ -84,7 +84,7 @@ class RobertaTokenizer(GPT2Tokenizer):
         mask_token="<mask>",
         **kwargs
     ):
-        super(RobertaTokenizer, self).__init__(
+        super().__init__(
             vocab_file=vocab_file,
             merges_file=merges_file,
             errors=errors,
@@ -144,9 +144,7 @@ class RobertaTokenizer(GPT2Tokenizer):
     def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
         """
         Creates a mask from the two sequences passed to be used in a sequence-pair classification task.
-        A RoBERTa sequence pair mask has the following format:
-        0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1
-        | first sequence    | second sequence
+        RoBERTa does not make use of token type ids, therefore a list of zeros is returned.
 
         if token_ids_1 is None, only returns the first portion of the mask (0's).
         """
@@ -155,4 +153,4 @@ class RobertaTokenizer(GPT2Tokenizer):
 
         if token_ids_1 is None:
             return len(cls + token_ids_0 + sep) * [0]
-        return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1]
+        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

src/transformers/tokenization_t5.py

@@ -91,7 +91,7 @@ class T5Tokenizer(PreTrainedTokenizer):
                 additional_special_tokens = []
             additional_special_tokens.extend(["<extra_id_{}>".format(i) for i in range(extra_ids)])
 
-        super(T5Tokenizer, self).__init__(
+        super().__init__(
             eos_token=eos_token,
             unk_token=unk_token,
             pad_token=pad_token,

src/transformers/tokenization_transfo_xl.py

@@ -78,7 +78,7 @@ class TransfoXLTokenizer(PreTrainedTokenizer):
         additional_special_tokens=["<formula>"],
         **kwargs
     ):
-        super(TransfoXLTokenizer, self).__init__(
+        super().__init__(
             unk_token=unk_token, eos_token=eos_token, additional_special_tokens=additional_special_tokens, **kwargs
         )
 

src/transformers/tokenization_utils.py

@@ -264,7 +264,7 @@ class PreTrainedTokenizer(object):
                 - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``.
                 - a string with the `identifier name` of a predefined tokenizer that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``.
                 - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``.
-                - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``.
+                - (not applicable to all derived classes, deprecated) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``.
 
             cache_dir: (`optional`) string:
                 Path to a directory in which a downloaded predefined tokenizer vocabulary files should be cached if the standard cache should not be used.
@@ -331,57 +331,42 @@ class PreTrainedTokenizer(object):
             # Get the vocabulary from local files
             logger.info(
                 "Model name '{}' not found in model shortcut name list ({}). "
-                "Assuming '{}' is a path or url to a directory containing tokenizer files.".format(
+                "Assuming '{}' is a path, a model identifier, or url to a directory containing tokenizer files.".format(
                     pretrained_model_name_or_path, ", ".join(s3_models), pretrained_model_name_or_path
                 )
             )
 
-            # Look for the tokenizer main vocabulary files
-            for file_id, file_name in cls.vocab_files_names.items():
-                if os.path.isdir(pretrained_model_name_or_path):
-                    # If a directory is provided we look for the standard filenames
-                    full_file_name = os.path.join(pretrained_model_name_or_path, file_name)
-                    if not os.path.exists(full_file_name):
-                        logger.info("Didn't find file {}. We won't load it.".format(full_file_name))
-                        full_file_name = None
-                elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
-                    # If a path to a file is provided we use it (will only work for non-BPE tokenizer using a single vocabulary file)
-                    full_file_name = pretrained_model_name_or_path
-                else:
-                    full_file_name = hf_bucket_url(pretrained_model_name_or_path, postfix=file_name)
-
-                vocab_files[file_id] = full_file_name
-
-            # Look for the additional tokens files
-            additional_files_names = {
-                "added_tokens_file": ADDED_TOKENS_FILE,
-                "special_tokens_map_file": SPECIAL_TOKENS_MAP_FILE,
-                "tokenizer_config_file": TOKENIZER_CONFIG_FILE,
-            }
-
-            # If a path to a file was provided, get the parent directory
-            saved_directory = pretrained_model_name_or_path
-            if os.path.exists(saved_directory) and not os.path.isdir(saved_directory):
-                saved_directory = os.path.dirname(saved_directory)
-
-            for file_id, file_name in additional_files_names.items():
-                full_file_name = os.path.join(saved_directory, file_name)
-                if not os.path.exists(full_file_name):
-                    logger.info("Didn't find file {}. We won't load it.".format(full_file_name))
-                    full_file_name = None
-                vocab_files[file_id] = full_file_name
-
-            if all(full_file_name is None for full_file_name in vocab_files.values()):
-                raise EnvironmentError(
-                    "Model name '{}' was not found in tokenizers model name list ({}). "
-                    "We assumed '{}' was a path or url to a directory containing vocabulary files "
-                    "named {} but couldn't find such vocabulary files at this path or url.".format(
-                        pretrained_model_name_or_path,
-                        ", ".join(s3_models),
-                        pretrained_model_name_or_path,
-                        list(cls.vocab_files_names.values()),
+            if os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
+                if len(cls.vocab_files_names) > 1:
+                    raise ValueError(
+                        "Calling {}.from_pretrained() with the path to a single file or url is not supported."
+                        "Use a model identifier or the path to a directory instead.".format(cls.__name__)
+                    )
+                logger.warning(
+                    "Calling {}.from_pretrained() with the path to a single file or url is deprecated".format(
+                        cls.__name__
                     )
                 )
+                file_id = list(cls.vocab_files_names.keys())[0]
+                vocab_files[file_id] = pretrained_model_name_or_path
+            else:
+                # At this point pretrained_model_name_or_path is either a directory or a model identifier name
+                additional_files_names = {
+                    "added_tokens_file": ADDED_TOKENS_FILE,
+                    "special_tokens_map_file": SPECIAL_TOKENS_MAP_FILE,
+                    "tokenizer_config_file": TOKENIZER_CONFIG_FILE,
+                }
+                # Look for the tokenizer main vocabulary files + the additional tokens files
+                for file_id, file_name in {**cls.vocab_files_names, **additional_files_names}.items():
+                    if os.path.isdir(pretrained_model_name_or_path):
+                        full_file_name = os.path.join(pretrained_model_name_or_path, file_name)
+                        if not os.path.exists(full_file_name):
+                            logger.info("Didn't find file {}. We won't load it.".format(full_file_name))
+                            full_file_name = None
+                    else:
+                        full_file_name = hf_bucket_url(pretrained_model_name_or_path, postfix=file_name)
+
+                    vocab_files[file_id] = full_file_name
 
         # Get files from url, cache, or disk depending on the case
         try:
@@ -414,6 +399,18 @@ class PreTrainedTokenizer(object):
 
             raise EnvironmentError(msg)
 
+        if all(full_file_name is None for full_file_name in resolved_vocab_files.values()):
+            raise EnvironmentError(
+                "Model name '{}' was not found in tokenizers model name list ({}). "
+                "We assumed '{}' was a path, a model identifier, or url to a directory containing vocabulary files "
+                "named {} but couldn't find such vocabulary files at this path or url.".format(
+                    pretrained_model_name_or_path,
+                    ", ".join(s3_models),
+                    pretrained_model_name_or_path,
+                    list(cls.vocab_files_names.values()),
+                )
+            )
+
         for file_id, file_path in vocab_files.items():
             if file_path == resolved_vocab_files[file_id]:
                 logger.info("loading file {}".format(file_path))
@@ -503,7 +500,8 @@ class PreTrainedTokenizer(object):
         tokenizer_config_file = os.path.join(save_directory, TOKENIZER_CONFIG_FILE)
 
         tokenizer_config = copy.deepcopy(self.init_kwargs)
-        tokenizer_config["init_inputs"] = copy.deepcopy(self.init_inputs)
+        if len(self.init_inputs) > 0:
+            tokenizer_config["init_inputs"] = copy.deepcopy(self.init_inputs)
         for file_id in self.vocab_files_names.keys():
             tokenizer_config.pop(file_id, None)
 
@@ -513,12 +511,10 @@ class PreTrainedTokenizer(object):
         with open(special_tokens_map_file, "w", encoding="utf-8") as f:
             f.write(json.dumps(self.special_tokens_map, ensure_ascii=False))
 
-        with open(added_tokens_file, "w", encoding="utf-8") as f:
-            if self.added_tokens_encoder:
+        if len(self.added_tokens_encoder) > 0:
+            with open(added_tokens_file, "w", encoding="utf-8") as f:
                 out_str = json.dumps(self.added_tokens_encoder, ensure_ascii=False)
-            else:
-                out_str = "{}"
-            f.write(out_str)
+                f.write(out_str)
 
         vocab_files = self.save_vocabulary(save_directory)
 
@@ -800,7 +796,7 @@ class PreTrainedTokenizer(object):
                 - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length)
             pad_to_max_length: if set to True, the returned sequences will be padded according to the model's padding side and
                 padding index, up to their max length. If no max length is specified, the padding is done up to the model's max length.
-                The tokenizer padding sides are handled by the following strings:
+                The tokenizer padding sides are handled by the class attribute `padding_side` which can be set to the following strings:
                 - 'left': pads on the left of the sequences
                 - 'right': pads on the right of the sequences
                 Defaults to False: no padding.
@@ -863,7 +859,7 @@ class PreTrainedTokenizer(object):
                 - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length)
             pad_to_max_length: if set to True, the returned sequences will be padded according to the model's padding side and
                 padding index, up to their max length. If no max length is specified, the padding is done up to the model's max length.
-                The tokenizer padding sides are handled by the following strings:
+                The tokenizer padding sides are handled by the class attribute `padding_side` which can be set to the following strings:
                 - 'left': pads on the left of the sequences
                 - 'right': pads on the right of the sequences
                 Defaults to False: no padding.
@@ -1194,14 +1190,18 @@ class PreTrainedTokenizer(object):
         # Prepare inputs as tensors if asked
         if return_tensors == "tf" and is_tf_available():
             encoded_inputs["input_ids"] = tf.constant([encoded_inputs["input_ids"]])
-            encoded_inputs["token_type_ids"] = tf.constant([encoded_inputs["token_type_ids"]])
+
+            if "token_type_ids" in encoded_inputs:
+                encoded_inputs["token_type_ids"] = tf.constant([encoded_inputs["token_type_ids"]])
 
             if "attention_mask" in encoded_inputs:
                 encoded_inputs["attention_mask"] = tf.constant([encoded_inputs["attention_mask"]])
 
         elif return_tensors == "pt" and is_torch_available():
             encoded_inputs["input_ids"] = torch.tensor([encoded_inputs["input_ids"]])
-            encoded_inputs["token_type_ids"] = torch.tensor([encoded_inputs["token_type_ids"]])
+
+            if "token_type_ids" in encoded_inputs:
+                encoded_inputs["token_type_ids"] = torch.tensor([encoded_inputs["token_type_ids"]])
 
             if "attention_mask" in encoded_inputs:
                 encoded_inputs["attention_mask"] = torch.tensor([encoded_inputs["attention_mask"]])
@@ -1421,7 +1421,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer):
     _decoder = None
 
     def __init__(self, **kwargs):
-        super(PreTrainedTokenizerFast, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
     @property
     def tokenizer(self):

src/transformers/tokenization_xlm.py

@@ -578,7 +578,7 @@ class XLMTokenizer(PreTrainedTokenizer):
         do_lowercase_and_remove_accent=True,
         **kwargs
     ):
-        super(XLMTokenizer, self).__init__(
+        super().__init__(
             unk_token=unk_token,
             bos_token=bos_token,
             sep_token=sep_token,

src/transformers/tokenization_xlm_roberta.py

@@ -75,7 +75,7 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
         mask_token="<mask>",
         **kwargs
     ):
-        super(XLMRobertaTokenizer, self).__init__(
+        super().__init__(
             bos_token=bos_token,
             eos_token=eos_token,
             unk_token=unk_token,

src/transformers/tokenization_xlnet.py

@@ -77,7 +77,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
         additional_special_tokens=["<eop>", "<eod>"],
         **kwargs
     ):
-        super(XLNetTokenizer, self).__init__(
+        super().__init__(
             bos_token=bos_token,
             eos_token=eos_token,
             unk_token=unk_token,

templates/adding_a_new_model/configuration_xxx.py

@@ -58,6 +58,7 @@ class XxxConfig(PretrainedConfig):
             layer_norm_eps: The epsilon used by LayerNorm.
     """
     pretrained_config_archive_map = XXX_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "xxx"
 
     def __init__(
         self,
@@ -79,7 +80,7 @@ class XxxConfig(PretrainedConfig):
         summary_first_dropout=0.1,
         **kwargs
     ):
-        super(XxxConfig, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.vocab_size = vocab_size
         self.n_ctx = n_ctx
         self.n_positions = n_positions

templates/adding_a_new_model/modeling_tf_xxx.py

@@ -69,7 +69,7 @@ TFXxxOutput = tf.keras.layers.Layer
 
 class TFXxxLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFXxxLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.attention = TFXxxAttention(config, name="attention")
         self.intermediate = TFXxxIntermediate(config, name="intermediate")
         self.transformer_output = TFXxxOutput(config, name="output")
@@ -91,7 +91,7 @@ class TFXxxLayer(tf.keras.layers.Layer):
 ####################################################
 class TFXxxMainLayer(tf.keras.layers.Layer):
     def __init__(self, config, **kwargs):
-        super(TFXxxMainLayer, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
     def _resize_token_embeddings(self, new_num_tokens):
         raise NotImplementedError  # Not implemented yet in the library fr TF 2.0 models
@@ -307,7 +307,7 @@ class TFXxxModel(TFXxxPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXxxModel, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.transformer = TFXxxMainLayer(config, name="transformer")
 
     def call(self, inputs, **kwargs):
@@ -348,7 +348,7 @@ class TFXxxForMaskedLM(TFXxxPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXxxForMaskedLM, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
 
         self.transformer = TFXxxMainLayer(config, name="transformer")
         self.mlm = TFXxxMLMHead(config, self.transformer.embeddings, name="mlm")
@@ -397,7 +397,7 @@ class TFXxxForSequenceClassification(TFXxxPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXxxForSequenceClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.transformer = TFXxxMainLayer(config, name="transformer")
@@ -452,7 +452,7 @@ class TFXxxForTokenClassification(TFXxxPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXxxForTokenClassification, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.transformer = TFXxxMainLayer(config, name="transformer")
@@ -509,7 +509,7 @@ class TFXxxForQuestionAnswering(TFXxxPreTrainedModel):
     """
 
     def __init__(self, config, *inputs, **kwargs):
-        super(TFXxxForQuestionAnswering, self).__init__(config, *inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
         self.num_labels = config.num_labels
 
         self.transformer = TFXxxMainLayer(config, name="transformer")

templates/adding_a_new_model/modeling_xxx.py

@@ -138,7 +138,7 @@ XxxOutput = nn.Module
 
 class XxxLayer(nn.Module):
     def __init__(self, config):
-        super(XxxLayer, self).__init__()
+        super().__init__()
         self.attention = XxxAttention(config)
         self.intermediate = XxxIntermediate(config)
         self.output = XxxOutput(config)
@@ -298,7 +298,7 @@ class XxxModel(XxxPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XxxModel, self).__init__(config)
+        super().__init__(config)
 
         self.embeddings = XxxEmbeddings(config)
         self.encoder = XxxEncoder(config)
@@ -426,7 +426,7 @@ class XxxForMaskedLM(XxxPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XxxForMaskedLM, self).__init__(config)
+        super().__init__(config)
 
         self.transformer = XxxModel(config)
         self.lm_head = nn.Linear(config.n_embd, config.vocab_size)
@@ -507,7 +507,7 @@ class XxxForSequenceClassification(XxxPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XxxForSequenceClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.transformer = XxxModel(config)
@@ -593,7 +593,7 @@ class XxxForTokenClassification(XxxPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XxxForTokenClassification, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.transformer = XxxModel(config)
@@ -692,7 +692,7 @@ class XxxForQuestionAnswering(XxxPreTrainedModel):
     """
 
     def __init__(self, config):
-        super(XxxForQuestionAnswering, self).__init__(config)
+        super().__init__(config)
         self.num_labels = config.num_labels
 
         self.transformer = XxxModel(config)

templates/adding_a_new_model/tests/test_tokenization_xxx.py

@@ -27,7 +27,7 @@ class XxxTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
     tokenizer_class = XxxTokenizer
 
     def setUp(self):
-        super(XxxTokenizationTest, self).setUp()
+        super().setUp()
 
         vocab_tokens = [
             "[UNK]",

templates/adding_a_new_model/tokenization_xxx.py

@@ -109,7 +109,7 @@ class XxxTokenizer(PreTrainedTokenizer):
                 Whether to lower case the input
                 Only has an effect when do_basic_tokenize=True
         """
-        super(XxxTokenizer, self).__init__(
+        super().__init__(
             unk_token=unk_token,
             sep_token=sep_token,
             pad_token=pad_token,

tests/fixtures/dummy-config.json

@@ -0,0 +1,3 @@
+{
+  "model_type": "roberta"
+}

tests/test_configuration_auto.py

@@ -0,0 +1,54 @@
+# coding=utf-8
+# Copyright 2019-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+from transformers.configuration_auto import CONFIG_MAPPING, AutoConfig
+from transformers.configuration_bert import BertConfig
+from transformers.configuration_roberta import RobertaConfig
+
+from .utils import DUMMY_UNKWOWN_IDENTIFIER
+
+
+SAMPLE_ROBERTA_CONFIG = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/dummy-config.json")
+
+
+class AutoConfigTest(unittest.TestCase):
+    def test_config_from_model_shortcut(self):
+        config = AutoConfig.from_pretrained("bert-base-uncased")
+        self.assertIsInstance(config, BertConfig)
+
+    def test_config_model_type_from_local_file(self):
+        config = AutoConfig.from_pretrained(SAMPLE_ROBERTA_CONFIG)
+        self.assertIsInstance(config, RobertaConfig)
+
+    def test_config_model_type_from_model_identifier(self):
+        config = AutoConfig.from_pretrained(DUMMY_UNKWOWN_IDENTIFIER)
+        self.assertIsInstance(config, RobertaConfig)
+
+    def test_config_for_model_str(self):
+        config = AutoConfig.for_model("roberta")
+        self.assertIsInstance(config, RobertaConfig)
+
+    def test_pattern_matching_fallback(self):
+        """
+        In cases where config.json doesn't include a model_type,
+        perform a few safety checks on the config mapping's order.
+        """
+        # no key string should be included in a later key string (typical failure case)
+        keys = list(CONFIG_MAPPING.keys())
+        for i, key in enumerate(keys):
+            self.assertFalse(any(key in later_key for later_key in keys[i + 1 :]))

tests/test_modeling_auto.py

@@ -19,7 +19,7 @@ import unittest
 
 from transformers import is_torch_available
 
-from .utils import SMALL_MODEL_IDENTIFIER, require_torch, slow
+from .utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, require_torch, slow
 
 
 if is_torch_available():
@@ -30,6 +30,7 @@ if is_torch_available():
         BertModel,
         AutoModelWithLMHead,
         BertForMaskedLM,
+        RobertaForMaskedLM,
         AutoModelForSequenceClassification,
         BertForSequenceClassification,
         AutoModelForQuestionAnswering,
@@ -83,7 +84,7 @@ class AutoModelTest(unittest.TestCase):
             self.assertIsNotNone(model)
             self.assertIsInstance(model, BertForSequenceClassification)
 
-    @slow
+    # @slow
     def test_question_answering_model_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
         for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
@@ -100,3 +101,12 @@ class AutoModelTest(unittest.TestCase):
         logging.basicConfig(level=logging.INFO)
         model = AutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER)
         self.assertIsInstance(model, BertForMaskedLM)
+        self.assertEqual(model.num_parameters(), 14830)
+        self.assertEqual(model.num_parameters(only_trainable=True), 14830)
+
+    def test_from_identifier_from_model_type(self):
+        logging.basicConfig(level=logging.INFO)
+        model = AutoModelWithLMHead.from_pretrained(DUMMY_UNKWOWN_IDENTIFIER)
+        self.assertIsInstance(model, RobertaForMaskedLM)
+        self.assertEqual(model.num_parameters(), 14830)
+        self.assertEqual(model.num_parameters(only_trainable=True), 14830)

tests/test_modeling_common.py

@@ -15,13 +15,11 @@
 
 
 import copy
-import json
 import logging
 import os.path
 import random
 import tempfile
 import unittest
-import uuid
 
 from transformers import is_torch_available
 
@@ -238,11 +236,14 @@ class ModelTesterMixin:
             loaded_model.to(torch_device)
             loaded_model.eval()
 
-            model_params = model.parameters()
-            loaded_model_params = loaded_model.parameters()
+            model_state_dict = model.state_dict()
+            loaded_model_state_dict = loaded_model.state_dict()
+
+            self.assertEqual(set(model_state_dict.keys()), set(loaded_model_state_dict.keys()))
 
             models_equal = True
-            for p1, p2 in zip(model_params, loaded_model_params):
+            for layer_name, p1 in model_state_dict.items():
+                p2 = loaded_model_state_dict[layer_name]
                 if p1.data.ne(p2.data).sum() > 0:
                     models_equal = False
 
@@ -487,6 +488,8 @@ class ModelTesterMixin:
             self.assertEqual(model.config.vocab_size, model_vocab_size + 10)
             # Check that it actually resizes the embeddings matrix
             self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] + 10)
+            # Check that the model can still do a forward pass successfully (every parameter should be resized)
+            model(**inputs_dict)
 
             # Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size
             model_embed = model.resize_token_embeddings(model_vocab_size - 15)
@@ -494,6 +497,11 @@ class ModelTesterMixin:
             # Check that it actually resizes the embeddings matrix
             self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] - 15)
 
+            # Check that the model can still do a forward pass successfully (every parameter should be resized)
+            # Input ids should be clamped to the maximum size of the vocabulary
+            inputs_dict["input_ids"].clamp_(max=model_vocab_size - 15 - 1)
+            model(**inputs_dict)
+
             # Check that adding and removing tokens has not modified the first part of the embedding matrix.
             models_equal = True
             for p1, p2 in zip(cloned_embeddings, model_embed.weight):
@@ -592,39 +600,6 @@ class ModelTesterMixin:
                 model(**inputs_dict)
 
 
-class ConfigTester(object):
-    def __init__(self, parent, config_class=None, **kwargs):
-        self.parent = parent
-        self.config_class = config_class
-        self.inputs_dict = kwargs
-
-    def create_and_test_config_common_properties(self):
-        config = self.config_class(**self.inputs_dict)
-        self.parent.assertTrue(hasattr(config, "vocab_size"))
-        self.parent.assertTrue(hasattr(config, "hidden_size"))
-        self.parent.assertTrue(hasattr(config, "num_attention_heads"))
-        self.parent.assertTrue(hasattr(config, "num_hidden_layers"))
-
-    def create_and_test_config_to_json_string(self):
-        config = self.config_class(**self.inputs_dict)
-        obj = json.loads(config.to_json_string())
-        for key, value in self.inputs_dict.items():
-            self.parent.assertEqual(obj[key], value)
-
-    def create_and_test_config_to_json_file(self):
-        config_first = self.config_class(**self.inputs_dict)
-        json_file_path = os.path.join(os.getcwd(), "config_" + str(uuid.uuid4()) + ".json")
-        config_first.to_json_file(json_file_path)
-        config_second = self.config_class.from_json_file(json_file_path)
-        os.remove(json_file_path)
-        self.parent.assertEqual(config_second.to_dict(), config_first.to_dict())
-
-    def run_common_tests(self):
-        self.create_and_test_config_common_properties()
-        self.create_and_test_config_to_json_string()
-        self.create_and_test_config_to_json_file()
-
-
 global_rng = random.Random()
 
 

tests/test_modeling_tf_auto.py

@@ -19,7 +19,7 @@ import unittest
 
 from transformers import is_tf_available
 
-from .utils import SMALL_MODEL_IDENTIFIER, require_tf, slow
+from .utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, require_tf, slow
 
 
 if is_tf_available():
@@ -30,6 +30,7 @@ if is_tf_available():
         TFBertModel,
         TFAutoModelWithLMHead,
         TFBertForMaskedLM,
+        TFRobertaForMaskedLM,
         TFAutoModelForSequenceClassification,
         TFBertForSequenceClassification,
         TFAutoModelForQuestionAnswering,
@@ -99,3 +100,12 @@ class TFAutoModelTest(unittest.TestCase):
         logging.basicConfig(level=logging.INFO)
         model = TFAutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER)
         self.assertIsInstance(model, TFBertForMaskedLM)
+        self.assertEqual(model.num_parameters(), 14830)
+        self.assertEqual(model.num_parameters(only_trainable=True), 14830)
+
+    def test_from_identifier_from_model_type(self):
+        logging.basicConfig(level=logging.INFO)
+        model = TFAutoModelWithLMHead.from_pretrained(DUMMY_UNKWOWN_IDENTIFIER)
+        self.assertIsInstance(model, TFRobertaForMaskedLM)
+        self.assertEqual(model.num_parameters(), 14830)
+        self.assertEqual(model.num_parameters(only_trainable=True), 14830)

tests/test_pipelines.py

@@ -21,16 +21,16 @@ TF_QA_FINETUNED_MODELS = {
 TF_NER_FINETUNED_MODELS = {
     (
         "bert-base-cased",
-        "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-tf_model.h5",
-        "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json",
+        "dbmdz/bert-large-cased-finetuned-conll03-english",
+        "dbmdz/bert-large-cased-finetuned-conll03-english",
     )
 }
 
 NER_FINETUNED_MODELS = {
     (
         "bert-base-cased",
-        "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-pytorch_model.bin",
-        "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json",
+        "dbmdz/bert-large-cased-finetuned-conll03-english",
+        "dbmdz/bert-large-cased-finetuned-conll03-english",
     )
 }
 
@@ -49,16 +49,16 @@ TF_FEATURE_EXTRACT_FINETUNED_MODELS = {
 TF_TEXT_CLASSIF_FINETUNED_MODELS = {
     (
         "bert-base-uncased",
-        "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-tf_model.h5",
-        "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json",
+        "distilbert-base-uncased-finetuned-sst-2-english",
+        "distilbert-base-uncased-finetuned-sst-2-english",
     )
 }
 
 TEXT_CLASSIF_FINETUNED_MODELS = {
     (
         "bert-base-uncased",
-        "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin",
-        "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json",
+        "distilbert-base-uncased-finetuned-sst-2-english",
+        "distilbert-base-uncased-finetuned-sst-2-english",
     )
 }
 

tests/test_tokenization_albert.py

@@ -30,7 +30,7 @@ class AlbertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
     tokenizer_class = AlbertTokenizer
 
     def setUp(self):
-        super(AlbertTokenizationTest, self).setUp()
+        super().setUp()
 
         # We have a SentencePiece fixture for testing
         tokenizer = AlbertTokenizer(SAMPLE_VOCAB)

tests/test_tokenization_auto.py

@@ -23,22 +23,23 @@ from transformers import (
     AutoTokenizer,
     BertTokenizer,
     GPT2Tokenizer,
+    RobertaTokenizer,
 )
 
-from .utils import SMALL_MODEL_IDENTIFIER, slow
+from .utils import DUMMY_UNKWOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, slow  # noqa: F401
 
 
 class AutoTokenizerTest(unittest.TestCase):
-    @slow
+    # @slow
     def test_tokenizer_from_pretrained(self):
         logging.basicConfig(level=logging.INFO)
-        for model_name in list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]:
+        for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
             tokenizer = AutoTokenizer.from_pretrained(model_name)
             self.assertIsNotNone(tokenizer)
             self.assertIsInstance(tokenizer, BertTokenizer)
             self.assertGreater(len(tokenizer), 0)
 
-        for model_name in list(GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]:
+        for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
             tokenizer = AutoTokenizer.from_pretrained(model_name)
             self.assertIsNotNone(tokenizer)
             self.assertIsInstance(tokenizer, GPT2Tokenizer)
@@ -49,3 +50,23 @@ class AutoTokenizerTest(unittest.TestCase):
         tokenizer = AutoTokenizer.from_pretrained(SMALL_MODEL_IDENTIFIER)
         self.assertIsInstance(tokenizer, BertTokenizer)
         self.assertEqual(len(tokenizer), 12)
+
+    def test_tokenizer_from_model_type(self):
+        logging.basicConfig(level=logging.INFO)
+        tokenizer = AutoTokenizer.from_pretrained(DUMMY_UNKWOWN_IDENTIFIER)
+        self.assertIsInstance(tokenizer, RobertaTokenizer)
+        self.assertEqual(len(tokenizer), 20)
+
+    def test_tokenizer_identifier_with_correct_config(self):
+        logging.basicConfig(level=logging.INFO)
+        for tokenizer_class in [BertTokenizer, AutoTokenizer]:
+            tokenizer = tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased")
+            self.assertIsInstance(tokenizer, BertTokenizer)
+            self.assertEqual(tokenizer.basic_tokenizer.do_lower_case, False)
+            self.assertEqual(tokenizer.max_len, 512)
+
+    def test_tokenizer_identifier_non_existent(self):
+        logging.basicConfig(level=logging.INFO)
+        for tokenizer_class in [BertTokenizer, AutoTokenizer]:
+            with self.assertRaises(EnvironmentError):
+                _ = tokenizer_class.from_pretrained("julien-c/herlolip-not-exists")

tests/test_tokenization_bert.py

@@ -38,7 +38,7 @@ class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
     test_rust_tokenizer = True
 
     def setUp(self):
-        super(BertTokenizationTest, self).setUp()
+        super().setUp()
 
         vocab_tokens = [
             "[UNK]",
@@ -119,6 +119,13 @@ class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "), ["HeLLo", "!", "how", "Are", "yoU", "?"]
         )
 
+    def test_basic_tokenizer_respects_never_split_tokens(self):
+        tokenizer = BasicTokenizer(do_lower_case=False, never_split=["[UNK]"])
+
+        self.assertListEqual(
+            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"), ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]
+        )
+
     def test_wordpiece_tokenizer(self):
         vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]