using black -t py35

examples/contrib/run_camembert.py

@@ -30,7 +30,7 @@ def fill_mask(masked_input, model, tokenizer, topk=5):
             )
         else:
             topk_filled_outputs.append(
-                (masked_input.replace(masked_token, predicted_token), values[index].item(), predicted_token,)
+                (masked_input.replace(masked_token, predicted_token), values[index].item(), predicted_token)
             )
     return topk_filled_outputs
 

examples/contrib/run_openai_gpt.py

@@ -83,7 +83,7 @@ def pre_process_datasets(encoded_datasets, input_len, cap_length, start_token, d
         mc_token_ids = np.zeros((n_batch, 2), dtype=np.int64)
         lm_labels = np.full((n_batch, 2, input_len), fill_value=-100, dtype=np.int64)
         mc_labels = np.zeros((n_batch,), dtype=np.int64)
-        for i, (story, cont1, cont2, mc_label), in enumerate(dataset):
+        for i, (story, cont1, cont2, mc_label) in enumerate(dataset):
             with_cont1 = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
             with_cont2 = [start_token] + story[:cap_length] + [delimiter_token] + cont2[:cap_length] + [clf_token]
             input_ids[i, 0, : len(with_cont1)] = with_cont1

examples/contrib/run_swag.py

@@ -51,9 +51,7 @@ logger = logging.getLogger(__name__)
 
 ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in [BertConfig]), ())
 
-MODEL_CLASSES = {
+MODEL_CLASSES = {"bert": (BertConfig, BertForMultipleChoice, BertTokenizer)}
-    "bert": (BertConfig, BertForMultipleChoice, BertTokenizer),
-}
 
 
 class SwagExample(object):
@@ -63,12 +61,7 @@ class SwagExample(object):
         self.swag_id = swag_id
         self.context_sentence = context_sentence
         self.start_ending = start_ending
-        self.endings = [
+        self.endings = [ending_0, ending_1, ending_2, ending_3]
-            ending_0,
-            ending_1,
-            ending_2,
-            ending_3,
-        ]
         self.label = label
 
     def __str__(self):

examples/distillation/utils.py

@@ -117,9 +117,7 @@ def init_gpu_params(params):
     # initialize multi-GPU
     if params.multi_gpu:
         logger.info("Initializing PyTorch distributed")
-        torch.distributed.init_process_group(
+        torch.distributed.init_process_group(init_method="env://", backend="nccl")
-            init_method="env://", backend="nccl",
-        )
 
 
 def set_seed(args):

examples/mm-imdb/utils_mmimdb.py

@@ -138,6 +138,6 @@ def get_image_transforms():
             transforms.Resize(256),
             transforms.CenterCrop(224),
             transforms.ToTensor(),
-            transforms.Normalize(mean=[0.46777044, 0.44531429, 0.40661017], std=[0.12221994, 0.12145835, 0.14380469],),
+            transforms.Normalize(mean=[0.46777044, 0.44531429, 0.40661017], std=[0.12221994, 0.12145835, 0.14380469]),
         ]
     )

examples/pplm/run_pplm.py

@@ -718,7 +718,7 @@ if __name__ == "__main__":
     parser.add_argument("--cond_text", type=str, default="The lake", help="Prefix texts to condition on")
     parser.add_argument("--uncond", action="store_true", help="Generate from end-of-text as prefix")
     parser.add_argument(
-        "--num_samples", type=int, default=1, help="Number of samples to generate from the modified latents",
+        "--num_samples", type=int, default=1, help="Number of samples to generate from the modified latents"
     )
     parser.add_argument(
         "--bag_of_words",
@@ -741,9 +741,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--discrim_meta", type=str, default=None, help="Meta information for the generic discriminator"
     )
-    parser.add_argument(
+    parser.add_argument("--class_label", type=int, default=-1, help="Class label used for the discriminator")
-        "--class_label", type=int, default=-1, help="Class label used for the discriminator",
-    )
     parser.add_argument("--length", type=int, default=100)
     parser.add_argument("--stepsize", type=float, default=0.02)
     parser.add_argument("--temperature", type=float, default=1.0)
@@ -757,9 +755,7 @@ if __name__ == "__main__":
         default=0,
         help="Length of past which is being optimized; " "0 corresponds to infinite window length",
     )
-    parser.add_argument(
+    parser.add_argument("--horizon_length", type=int, default=1, help="Length of future to optimize over")
-        "--horizon_length", type=int, default=1, help="Length of future to optimize over",
-    )
     parser.add_argument("--decay", action="store_true", help="whether to decay or not")
     parser.add_argument("--gamma", type=float, default=1.5)
     parser.add_argument("--gm_scale", type=float, default=0.9)

examples/pplm/run_pplm_discrim_train.py

@@ -242,7 +242,7 @@ def train_discriminator(
 
         text = torchtext_data.Field()
         label = torchtext_data.Field(sequential=False)
-        train_data, val_data, test_data = datasets.SST.splits(text, label, fine_grained=True, train_subtrees=True,)
+        train_data, val_data, test_data = datasets.SST.splits(text, label, fine_grained=True, train_subtrees=True)
 
         x = []
         y = []

examples/run_generation.py

@@ -41,7 +41,7 @@ from transformers import (
 
 
 logging.basicConfig(
-    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
 )
 logger = logging.getLogger(__name__)
 

examples/summarization/configuration_bertabs.py

@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 BERTABS_FINETUNED_CONFIG_MAP = {
-    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-config.json",
+    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-config.json"
 }
 
 

examples/summarization/convert_bertabs_original_pytorch_checkpoint.py

@@ -164,13 +164,11 @@ def convert_bertabs_checkpoints(path_to_checkpoints, dump_path):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--bertabs_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump.",
+        "--bertabs_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
     )
     parser.add_argument(
-        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model.",
+        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
     )
     args = parser.parse_args()
 
-    convert_bertabs_checkpoints(
+    convert_bertabs_checkpoints(args.bertabs_checkpoint_path, args.pytorch_dump_folder_path)
-        args.bertabs_checkpoint_path, args.pytorch_dump_folder_path,
-    )

examples/summarization/modeling_bertabs.py

@@ -34,7 +34,7 @@ from transformers import BertConfig, BertModel, PreTrainedModel
 MAX_SIZE = 5000
 
 BERTABS_FINETUNED_MODEL_MAP = {
-    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-pytorch_model.bin",
+    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-pytorch_model.bin"
 }
 
 
@@ -106,10 +106,10 @@ class BertAbs(BertAbsPreTrainedModel):
                 p.data.zero_()
 
     def forward(
-        self, encoder_input_ids, decoder_input_ids, token_type_ids, encoder_attention_mask, decoder_attention_mask,
+        self, encoder_input_ids, decoder_input_ids, token_type_ids, encoder_attention_mask, decoder_attention_mask
     ):
         encoder_output = self.bert(
-            input_ids=encoder_input_ids, token_type_ids=token_type_ids, attention_mask=encoder_attention_mask,
+            input_ids=encoder_input_ids, token_type_ids=token_type_ids, attention_mask=encoder_attention_mask
         )
         encoder_hidden_states = encoder_output[0]
         dec_state = self.decoder.init_decoder_state(encoder_input_ids, encoder_hidden_states)
@@ -308,7 +308,7 @@ class TransformerDecoderLayer(nn.Module):
         self.register_buffer("mask", mask)
 
     def forward(
-        self, inputs, memory_bank, src_pad_mask, tgt_pad_mask, previous_input=None, layer_cache=None, step=None,
+        self, inputs, memory_bank, src_pad_mask, tgt_pad_mask, previous_input=None, layer_cache=None, step=None
     ):
         """
         Args:
@@ -332,13 +332,13 @@ class TransformerDecoderLayer(nn.Module):
             all_input = torch.cat((previous_input, input_norm), dim=1)
             dec_mask = None
 
-        query = self.self_attn(all_input, all_input, input_norm, mask=dec_mask, layer_cache=layer_cache, type="self",)
+        query = self.self_attn(all_input, all_input, input_norm, mask=dec_mask, layer_cache=layer_cache, type="self")
 
         query = self.drop(query) + inputs
 
         query_norm = self.layer_norm_2(query)
         mid = self.context_attn(
-            memory_bank, memory_bank, query_norm, mask=src_pad_mask, layer_cache=layer_cache, type="context",
+            memory_bank, memory_bank, query_norm, mask=src_pad_mask, layer_cache=layer_cache, type="context"
         )
         output = self.feed_forward(self.drop(mid) + query)
 
@@ -422,9 +422,7 @@ class MultiHeadedAttention(nn.Module):
         if self.use_final_linear:
             self.final_linear = nn.Linear(model_dim, model_dim)
 
-    def forward(
+    def forward(self, key, value, query, mask=None, layer_cache=None, type=None, predefined_graph_1=None):
-        self, key, value, query, mask=None, layer_cache=None, type=None, predefined_graph_1=None,
-    ):
         """
         Compute the context vector and the attention vectors.
 
@@ -458,11 +456,7 @@ class MultiHeadedAttention(nn.Module):
         # 1) Project key, value, and query.
         if layer_cache is not None:
             if type == "self":
-                query, key, value = (
+                query, key, value = (self.linear_query(query), self.linear_keys(query), self.linear_values(query))
-                    self.linear_query(query),
-                    self.linear_keys(query),
-                    self.linear_values(query),
-                )
 
                 key = shape(key)
                 value = shape(value)
@@ -483,10 +477,7 @@ class MultiHeadedAttention(nn.Module):
                         key = shape(key)
                         value = shape(value)
                     else:
-                        key, value = (
+                        key, value = (layer_cache["memory_keys"], layer_cache["memory_values"])
-                            layer_cache["memory_keys"],
-                            layer_cache["memory_values"],
-                        )
                     layer_cache["memory_keys"] = key
                     layer_cache["memory_values"] = value
                 else:
@@ -999,12 +990,8 @@ class BertSumOptimizer(object):
         self.warmup_steps = warmup_steps
 
         self.optimizers = {
-            "encoder": torch.optim.Adam(
+            "encoder": torch.optim.Adam(model.encoder.parameters(), lr=lr["encoder"], betas=(beta_1, beta_2), eps=eps),
-                model.encoder.parameters(), lr=lr["encoder"], betas=(beta_1, beta_2), eps=eps,
+            "decoder": torch.optim.Adam(model.decoder.parameters(), lr=lr["decoder"], betas=(beta_1, beta_2), eps=eps),
-            ),
-            "decoder": torch.optim.Adam(
-                model.decoder.parameters(), lr=lr["decoder"], betas=(beta_1, beta_2), eps=eps,
-            ),
         }
 
         self._step = 0

examples/summarization/run_summarization.py

@@ -188,7 +188,7 @@ def build_data_iterator(args, tokenizer):
     def collate_fn(data):
         return collate(data, tokenizer, block_size=512, device=args.device)
 
-    iterator = DataLoader(dataset, sampler=sampler, batch_size=args.batch_size, collate_fn=collate_fn,)
+    iterator = DataLoader(dataset, sampler=sampler, batch_size=args.batch_size, collate_fn=collate_fn)
 
     return iterator
 
@@ -265,24 +265,14 @@ def main():
         help="Compute the ROUGE metrics during evaluation. Only available for the CNN/DailyMail dataset.",
     )
     # EVALUATION options
-    parser.add_argument(
+    parser.add_argument("--no_cuda", default=False, type=bool, help="Whether to force the execution on CPU.")
-        "--no_cuda", default=False, type=bool, help="Whether to force the execution on CPU.",
+    parser.add_argument("--batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.")
-    )
-    parser.add_argument(
-        "--batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.",
-    )
     # BEAM SEARCH arguments
+    parser.add_argument("--min_length", default=50, type=int, help="Minimum number of tokens for the summaries.")
+    parser.add_argument("--max_length", default=200, type=int, help="Maixmum number of tokens for the summaries.")
+    parser.add_argument("--beam_size", default=5, type=int, help="The number of beams to start with for each example.")
     parser.add_argument(
-        "--min_length", default=50, type=int, help="Minimum number of tokens for the summaries.",
+        "--alpha", default=0.95, type=float, help="The value of alpha for the length penalty in the beam search."
-    )
-    parser.add_argument(
-        "--max_length", default=200, type=int, help="Maixmum number of tokens for the summaries.",
-    )
-    parser.add_argument(
-        "--beam_size", default=5, type=int, help="The number of beams to start with for each example.",
-    )
-    parser.add_argument(
-        "--alpha", default=0.95, type=float, help="The value of alpha for the length penalty in the beam search.",
     )
     parser.add_argument(
         "--block_trigram",

examples/utils_multiple_choice.py

@@ -320,7 +320,7 @@ def convert_examples_to_features(
             else:
                 text_b = example.question + " " + ending
 
-            inputs = tokenizer.encode_plus(text_a, text_b, add_special_tokens=True, max_length=max_length,)
+            inputs = tokenizer.encode_plus(text_a, text_b, add_special_tokens=True, max_length=max_length)
             if "num_truncated_tokens" in inputs and inputs["num_truncated_tokens"] > 0:
                 logger.info(
                     "Attention! you are cropping tokens (swag task is ok). "
@@ -362,7 +362,7 @@ def convert_examples_to_features(
                 logger.info("token_type_ids: {}".format(" ".join(map(str, token_type_ids))))
                 logger.info("label: {}".format(label))
 
-        features.append(InputFeatures(example_id=example.example_id, choices_features=choices_features, label=label,))
+        features.append(InputFeatures(example_id=example.example_id, choices_features=choices_features, label=label))
 
     return features
 

setup.py

@@ -62,6 +62,7 @@ extras["mecab"] = ["mecab-python3"]
 extras["sklearn"] = ["scikit-learn"]
 extras["tf"] = ["tensorflow"]
 extras["torch"] = ["torch"]
+extras["fast"] = ["tokenizers == 0.0.10"]
 
 extras["serving"] = ["pydantic", "uvicorn", "fastapi"]
 extras["all"] = extras["serving"] + ["tensorflow", "torch"]
@@ -86,7 +87,6 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.0.10",
         # accessing files from S3 directly
         "boto3",
         # filesystem locks e.g. to prevent parallel downloads

src/transformers/__init__.py

@@ -70,6 +70,7 @@ from .file_utils import (
     add_end_docstrings,
     add_start_docstrings,
     cached_path,
+    is_fast_tokenizers_available,
     is_tf_available,
     is_torch_available,
 )

src/transformers/configuration_camembert.py

@@ -24,7 +24,7 @@ from .configuration_roberta import RobertaConfig
 logger = logging.getLogger(__name__)
 
 CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json",
+    "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json"
 }
 
 

src/transformers/configuration_transfo_xl.py

@@ -24,7 +24,7 @@ from .configuration_utils import PretrainedConfig
 logger = logging.getLogger(__name__)
 
 TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-config.json",
+    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-config.json"
 }
 
 

src/transformers/data/metrics/__init__.py

@@ -35,20 +35,12 @@ if _has_sklearn:
     def acc_and_f1(preds, labels):
         acc = simple_accuracy(preds, labels)
         f1 = f1_score(y_true=labels, y_pred=preds)
-        return {
+        return {"acc": acc, "f1": f1, "acc_and_f1": (acc + f1) / 2}
-            "acc": acc,
-            "f1": f1,
-            "acc_and_f1": (acc + f1) / 2,
-        }
 
     def pearson_and_spearman(preds, labels):
         pearson_corr = pearsonr(preds, labels)[0]
         spearman_corr = spearmanr(preds, labels)[0]
-        return {
+        return {"pearson": pearson_corr, "spearmanr": spearman_corr, "corr": (pearson_corr + spearman_corr) / 2}
-            "pearson": pearson_corr,
-            "spearmanr": spearman_corr,
-            "corr": (pearson_corr + spearman_corr) / 2,
-        }
 
     def glue_compute_metrics(task_name, preds, labels):
         assert len(preds) == len(labels)

src/transformers/data/processors/glue.py

@@ -86,7 +86,7 @@ def glue_convert_examples_to_features(
             example = processor.get_example_from_tensor_dict(example)
             example = processor.tfds_map(example)
 
-        inputs = tokenizer.encode_plus(example.text_a, example.text_b, add_special_tokens=True, max_length=max_length,)
+        inputs = tokenizer.encode_plus(example.text_a, example.text_b, add_special_tokens=True, max_length=max_length)
         input_ids, token_type_ids = inputs["input_ids"], inputs["token_type_ids"]
 
         # The mask has 1 for real tokens and 0 for padding tokens. Only real

src/transformers/data/processors/utils.py

@@ -244,7 +244,7 @@ class SingleSentenceClassificationProcessor(DataProcessor):
                 logger.info("Tokenizing example %d", ex_index)
 
             input_ids = tokenizer.encode(
-                example.text_a, add_special_tokens=True, max_length=min(max_length, tokenizer.max_len),
+                example.text_a, add_special_tokens=True, max_length=min(max_length, tokenizer.max_len)
             )
             all_input_ids.append(input_ids)
 

src/transformers/data/processors/xnli.py

@@ -72,14 +72,8 @@ class XnliProcessor(DataProcessor):
         return ["contradiction", "entailment", "neutral"]
 
 
-xnli_processors = {
+xnli_processors = {"xnli": XnliProcessor}
-    "xnli": XnliProcessor,
-}
 
-xnli_output_modes = {
+xnli_output_modes = {"xnli": "classification"}
-    "xnli": "classification",
-}
 
-xnli_tasks_num_labels = {
+xnli_tasks_num_labels = {"xnli": 3}
-    "xnli": 3,
-}

src/transformers/file_utils.py

@@ -55,6 +55,20 @@ try:
 except (ImportError, AssertionError):
     _tf_available = False  # pylint: disable=invalid-name
 
+try:
+    os.environ.setdefault("USE_FAST_TOKENIZERS", "YES")
+    if os.environ["USE_FAST_TOKENIZERS"].upper() in ("1", "ON", "YES"):
+        import tokenizers
+
+        _fast_tokenizers_available = True  # pylint: disable=invalid-name
+        logger.info("Fast Tokenizers version {} available.".format(tokenizers.__version__))
+    else:
+        logger.info("USE_FAST_TOKENIZERS override through env variable, disabling fast Tokenizers")
+        _fast_tokenizers_available = False
+except ImportError:
+    _fast_tokenizers_available = False  # pylint: disable=invalid-name
+
+
 try:
     from torch.hub import _get_torch_home
 
@@ -97,6 +111,10 @@ def is_torch_available():
     return _torch_available
 
 
+def is_fast_tokenizers_available():
+    return _fast_tokenizers_available
+
+
 def is_tf_available():
 
     return _tf_available

src/transformers/modeling_albert.py

@@ -837,7 +837,7 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_bert.py

@@ -810,7 +810,7 @@ class BertModel(BertPreTrainedModel):
         sequence_output = encoder_outputs[0]
         pooled_output = self.pooler(sequence_output)
 
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[
+        outputs = (sequence_output, pooled_output) + encoder_outputs[
             1:
         ]  # add hidden_states and attentions if they are here
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
@@ -895,7 +895,7 @@ class BertForPreTraining(BertPreTrainedModel):
         sequence_output, pooled_output = outputs[:2]
         prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)
 
-        outputs = (prediction_scores, seq_relationship_score,) + outputs[
+        outputs = (prediction_scores, seq_relationship_score) + outputs[
             2:
         ]  # add hidden states and attention if they are here
 
@@ -1440,7 +1440,7 @@ class BertForQuestionAnswering(BertPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_camembert.py

@@ -32,7 +32,7 @@ from .modeling_roberta import (
 logger = logging.getLogger(__name__)
 
 CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin",
+    "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin"
 }
 
 

src/transformers/modeling_distilbert.py

@@ -707,7 +707,7 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel):
         start_logits = start_logits.squeeze(-1)  # (bs, max_query_len)
         end_logits = end_logits.squeeze(-1)  # (bs, max_query_len)
 
-        outputs = (start_logits, end_logits,) + distilbert_output[1:]
+        outputs = (start_logits, end_logits) + distilbert_output[1:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_encoder_decoder.py

@@ -325,7 +325,7 @@ class Model2Model(PreTrainedEncoderDecoder):
             encoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
             decoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
             *args,
-            **kwargs
+            **kwargs,
         )
 
         return model

src/transformers/modeling_mmbt.py

@@ -310,7 +310,7 @@ class MMBTModel(nn.Module):
         sequence_output = encoder_outputs[0]
         pooled_output = self.transformer.pooler(sequence_output)
 
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[
+        outputs = (sequence_output, pooled_output) + encoder_outputs[
             1:
         ]  # add hidden_states and attentions if they are here
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)

src/transformers/modeling_roberta.py

@@ -715,7 +715,7 @@ class RobertaForQuestionAnswering(BertPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_tf_albert.py

@@ -698,7 +698,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel):
         pooled_output = self.pooler(sequence_output[:, 0])
 
         # add hidden_states and attentions if they are here
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]
+        outputs = (sequence_output, pooled_output) + encoder_outputs[1:]
         # sequence_output, pooled_output, (hidden_states), (attentions)
         return outputs
 

src/transformers/modeling_tf_bert.py

@@ -568,7 +568,7 @@ class TFBertMainLayer(tf.keras.layers.Layer):
         sequence_output = encoder_outputs[0]
         pooled_output = self.pooler(sequence_output)
 
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[
+        outputs = (sequence_output, pooled_output) + encoder_outputs[
             1:
         ]  # add hidden_states and attentions if they are here
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
@@ -766,7 +766,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel):
         prediction_scores = self.mlm(sequence_output, training=kwargs.get("training", False))
         seq_relationship_score = self.nsp(pooled_output)
 
-        outputs = (prediction_scores, seq_relationship_score,) + outputs[
+        outputs = (prediction_scores, seq_relationship_score) + outputs[
             2:
         ]  # add hidden states and attention if they are here
 
@@ -1139,6 +1139,6 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
 
         return outputs  # start_logits, end_logits, (hidden_states), (attentions)

src/transformers/modeling_tf_distilbert.py

@@ -822,5 +822,5 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + distilbert_output[1:]
+        outputs = (start_logits, end_logits) + distilbert_output[1:]
         return outputs  # start_logits, end_logits, (hidden_states), (attentions)

src/transformers/modeling_tf_transfo_xl.py

@@ -30,7 +30,7 @@ from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list
 logger = logging.getLogger(__name__)
 
 TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-tf_model.h5",
+    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-tf_model.h5"
 }
 
 

src/transformers/modeling_tf_transfo_xl_utilities.py

@@ -64,7 +64,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer):
                 else:
                     self.out_projs.append(None)
                 weight = self.add_weight(
-                    shape=(self.vocab_size, self.d_embed,),
+                    shape=(self.vocab_size, self.d_embed),
                     initializer="zeros",
                     trainable=True,
                     name="out_layers_._{}_._weight".format(i),
@@ -86,7 +86,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer):
                 )
                 self.out_projs.append(weight)
                 weight = self.add_weight(
-                    shape=(r_idx - l_idx, d_emb_i,),
+                    shape=(r_idx - l_idx, d_emb_i),
                     initializer="zeros",
                     trainable=True,
                     name="out_layers_._{}_._weight".format(i),

src/transformers/modeling_tf_utils.py

@@ -250,7 +250,7 @@ class TFPreTrainedModel(tf.keras.Model):
                 return_unused_kwargs=True,
                 force_download=force_download,
                 resume_download=resume_download,
-                **kwargs
+                **kwargs,
             )
         else:
             model_kwargs = kwargs

src/transformers/modeling_tf_xlm.py

@@ -800,7 +800,7 @@ class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + transformer_outputs[
+        outputs = (start_logits, end_logits) + transformer_outputs[
             1:
         ]  # Keep mems, hidden states, attentions if there are in it
 

src/transformers/modeling_tf_xlnet.py

@@ -1074,7 +1074,7 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + transformer_outputs[
+        outputs = (start_logits, end_logits) + transformer_outputs[
             1:
         ]  # Keep mems, hidden states, attentions if there are in it
 

src/transformers/modeling_transfo_xl.py

@@ -34,7 +34,7 @@ from .modeling_utils import PreTrainedModel
 logger = logging.getLogger(__name__)
 
 TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-pytorch_model.bin",
+    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-pytorch_model.bin"
 }
 
 

src/transformers/modeling_utils.py

@@ -355,7 +355,7 @@ class PreTrainedModel(nn.Module):
                 force_download=force_download,
                 resume_download=resume_download,
                 proxies=proxies,
-                **kwargs
+                **kwargs,
             )
         else:
             model_kwargs = kwargs

src/transformers/modeling_xlm.py

@@ -896,10 +896,7 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (
+        outputs = (start_logits, end_logits)
-            start_logits,
-            end_logits,
-        )
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_xlnet.py

@@ -1505,7 +1505,7 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/pipelines.py

@@ -643,7 +643,7 @@ class QuestionAnsweringPipeline(Pipeline):
             framework=framework,
             args_parser=QuestionAnsweringArgumentHandler(),
             device=device,
-            **kwargs
+            **kwargs,
         )
 
     @staticmethod

src/transformers/tokenization_albert.py

@@ -87,7 +87,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
 
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens

src/transformers/tokenization_bert.py

@@ -20,11 +20,15 @@ import logging
 import os
 import unicodedata
 
-import tokenizers as tk
+from .file_utils import is_fast_tokenizers_available
-
 from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
 
 
+if is_fast_tokenizers_available():
+    import tokenizers as tk
+else:
+    tk = None
+
 logger = logging.getLogger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
@@ -169,7 +173,7 @@ class BertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens
@@ -560,10 +564,18 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
 
-        self._tokenizer = tk.Tokenizer(tk.models.WordPiece.from_files(vocab_file, unk_token=unk_token))
+        if is_fast_tokenizers_available():
+            self._tokenizer = tk.Tokenizer(tk.models.WordPiece.from_files(vocab_file, unk_token=unk_token))
+        else:
+            logger.error(
+                "Using fast Tokenizers requires the `tokenizers` library. "
+                "Please install it with `pip install tokenizers`."
+            )
+            raise ImportError()
+
         self._update_special_tokens()
         self._tokenizer.with_pre_tokenizer(
             tk.pre_tokenizers.BertPreTokenizer.new(

src/transformers/tokenization_bert_japanese.py

@@ -113,7 +113,7 @@ class BertJapaneseTokenizer(BertTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens

src/transformers/tokenization_camembert.py

@@ -32,13 +32,11 @@ VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.bpe.model"}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {
-        "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model",
+        "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model"
     }
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"camembert-base": None}
-    "camembert-base": None,
-}
 
 
 class CamembertTokenizer(PreTrainedTokenizer):
@@ -76,7 +74,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens

src/transformers/tokenization_ctrl.py

@@ -26,19 +26,14 @@ from .tokenization_utils import PreTrainedTokenizer
 
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
     "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"ctrl": 256}
-    "ctrl": 256,
-}
 
 CONTROL_CODES = {
     "Pregnancy": 168629,

src/transformers/tokenization_gpt2.py

@@ -21,17 +21,19 @@ import os
 from functools import lru_cache
 
 import regex as re
-import tokenizers as tk
 
+from .file_utils import is_fast_tokenizers_available
 from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
 
 
+if is_fast_tokenizers_available():
+    import tokenizers as tk
+else:
+    tk = None
+
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {
@@ -272,7 +274,15 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
             bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs
         )
 
-        self._tokenizer = tk.Tokenizer(tk.models.BPE.from_files(vocab_file, merges_file))
+        if is_fast_tokenizers_available():
+            self._tokenizer = tk.Tokenizer(tk.models.BPE.from_files(vocab_file, merges_file))
+        else:
+            logger.error(
+                "Using fast Tokenizers requires the `tokenizers` library. "
+                "Please install it with `pip install tokenizers`."
+            )
+            raise ImportError()
+
         self._update_special_tokens()
         self._tokenizer.with_pre_tokenizer(tk.pre_tokenizers.ByteLevel.new(add_prefix_space=add_prefix_space))
         self._tokenizer.with_decoder(tk.decoders.ByteLevel.new())

src/transformers/tokenization_openai.py

@@ -26,19 +26,14 @@ from .tokenization_utils import PreTrainedTokenizer
 
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-vocab.json"},
     "merges_file": {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt"},
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"openai-gpt": 512}
-    "openai-gpt": 512,
-}
 
 
 def get_pairs(word):

src/transformers/tokenization_roberta.py

@@ -22,10 +22,7 @@ from .tokenization_gpt2 import GPT2Tokenizer
 
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {
@@ -95,7 +92,7 @@ class RobertaTokenizer(GPT2Tokenizer):
             cls_token=cls_token,
             pad_token=pad_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens

src/transformers/tokenization_t5.py

@@ -96,7 +96,7 @@ class T5Tokenizer(PreTrainedTokenizer):
             unk_token=unk_token,
             pad_token=pad_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs
+            **kwargs,
         )
 
         try:

src/transformers/tokenization_transfo_xl.py

@@ -40,16 +40,14 @@ VOCAB_FILES_NAMES = {"pretrained_vocab_file": "vocab.bin", "vocab_file": "vocab.
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "pretrained_vocab_file": {
-        "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-vocab.bin",
+        "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-vocab.bin"
     }
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"transfo-xl-wt103": None}
-    "transfo-xl-wt103": None,
-}
 
 PRETRAINED_CORPUS_ARCHIVE_MAP = {
-    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-corpus.bin",
+    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-corpus.bin"
 }
 CORPUS_NAME = "corpus.bin"
 

src/transformers/tokenization_utils.py

@@ -817,7 +817,7 @@ class PreTrainedTokenizer(object):
             truncation_strategy=truncation_strategy,
             pad_to_max_length=pad_to_max_length,
             return_tensors=return_tensors,
-            **kwargs
+            **kwargs,
         )
 
         return encoded_inputs["input_ids"]
@@ -1495,9 +1495,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer):
         return_overflowing_tokens=False,
         return_special_tokens_mask=False,
     ):
-        encoding_dict = {
+        encoding_dict = {"input_ids": encoding.ids}
-            "input_ids": encoding.ids,
-        }
         if return_token_type_ids:
             encoding_dict["token_type_ids"] = encoding.type_ids
         if return_attention_mask:

src/transformers/tokenization_xlm.py

@@ -29,10 +29,7 @@ from .tokenization_utils import PreTrainedTokenizer
 
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {
@@ -586,7 +583,7 @@ class XLMTokenizer(PreTrainedTokenizer):
             cls_token=cls_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs
+            **kwargs,
         )
 
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens

src/transformers/tokenization_xlm_roberta.py

@@ -83,7 +83,7 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
             cls_token=cls_token,
             pad_token=pad_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens

src/transformers/tokenization_xlnet.py

@@ -34,10 +34,7 @@ PRETRAINED_VOCAB_FILES_MAP = {
     }
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"xlnet-base-cased": None, "xlnet-large-cased": None}
-    "xlnet-base-cased": None,
-    "xlnet-large-cased": None,
-}
 
 SPIECE_UNDERLINE = "▁"
 
@@ -86,7 +83,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
             cls_token=cls_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs
+            **kwargs,
         )
 
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens

templates/adding_a_new_model/modeling_tf_xxx.py

@@ -527,6 +527,6 @@ class TFXxxForQuestionAnswering(TFXxxPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
 
         return outputs  # start_logits, end_logits, (hidden_states), (attentions)

templates/adding_a_new_model/modeling_xxx.py

@@ -728,7 +728,7 @@ class XxxForQuestionAnswering(XxxPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

templates/adding_a_new_model/tests/test_modeling_tf_xxx.py

@@ -144,10 +144,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output.numpy(), "pooled_output": pooled_output.numpy()}
-                "sequence_output": sequence_output.numpy(),
-                "pooled_output": pooled_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -159,9 +156,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForMaskedLM(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (prediction_scores,) = model(inputs)
-            result = {
+            result = {"prediction_scores": prediction_scores.numpy()}
-                "prediction_scores": prediction_scores.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -173,9 +168,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForSequenceClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_labels])
 
         def create_and_check_xxx_for_token_classification(
@@ -185,9 +178,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForTokenClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, self.num_labels]
             )
@@ -198,10 +189,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForQuestionAnswering(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             start_logits, end_logits = model(inputs)
-            result = {
+            result = {"start_logits": start_logits.numpy(), "end_logits": end_logits.numpy()}
-                "start_logits": start_logits.numpy(),
-                "end_logits": end_logits.numpy(),
-            }
             self.parent.assertListEqual(list(result["start_logits"].shape), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].shape), [self.batch_size, self.seq_length])
 

templates/adding_a_new_model/tests/test_modeling_xxx.py

@@ -141,10 +141,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
             sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -159,10 +156,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
             loss, prediction_scores = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels
             )
-            result = {
+            result = {"loss": loss, "prediction_scores": prediction_scores}
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -181,11 +175,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
                 start_positions=sequence_labels,
                 end_positions=sequence_labels,
             )
-            result = {
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -200,10 +190,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels
             )
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 
@@ -217,10 +204,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels
             )
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
             self.parent.assertListEqual(
                 list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]
             )

templates/adding_a_new_model/tokenization_xxx.py

@@ -48,10 +48,7 @@ PRETRAINED_VOCAB_FILES_MAP = {
 ####################################################
 # Mapping from model shortcut names to max length of inputs
 ####################################################
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"xxx-base-uncased": 512, "xxx-large-uncased": 512}
-    "xxx-base-uncased": 512,
-    "xxx-large-uncased": 512,
-}
 
 ####################################################
 # Mapping from model shortcut names to a dictionary of additional
@@ -115,7 +112,7 @@ class XxxTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens

tests/test_modeling_albert.py

@@ -141,10 +141,7 @@ class AlbertModelTest(ModelTesterMixin, unittest.TestCase):
             sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -159,10 +156,7 @@ class AlbertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, prediction_scores = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels
             )
-            result = {
+            result = {"loss": loss, "prediction_scores": prediction_scores}
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -181,11 +175,7 @@ class AlbertModelTest(ModelTesterMixin, unittest.TestCase):
                 start_positions=sequence_labels,
                 end_positions=sequence_labels,
             )
-            result = {
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -200,10 +190,7 @@ class AlbertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels
             )
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 

tests/test_modeling_bert.py

@@ -180,10 +180,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -219,10 +216,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             )
             sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -237,10 +231,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, prediction_scores = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels
             )
-            result = {
+            result = {"loss": loss, "prediction_scores": prediction_scores}
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -276,10 +267,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
                 masked_lm_labels=token_labels,
                 encoder_hidden_states=encoder_hidden_states,
             )
-            result = {
+            result = {"loss": loss, "prediction_scores": prediction_scores}
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -297,10 +285,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
                 token_type_ids=token_type_ids,
                 next_sentence_label=sequence_labels,
             )
-            result = {
+            result = {"loss": loss, "seq_relationship_score": seq_relationship_score}
-                "loss": loss,
-                "seq_relationship_score": seq_relationship_score,
-            }
             self.parent.assertListEqual(list(result["seq_relationship_score"].size()), [self.batch_size, 2])
             self.check_loss_output(result)
 
@@ -341,11 +326,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
                 start_positions=sequence_labels,
                 end_positions=sequence_labels,
             )
-            result = {
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -360,10 +341,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels
             )
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 
@@ -377,10 +355,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels
             )
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
             self.parent.assertListEqual(
                 list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]
             )
@@ -402,10 +377,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
                 token_type_ids=multiple_choice_token_type_ids,
                 labels=choice_labels,
             )
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_choices])
             self.check_loss_output(result)
 

tests/test_modeling_ctrl.py

@@ -150,10 +150,7 @@ class CTRLModelTest(ModelTesterMixin, unittest.TestCase):
             model(input_ids, token_type_ids=token_type_ids)
             sequence_output, presents = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output, "presents": presents}
-                "sequence_output": sequence_output,
-                "presents": presents,
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )

tests/test_modeling_distilbert.py

@@ -138,9 +138,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             (sequence_output,) = model(input_ids, input_mask)
             (sequence_output,) = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output}
-                "sequence_output": sequence_output,
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -152,10 +150,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             model.to(torch_device)
             model.eval()
             loss, prediction_scores = model(input_ids, attention_mask=input_mask, masked_lm_labels=token_labels)
-            result = {
+            result = {"loss": loss, "prediction_scores": prediction_scores}
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -170,11 +165,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, start_logits, end_logits = model(
                 input_ids, attention_mask=input_mask, start_positions=sequence_labels, end_positions=sequence_labels
             )
-            result = {
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -187,10 +178,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             model.to(torch_device)
             model.eval()
             loss, logits = model(input_ids, attention_mask=input_mask, labels=sequence_labels)
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 
@@ -203,10 +191,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             model.eval()
 
             loss, logits = model(input_ids, attention_mask=input_mask, labels=token_labels)
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
             self.parent.assertListEqual(
                 list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]
             )

tests/test_modeling_gpt2.py

@@ -153,10 +153,7 @@ class GPT2ModelTest(ModelTesterMixin, unittest.TestCase):
             model(input_ids, token_type_ids=token_type_ids)
             sequence_output, presents = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output, "presents": presents}
-                "sequence_output": sequence_output,
-                "presents": presents,
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )

tests/test_modeling_roberta.py

@@ -138,10 +138,7 @@ class RobertaModelTest(ModelTesterMixin, unittest.TestCase):
             sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -156,10 +153,7 @@ class RobertaModelTest(ModelTesterMixin, unittest.TestCase):
             loss, prediction_scores = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels
             )
-            result = {
+            result = {"loss": loss, "prediction_scores": prediction_scores}
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -175,10 +169,7 @@ class RobertaModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels
             )
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
             self.parent.assertListEqual(
                 list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]
             )

tests/test_modeling_t5.py

@@ -136,10 +136,7 @@ class T5ModelTest(ModelTesterMixin, unittest.TestCase):
                 encoder_input_ids=encoder_input_ids, decoder_input_ids=decoder_input_ids
             )
 
-            result = {
+            result = {"encoder_output": encoder_output, "decoder_output": decoder_output}
-                "encoder_output": encoder_output,
-                "decoder_output": decoder_output,
-            }
             self.parent.assertListEqual(
                 list(result["encoder_output"].size()), [self.batch_size, self.encoder_seq_length, self.hidden_size]
             )
@@ -165,10 +162,7 @@ class T5ModelTest(ModelTesterMixin, unittest.TestCase):
                 decoder_lm_labels=decoder_lm_labels,
             )
             loss, prediction_scores = outputs[0], outputs[1]
-            result = {
+            result = {"loss": loss, "prediction_scores": prediction_scores}
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.decoder_seq_length, self.vocab_size]
             )

tests/test_modeling_tf_albert.py

@@ -141,10 +141,7 @@ class TFAlbertModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output.numpy(), "pooled_output": pooled_output.numpy()}
-                "sequence_output": sequence_output.numpy(),
-                "pooled_output": pooled_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -156,9 +153,7 @@ class TFAlbertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFAlbertForMaskedLM(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (prediction_scores,) = model(inputs)
-            result = {
+            result = {"prediction_scores": prediction_scores.numpy()}
-                "prediction_scores": prediction_scores.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -170,9 +165,7 @@ class TFAlbertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFAlbertForSequenceClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_labels])
 
         def prepare_config_and_inputs_for_common(self):

tests/test_modeling_tf_bert.py

@@ -150,10 +150,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
+            result = {"sequence_output": sequence_output.numpy(), "pooled_output": pooled_output.numpy()}
-                "sequence_output": sequence_output.numpy(),
-                "pooled_output": pooled_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -165,9 +162,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForMaskedLM(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (prediction_scores,) = model(inputs)
-            result = {
+            result = {"prediction_scores": prediction_scores.numpy()}
-                "prediction_scores": prediction_scores.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -178,9 +173,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForNextSentencePrediction(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (seq_relationship_score,) = model(inputs)
-            result = {
+            result = {"seq_relationship_score": seq_relationship_score.numpy()}
-                "seq_relationship_score": seq_relationship_score.numpy(),
-            }
             self.parent.assertListEqual(list(result["seq_relationship_score"].shape), [self.batch_size, 2])
 
         def create_and_check_bert_for_pretraining(
@@ -205,9 +198,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForSequenceClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_labels])
 
         def create_and_check_bert_for_multiple_choice(
@@ -224,9 +215,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
                 "token_type_ids": multiple_choice_token_type_ids,
             }
             (logits,) = model(inputs)
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_choices])
 
         def create_and_check_bert_for_token_classification(
@@ -236,9 +225,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForTokenClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, self.num_labels]
             )
@@ -249,10 +236,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForQuestionAnswering(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             start_logits, end_logits = model(inputs)
-            result = {
+            result = {"start_logits": start_logits.numpy(), "end_logits": end_logits.numpy()}
-                "start_logits": start_logits.numpy(),
-                "end_logits": end_logits.numpy(),
-            }
             self.parent.assertListEqual(list(result["start_logits"].shape), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].shape), [self.batch_size, self.seq_length])
 

tests/test_modeling_tf_ctrl.py

@@ -145,9 +145,7 @@ class TFCTRLModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output = model(input_ids)[0]
 
-            result = {
+            result = {"sequence_output": sequence_output.numpy()}
-                "sequence_output": sequence_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -156,9 +154,7 @@ class TFCTRLModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFCTRLLMHeadModel(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             prediction_scores = model(inputs)[0]
-            result = {
+            result = {"prediction_scores": prediction_scores.numpy()}
-                "prediction_scores": prediction_scores.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )

tests/test_modeling_tf_distilbert.py

@@ -142,9 +142,7 @@ class TFDistilBertModelTest(TFModelTesterMixin, unittest.TestCase):
 
             (sequence_output,) = model(inputs)
 
-            result = {
+            result = {"sequence_output": sequence_output.numpy()}
-                "sequence_output": sequence_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -155,9 +153,7 @@ class TFDistilBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFDistilBertForMaskedLM(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask}
             (prediction_scores,) = model(inputs)
-            result = {
+            result = {"prediction_scores": prediction_scores.numpy()}
-                "prediction_scores": prediction_scores.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -168,10 +164,7 @@ class TFDistilBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFDistilBertForQuestionAnswering(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask}
             start_logits, end_logits = model(inputs)
-            result = {
+            result = {"start_logits": start_logits.numpy(), "end_logits": end_logits.numpy()}
-                "start_logits": start_logits.numpy(),
-                "end_logits": end_logits.numpy(),
-            }
             self.parent.assertListEqual(list(result["start_logits"].shape), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].shape), [self.batch_size, self.seq_length])
 
@@ -182,9 +175,7 @@ class TFDistilBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFDistilBertForSequenceClassification(config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask}
             (logits,) = model(inputs)
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_labels])
 
         def prepare_config_and_inputs_for_common(self):

tests/test_modeling_tf_gpt2.py

@@ -152,9 +152,7 @@ class TFGPT2ModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output = model(input_ids)[0]
 
-            result = {
+            result = {"sequence_output": sequence_output.numpy()}
-                "sequence_output": sequence_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -163,9 +161,7 @@ class TFGPT2ModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFGPT2LMHeadModel(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             prediction_scores = model(inputs)[0]
-            result = {
+            result = {"prediction_scores": prediction_scores.numpy()}
-                "prediction_scores": prediction_scores.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )

tests/test_modeling_tf_openai_gpt.py

@@ -153,9 +153,7 @@ class TFOpenAIGPTModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output = model(input_ids)[0]
 
-            result = {
+            result = {"sequence_output": sequence_output.numpy()}
-                "sequence_output": sequence_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -164,9 +162,7 @@ class TFOpenAIGPTModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFOpenAIGPTLMHeadModel(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             prediction_scores = model(inputs)[0]
-            result = {
+            result = {"prediction_scores": prediction_scores.numpy()}
-                "prediction_scores": prediction_scores.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )

tests/test_modeling_tf_roberta.py

@@ -138,9 +138,7 @@ class TFRobertaModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output = model(input_ids)[0]
 
-            result = {
+            result = {"sequence_output": sequence_output.numpy()}
-                "sequence_output": sequence_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -150,9 +148,7 @@ class TFRobertaModelTest(TFModelTesterMixin, unittest.TestCase):
         ):
             model = TFRobertaForMaskedLM(config=config)
             prediction_scores = model([input_ids, input_mask, token_type_ids])[0]
-            result = {
+            result = {"prediction_scores": prediction_scores.numpy()}
-                "prediction_scores": prediction_scores.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -164,9 +160,7 @@ class TFRobertaModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFRobertaForTokenClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, self.num_labels]
             )

tests/test_modeling_tf_t5.py

@@ -109,10 +109,7 @@ class TFT5ModelTest(TFModelTesterMixin, unittest.TestCase):
                 input_ids, decoder_attention_mask=input_mask, encoder_input_ids=input_ids
             )
 
-            result = {
+            result = {"encoder_output": encoder_output.numpy(), "decoder_output": decoder_output.numpy()}
-                "encoder_output": encoder_output.numpy(),
-                "decoder_output": decoder_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["encoder_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -128,9 +125,7 @@ class TFT5ModelTest(TFModelTesterMixin, unittest.TestCase):
                 "decoder_attention_mask": input_mask,
             }
             prediction_scores, decoder_output = model(inputs)
-            result = {
+            result = {"prediction_scores": prediction_scores.numpy()}
-                "prediction_scores": prediction_scores.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )

tests/test_modeling_tf_xlm.py

@@ -176,9 +176,7 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
             inputs = [input_ids, input_mask]
             outputs = model(inputs)
             sequence_output = outputs[0]
-            result = {
+            result = {"sequence_output": sequence_output.numpy()}
-                "sequence_output": sequence_output.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -201,9 +199,7 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
 
             logits = outputs[0]
 
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
 
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, self.vocab_size]
@@ -226,10 +222,7 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
 
             start_logits, end_logits = model(inputs)
 
-            result = {
+            result = {"start_logits": start_logits.numpy(), "end_logits": end_logits.numpy()}
-                "start_logits": start_logits.numpy(),
-                "end_logits": end_logits.numpy(),
-            }
 
             self.parent.assertListEqual(list(result["start_logits"].shape), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].shape), [self.batch_size, self.seq_length])
@@ -251,9 +244,7 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
 
             (logits,) = model(inputs)
 
-            result = {
+            result = {"logits": logits.numpy()}
-                "logits": logits.numpy(),
-            }
 
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.type_sequence_label_size])
 

tests/test_modeling_tf_xlnet.py

@@ -183,10 +183,7 @@ class TFXLNetModelTest(TFModelTesterMixin, unittest.TestCase):
 
             outputs, mems_1 = model(inputs)
 
-            result = {
+            result = {"mems_1": [mem.numpy() for mem in mems_1], "outputs": outputs.numpy()}
-                "mems_1": [mem.numpy() for mem in mems_1],
-                "outputs": outputs.numpy(),
-            }
 
             config.mem_len = 0
             model = TFXLNetModel(config)
@@ -302,10 +299,7 @@ class TFXLNetModelTest(TFModelTesterMixin, unittest.TestCase):
 
             logits, mems_1 = model(input_ids_1)
 
-            result = {
+            result = {"mems_1": [mem.numpy() for mem in mems_1], "logits": logits.numpy()}
-                "mems_1": [mem.numpy() for mem in mems_1],
-                "logits": logits.numpy(),
-            }
 
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.type_sequence_label_size])
             self.parent.assertListEqual(
@@ -335,10 +329,7 @@ class TFXLNetModelTest(TFModelTesterMixin, unittest.TestCase):
                 # 'token_type_ids': token_type_ids
             }
             logits, mems_1 = model(inputs)
-            result = {
+            result = {"mems_1": [mem.numpy() for mem in mems_1], "logits": logits.numpy()}
-                "mems_1": [mem.numpy() for mem in mems_1],
-                "logits": logits.numpy(),
-            }
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, config.num_labels]
             )

tests/test_modeling_xlm.py

@@ -185,9 +185,7 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
             outputs = model(input_ids, langs=token_type_ids)
             outputs = model(input_ids)
             sequence_output = outputs[0]
-            result = {
+            result = {"sequence_output": sequence_output}
-                "sequence_output": sequence_output,
-            }
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -209,10 +207,7 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
 
             loss, logits = model(input_ids, token_type_ids=token_type_ids, labels=token_labels)
 
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(
@@ -239,11 +234,7 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
             outputs = model(input_ids, start_positions=sequence_labels, end_positions=sequence_labels)
             loss, start_logits, end_logits = outputs
 
-            result = {
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -333,10 +324,7 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
             (logits,) = model(input_ids)
             loss, logits = model(input_ids, labels=sequence_labels)
 
-            result = {
+            result = {"loss": loss, "logits": logits}
-                "loss": loss,
-                "logits": logits,
-            }
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(

tests/test_modeling_xlnet.py

@@ -187,10 +187,7 @@ class XLNetModelTest(ModelTesterMixin, unittest.TestCase):
             _, _ = model(input_ids_1, token_type_ids=segment_ids)
             outputs, mems_1 = model(input_ids_1)
 
-            result = {
+            result = {"mems_1": mems_1, "outputs": outputs}
-                "mems_1": mems_1,
-                "outputs": outputs,
-            }
 
             config.mem_len = 0
             model = XLNetModel(config)
@@ -385,11 +382,7 @@ class XLNetModelTest(ModelTesterMixin, unittest.TestCase):
             logits, mems_1 = model(input_ids_1)
             loss, logits, mems_1 = model(input_ids_1, labels=token_labels)
 
-            result = {
+            result = {"loss": loss, "mems_1": mems_1, "logits": logits}
-                "loss": loss,
-                "mems_1": mems_1,
-                "logits": logits,
-            }
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(
@@ -422,11 +415,7 @@ class XLNetModelTest(ModelTesterMixin, unittest.TestCase):
             logits, mems_1 = model(input_ids_1)
             loss, logits, mems_1 = model(input_ids_1, labels=sequence_labels)
 
-            result = {
+            result = {"loss": loss, "mems_1": mems_1, "logits": logits}
-                "loss": loss,
-                "mems_1": mems_1,
-                "logits": logits,
-            }
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(

tests/test_tokenization_bert.py

@@ -84,7 +84,7 @@ class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         tokenizer = self.get_tokenizer()
         rust_tokenizer = self.get_rust_tokenizer(add_special_tokens=False)
 
-        sequence = u"UNwant\u00E9d,running"
+        sequence = "UNwant\u00E9d,running"
 
         tokens = tokenizer.tokenize(sequence)
         rust_tokens = rust_tokenizer.tokenize(sequence)

tests/test_tokenization_common.py

@@ -42,15 +42,7 @@ class TokenizerTesterMixin:
 
     def test_tokenizers_common_properties(self):
         tokenizer = self.get_tokenizer()
-        attributes_list = [
+        attributes_list = ["bos_token", "eos_token", "unk_token", "sep_token", "pad_token", "cls_token", "mask_token"]
-            "bos_token",
-            "eos_token",
-            "unk_token",
-            "sep_token",
-            "pad_token",
-            "cls_token",
-            "mask_token",
-        ]
         for attr in attributes_list:
             self.assertTrue(hasattr(tokenizer, attr))
             self.assertTrue(hasattr(tokenizer, attr + "_id"))
@@ -280,7 +272,7 @@ class TokenizerTesterMixin:
         num_added_tokens = tokenizer.num_added_tokens()
         total_length = len(sequence) + num_added_tokens
         information = tokenizer.encode_plus(
-            seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride, return_overflowing_tokens=True,
+            seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride, return_overflowing_tokens=True
         )
 
         truncated_sequence = information["input_ids"]
@@ -303,7 +295,7 @@ class TokenizerTesterMixin:
 
         sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)
         truncated_second_sequence = tokenizer.build_inputs_with_special_tokens(
-            tokenizer.encode(seq_0, add_special_tokens=False), tokenizer.encode(seq_1, add_special_tokens=False)[:-2],
+            tokenizer.encode(seq_0, add_special_tokens=False), tokenizer.encode(seq_1, add_special_tokens=False)[:-2]
         )
 
         information = tokenizer.encode_plus(

tests/test_tokenization_gpt2.py

@@ -96,7 +96,7 @@ class GPT2TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         tokenizer = self.get_tokenizer()
         rust_tokenizer = self.get_rust_tokenizer(add_special_tokens=False, add_prefix_space=True)
 
-        sequence = u"lower newer"
+        sequence = "lower newer"
 
         # Testing tokenization
         tokens = tokenizer.tokenize(sequence, add_prefix_space=True)

tests/test_tokenization_transfo_xl.py

@@ -35,19 +35,7 @@ class TransfoXLTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
     def setUp(self):
         super(TransfoXLTokenizationTest, self).setUp()
 
-        vocab_tokens = [
+        vocab_tokens = ["<unk>", "[CLS]", "[SEP]", "want", "unwanted", "wa", "un", "running", ",", "low", "l"]
-            "<unk>",
-            "[CLS]",
-            "[SEP]",
-            "want",
-            "unwanted",
-            "wa",
-            "un",
-            "running",
-            ",",
-            "low",
-            "l",
-        ]
         self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
         with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))