using black -t py35

Typo in tokenization_utils.py

CONTRIBUTING.md

@@ -121,7 +121,7 @@ Follow these steps to start contributing:
 4. Set up a development environment by running the following command in a virtual environment:
 
    ```bash
-   $ pip install -e .[dev]
+   $ pip install -e ".[dev]"
    ```
 
    (If transformers was already installed in the virtual environment, remove

README.md

@@ -120,14 +120,14 @@ Depending on which framework is installed (TensorFlow 2.0 and/or PyTorch), the i
 Here's the easiest way to run tests for the library:
 
 ```bash
-pip install -e .[testing]
+pip install -e ".[testing]"
 make test
 ```
 
 and for the examples:
 
 ```bash
-pip install -e .[testing]
+pip install -e ".[testing]"
 pip install -r examples/requirements.txt
 make test-examples
 ```

docs/README.md

@@ -4,7 +4,7 @@ To generate the documentation, you first have to build it. Several packages are
 you can install them with the following command, at the root of the code repository:
 
 ```bash
-pip install -e .[docs]
+pip install -e ".[docs]"
 ```
 
 ## Packages installed

examples/contrib/run_camembert.py

@@ -30,7 +30,7 @@ def fill_mask(masked_input, model, tokenizer, topk=5):
             )
         else:
             topk_filled_outputs.append(
-                (masked_input.replace(masked_token, predicted_token), values[index].item(), predicted_token,)
+                (masked_input.replace(masked_token, predicted_token), values[index].item(), predicted_token)
             )
     return topk_filled_outputs
 

examples/contrib/run_openai_gpt.py

@@ -44,13 +44,10 @@ from transformers import (
     AdamW,
     OpenAIGPTDoubleHeadsModel,
     OpenAIGPTTokenizer,
-    cached_path,
     get_linear_schedule_with_warmup,
 )
 
 
-ROCSTORIES_URL = "https://s3.amazonaws.com/datasets.huggingface.co/ROCStories.tar.gz"
-
 logging.basicConfig(
     format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
 )
@@ -86,7 +83,7 @@ def pre_process_datasets(encoded_datasets, input_len, cap_length, start_token, d
         mc_token_ids = np.zeros((n_batch, 2), dtype=np.int64)
         lm_labels = np.full((n_batch, 2, input_len), fill_value=-100, dtype=np.int64)
         mc_labels = np.zeros((n_batch,), dtype=np.int64)
-        for i, (story, cont1, cont2, mc_label), in enumerate(dataset):
+        for i, (story, cont1, cont2, mc_label) in enumerate(dataset):
             with_cont1 = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
             with_cont2 = [start_token] + story[:cap_length] + [delimiter_token] + cont2[:cap_length] + [clf_token]
             input_ids[i, 0, : len(with_cont1)] = with_cont1
@@ -182,9 +179,6 @@ def main():
     model.to(device)
 
     # Load and encode the datasets
-    if not args.train_dataset and not args.eval_dataset:
-        roc_stories = cached_path(ROCSTORIES_URL)
-
     def tokenize_and_encode(obj):
         """ Tokenize and encode a nested object """
         if isinstance(obj, str):

examples/contrib/run_swag.py

@@ -51,9 +51,7 @@ logger = logging.getLogger(__name__)
 
 ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in [BertConfig]), ())
 
-MODEL_CLASSES = {
-    "bert": (BertConfig, BertForMultipleChoice, BertTokenizer),
-}
+MODEL_CLASSES = {"bert": (BertConfig, BertForMultipleChoice, BertTokenizer)}
 
 
 class SwagExample(object):
@@ -63,12 +61,7 @@ class SwagExample(object):
         self.swag_id = swag_id
         self.context_sentence = context_sentence
         self.start_ending = start_ending
-        self.endings = [
-            ending_0,
-            ending_1,
-            ending_2,
-            ending_3,
-        ]
+        self.endings = [ending_0, ending_1, ending_2, ending_3]
         self.label = label
 
     def __str__(self):

examples/contrib/run_transfo_xl.py

@@ -28,7 +28,7 @@ import time
 
 import torch
 
-from transformers import TransfoXLCorpus, TransfoXLLMHeadModel, TransfoXLTokenizer
+from transformers import TransfoXLCorpus, TransfoXLLMHeadModel
 
 
 logging.basicConfig(
@@ -73,9 +73,7 @@ def main():
     # The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
     # and tokenizing the dataset
     # The pre-processed corpus is a convertion (using the conversion script )
-    tokenizer = TransfoXLTokenizer.from_pretrained(args.model_name)
     corpus = TransfoXLCorpus.from_pretrained(args.model_name)
-    ntokens = len(corpus.vocab)
 
     va_iter = corpus.get_iterator("valid", args.batch_size, args.tgt_len, device=device, ext_len=args.ext_len)
     te_iter = corpus.get_iterator("test", args.batch_size, args.tgt_len, device=device, ext_len=args.ext_len)

examples/distillation/utils.py

@@ -117,9 +117,7 @@ def init_gpu_params(params):
     # initialize multi-GPU
     if params.multi_gpu:
         logger.info("Initializing PyTorch distributed")
-        torch.distributed.init_process_group(
-            init_method="env://", backend="nccl",
-        )
+        torch.distributed.init_process_group(init_method="env://", backend="nccl")
 
 
 def set_seed(args):

examples/mm-imdb/utils_mmimdb.py

@@ -138,6 +138,6 @@ def get_image_transforms():
             transforms.Resize(256),
             transforms.CenterCrop(224),
             transforms.ToTensor(),
-            transforms.Normalize(mean=[0.46777044, 0.44531429, 0.40661017], std=[0.12221994, 0.12145835, 0.14380469],),
+            transforms.Normalize(mean=[0.46777044, 0.44531429, 0.40661017], std=[0.12221994, 0.12145835, 0.14380469]),
         ]
     )

examples/pplm/run_pplm.py

@@ -718,7 +718,7 @@ if __name__ == "__main__":
     parser.add_argument("--cond_text", type=str, default="The lake", help="Prefix texts to condition on")
     parser.add_argument("--uncond", action="store_true", help="Generate from end-of-text as prefix")
     parser.add_argument(
-        "--num_samples", type=int, default=1, help="Number of samples to generate from the modified latents",
+        "--num_samples", type=int, default=1, help="Number of samples to generate from the modified latents"
     )
     parser.add_argument(
         "--bag_of_words",
@@ -741,9 +741,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--discrim_meta", type=str, default=None, help="Meta information for the generic discriminator"
     )
-    parser.add_argument(
-        "--class_label", type=int, default=-1, help="Class label used for the discriminator",
-    )
+    parser.add_argument("--class_label", type=int, default=-1, help="Class label used for the discriminator")
     parser.add_argument("--length", type=int, default=100)
     parser.add_argument("--stepsize", type=float, default=0.02)
     parser.add_argument("--temperature", type=float, default=1.0)
@@ -757,9 +755,7 @@ if __name__ == "__main__":
         default=0,
         help="Length of past which is being optimized; " "0 corresponds to infinite window length",
     )
-    parser.add_argument(
-        "--horizon_length", type=int, default=1, help="Length of future to optimize over",
-    )
+    parser.add_argument("--horizon_length", type=int, default=1, help="Length of future to optimize over")
     parser.add_argument("--decay", action="store_true", help="whether to decay or not")
     parser.add_argument("--gamma", type=float, default=1.5)
     parser.add_argument("--gm_scale", type=float, default=0.9)

examples/pplm/run_pplm_discrim_train.py

@@ -242,7 +242,7 @@ def train_discriminator(
 
         text = torchtext_data.Field()
         label = torchtext_data.Field(sequential=False)
-        train_data, val_data, test_data = datasets.SST.splits(text, label, fine_grained=True, train_subtrees=True,)
+        train_data, val_data, test_data = datasets.SST.splits(text, label, fine_grained=True, train_subtrees=True)
 
         x = []
         y = []

examples/run_generation.py

@@ -41,7 +41,7 @@ from transformers import (
 
 
 logging.basicConfig(
-    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
 )
 logger = logging.getLogger(__name__)
 

examples/run_multiple_choice.py

@@ -141,7 +141,7 @@ def train(args, train_dataset, model, tokenizer):
 
     global_step = 0
     tr_loss, logging_loss = 0.0, 0.0
-    best_dev_acc, best_dev_loss = 0.0, 99999999999.0
+    best_dev_acc = 0.0
     best_steps = 0
     model.zero_grad()
     train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
@@ -193,7 +193,6 @@ def train(args, train_dataset, model, tokenizer):
                             tb_writer.add_scalar("eval_{}".format(key), value, global_step)
                         if results["eval_acc"] > best_dev_acc:
                             best_dev_acc = results["eval_acc"]
-                            best_dev_loss = results["eval_loss"]
                             best_steps = global_step
                             if args.do_test:
                                 results_test = evaluate(args, model, tokenizer, test=True)

examples/summarization/configuration_bertabs.py

@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 BERTABS_FINETUNED_CONFIG_MAP = {
-    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-config.json",
+    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-config.json"
 }
 
 

examples/summarization/convert_bertabs_original_pytorch_checkpoint.py

@@ -164,13 +164,11 @@ def convert_bertabs_checkpoints(path_to_checkpoints, dump_path):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--bertabs_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump.",
+        "--bertabs_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
     )
     parser.add_argument(
-        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model.",
+        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
     )
     args = parser.parse_args()
 
-    convert_bertabs_checkpoints(
-        args.bertabs_checkpoint_path, args.pytorch_dump_folder_path,
-    )
+    convert_bertabs_checkpoints(args.bertabs_checkpoint_path, args.pytorch_dump_folder_path)

examples/summarization/modeling_bertabs.py

@@ -34,7 +34,7 @@ from transformers import BertConfig, BertModel, PreTrainedModel
 MAX_SIZE = 5000
 
 BERTABS_FINETUNED_MODEL_MAP = {
-    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-pytorch_model.bin",
+    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-pytorch_model.bin"
 }
 
 
@@ -106,10 +106,10 @@ class BertAbs(BertAbsPreTrainedModel):
                 p.data.zero_()
 
     def forward(
-        self, encoder_input_ids, decoder_input_ids, token_type_ids, encoder_attention_mask, decoder_attention_mask,
+        self, encoder_input_ids, decoder_input_ids, token_type_ids, encoder_attention_mask, decoder_attention_mask
     ):
         encoder_output = self.bert(
-            input_ids=encoder_input_ids, token_type_ids=token_type_ids, attention_mask=encoder_attention_mask,
+            input_ids=encoder_input_ids, token_type_ids=token_type_ids, attention_mask=encoder_attention_mask
         )
         encoder_hidden_states = encoder_output[0]
         dec_state = self.decoder.init_decoder_state(encoder_input_ids, encoder_hidden_states)
@@ -308,7 +308,7 @@ class TransformerDecoderLayer(nn.Module):
         self.register_buffer("mask", mask)
 
     def forward(
-        self, inputs, memory_bank, src_pad_mask, tgt_pad_mask, previous_input=None, layer_cache=None, step=None,
+        self, inputs, memory_bank, src_pad_mask, tgt_pad_mask, previous_input=None, layer_cache=None, step=None
     ):
         """
         Args:
@@ -332,13 +332,13 @@ class TransformerDecoderLayer(nn.Module):
             all_input = torch.cat((previous_input, input_norm), dim=1)
             dec_mask = None
 
-        query = self.self_attn(all_input, all_input, input_norm, mask=dec_mask, layer_cache=layer_cache, type="self",)
+        query = self.self_attn(all_input, all_input, input_norm, mask=dec_mask, layer_cache=layer_cache, type="self")
 
         query = self.drop(query) + inputs
 
         query_norm = self.layer_norm_2(query)
         mid = self.context_attn(
-            memory_bank, memory_bank, query_norm, mask=src_pad_mask, layer_cache=layer_cache, type="context",
+            memory_bank, memory_bank, query_norm, mask=src_pad_mask, layer_cache=layer_cache, type="context"
         )
         output = self.feed_forward(self.drop(mid) + query)
 
@@ -422,9 +422,7 @@ class MultiHeadedAttention(nn.Module):
         if self.use_final_linear:
             self.final_linear = nn.Linear(model_dim, model_dim)
 
-    def forward(
-        self, key, value, query, mask=None, layer_cache=None, type=None, predefined_graph_1=None,
-    ):
+    def forward(self, key, value, query, mask=None, layer_cache=None, type=None, predefined_graph_1=None):
         """
         Compute the context vector and the attention vectors.
 
@@ -446,8 +444,6 @@ class MultiHeadedAttention(nn.Module):
         batch_size = key.size(0)
         dim_per_head = self.dim_per_head
         head_count = self.head_count
-        key_len = key.size(1)
-        query_len = query.size(1)
 
         def shape(x):
             """  projection """
@@ -460,11 +456,7 @@ class MultiHeadedAttention(nn.Module):
         # 1) Project key, value, and query.
         if layer_cache is not None:
             if type == "self":
-                query, key, value = (
-                    self.linear_query(query),
-                    self.linear_keys(query),
-                    self.linear_values(query),
-                )
+                query, key, value = (self.linear_query(query), self.linear_keys(query), self.linear_values(query))
 
                 key = shape(key)
                 value = shape(value)
@@ -485,10 +477,7 @@ class MultiHeadedAttention(nn.Module):
                         key = shape(key)
                         value = shape(value)
                     else:
-                        key, value = (
-                            layer_cache["memory_keys"],
-                            layer_cache["memory_values"],
-                        )
+                        key, value = (layer_cache["memory_keys"], layer_cache["memory_values"])
                     layer_cache["memory_keys"] = key
                     layer_cache["memory_values"] = value
                 else:
@@ -504,9 +493,6 @@ class MultiHeadedAttention(nn.Module):
 
         query = shape(query)
 
-        key_len = key.size(2)
-        query_len = query.size(2)
-
         # 2) Calculate and scale scores.
         query = query / math.sqrt(dim_per_head)
         scores = torch.matmul(query, key.transpose(2, 3))
@@ -1004,12 +990,8 @@ class BertSumOptimizer(object):
         self.warmup_steps = warmup_steps
 
         self.optimizers = {
-            "encoder": torch.optim.Adam(
-                model.encoder.parameters(), lr=lr["encoder"], betas=(beta_1, beta_2), eps=eps,
-            ),
-            "decoder": torch.optim.Adam(
-                model.decoder.parameters(), lr=lr["decoder"], betas=(beta_1, beta_2), eps=eps,
-            ),
+            "encoder": torch.optim.Adam(model.encoder.parameters(), lr=lr["encoder"], betas=(beta_1, beta_2), eps=eps),
+            "decoder": torch.optim.Adam(model.decoder.parameters(), lr=lr["decoder"], betas=(beta_1, beta_2), eps=eps),
         }
 
         self._step = 0

examples/summarization/run_summarization.py

@@ -188,7 +188,7 @@ def build_data_iterator(args, tokenizer):
     def collate_fn(data):
         return collate(data, tokenizer, block_size=512, device=args.device)
 
-    iterator = DataLoader(dataset, sampler=sampler, batch_size=args.batch_size, collate_fn=collate_fn,)
+    iterator = DataLoader(dataset, sampler=sampler, batch_size=args.batch_size, collate_fn=collate_fn)
 
     return iterator
 
@@ -265,24 +265,14 @@ def main():
         help="Compute the ROUGE metrics during evaluation. Only available for the CNN/DailyMail dataset.",
     )
     # EVALUATION options
-    parser.add_argument(
-        "--no_cuda", default=False, type=bool, help="Whether to force the execution on CPU.",
-    )
-    parser.add_argument(
-        "--batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.",
-    )
+    parser.add_argument("--no_cuda", default=False, type=bool, help="Whether to force the execution on CPU.")
+    parser.add_argument("--batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.")
     # BEAM SEARCH arguments
+    parser.add_argument("--min_length", default=50, type=int, help="Minimum number of tokens for the summaries.")
+    parser.add_argument("--max_length", default=200, type=int, help="Maixmum number of tokens for the summaries.")
+    parser.add_argument("--beam_size", default=5, type=int, help="The number of beams to start with for each example.")
     parser.add_argument(
-        "--min_length", default=50, type=int, help="Minimum number of tokens for the summaries.",
-    )
-    parser.add_argument(
-        "--max_length", default=200, type=int, help="Maixmum number of tokens for the summaries.",
-    )
-    parser.add_argument(
-        "--beam_size", default=5, type=int, help="The number of beams to start with for each example.",
-    )
-    parser.add_argument(
-        "--alpha", default=0.95, type=float, help="The value of alpha for the length penalty in the beam search.",
+        "--alpha", default=0.95, type=float, help="The value of alpha for the length penalty in the beam search."
     )
     parser.add_argument(
         "--block_trigram",

examples/utils_multiple_choice.py

@@ -320,7 +320,7 @@ def convert_examples_to_features(
             else:
                 text_b = example.question + " " + ending
 
-            inputs = tokenizer.encode_plus(text_a, text_b, add_special_tokens=True, max_length=max_length,)
+            inputs = tokenizer.encode_plus(text_a, text_b, add_special_tokens=True, max_length=max_length)
             if "num_truncated_tokens" in inputs and inputs["num_truncated_tokens"] > 0:
                 logger.info(
                     "Attention! you are cropping tokens (swag task is ok). "
@@ -362,7 +362,7 @@ def convert_examples_to_features(
                 logger.info("token_type_ids: {}".format(" ".join(map(str, token_type_ids))))
                 logger.info("label: {}".format(label))
 
-        features.append(InputFeatures(example_id=example.example_id, choices_features=choices_features, label=label,))
+        features.append(InputFeatures(example_id=example.example_id, choices_features=choices_features, label=label))
 
     return features
 

setup.cfg

@@ -25,5 +25,5 @@ multi_line_output = 3
 use_parentheses = True
 
 [flake8]
-ignore = E203, E501, F841, W503
+ignore = E203, E501, W503
 max-line-length = 119

setup.py

@@ -62,6 +62,7 @@ extras["mecab"] = ["mecab-python3"]
 extras["sklearn"] = ["scikit-learn"]
 extras["tf"] = ["tensorflow"]
 extras["torch"] = ["torch"]
+extras["fast"] = ["tokenizers == 0.0.10"]
 
 extras["serving"] = ["pydantic", "uvicorn", "fastapi"]
 extras["all"] = extras["serving"] + ["tensorflow", "torch"]
@@ -86,7 +87,6 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.0.10",
         # accessing files from S3 directly
         "boto3",
         # filesystem locks e.g. to prevent parallel downloads

src/transformers/__init__.py

@@ -70,6 +70,7 @@ from .file_utils import (
     add_end_docstrings,
     add_start_docstrings,
     cached_path,
+    is_fast_tokenizers_available,
     is_tf_available,
     is_torch_available,
 )

src/transformers/commands/serving.py

@@ -109,8 +109,8 @@ class ServeCommand(BaseTransformersCLICommand):
         if not _serve_dependancies_installed:
             raise RuntimeError(
                 "Using serve command requires FastAPI and unicorn. "
-                "Please install transformers with [serving]: pip install transformers[serving]."
-                "Or install FastAPI and unicorn separatly."
+                'Please install transformers with [serving]: pip install "transformers[serving]".'
+                "Or install FastAPI and unicorn separately."
             )
         else:
             logger.info("Serving model over {}:{}".format(host, port))

src/transformers/configuration_camembert.py

@@ -24,7 +24,7 @@ from .configuration_roberta import RobertaConfig
 logger = logging.getLogger(__name__)
 
 CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json",
+    "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json"
 }
 
 

src/transformers/configuration_transfo_xl.py

@@ -24,7 +24,7 @@ from .configuration_utils import PretrainedConfig
 logger = logging.getLogger(__name__)
 
 TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-config.json",
+    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-config.json"
 }
 
 

src/transformers/data/metrics/__init__.py

@@ -19,7 +19,7 @@ try:
     from sklearn.metrics import matthews_corrcoef, f1_score
 
     _has_sklearn = True
-except (AttributeError, ImportError) as e:
+except (AttributeError, ImportError):
     _has_sklearn = False
 
 
@@ -35,20 +35,12 @@ if _has_sklearn:
     def acc_and_f1(preds, labels):
         acc = simple_accuracy(preds, labels)
         f1 = f1_score(y_true=labels, y_pred=preds)
-        return {
-            "acc": acc,
-            "f1": f1,
-            "acc_and_f1": (acc + f1) / 2,
-        }
+        return {"acc": acc, "f1": f1, "acc_and_f1": (acc + f1) / 2}
 
     def pearson_and_spearman(preds, labels):
         pearson_corr = pearsonr(preds, labels)[0]
         spearman_corr = spearmanr(preds, labels)[0]
-        return {
-            "pearson": pearson_corr,
-            "spearmanr": spearman_corr,
-            "corr": (pearson_corr + spearman_corr) / 2,
-        }
+        return {"pearson": pearson_corr, "spearmanr": spearman_corr, "corr": (pearson_corr + spearman_corr) / 2}
 
     def glue_compute_metrics(task_name, preds, labels):
         assert len(preds) == len(labels)

src/transformers/data/processors/glue.py

@@ -86,7 +86,7 @@ def glue_convert_examples_to_features(
             example = processor.get_example_from_tensor_dict(example)
             example = processor.tfds_map(example)
 
-        inputs = tokenizer.encode_plus(example.text_a, example.text_b, add_special_tokens=True, max_length=max_length,)
+        inputs = tokenizer.encode_plus(example.text_a, example.text_b, add_special_tokens=True, max_length=max_length)
         input_ids, token_type_ids = inputs["input_ids"], inputs["token_type_ids"]
 
         # The mask has 1 for real tokens and 0 for padding tokens. Only real

src/transformers/data/processors/utils.py

@@ -244,7 +244,7 @@ class SingleSentenceClassificationProcessor(DataProcessor):
                 logger.info("Tokenizing example %d", ex_index)
 
             input_ids = tokenizer.encode(
-                example.text_a, add_special_tokens=True, max_length=min(max_length, tokenizer.max_len),
+                example.text_a, add_special_tokens=True, max_length=min(max_length, tokenizer.max_len)
             )
             all_input_ids.append(input_ids)
 

src/transformers/data/processors/xnli.py

@@ -72,14 +72,8 @@ class XnliProcessor(DataProcessor):
         return ["contradiction", "entailment", "neutral"]
 
 
-xnli_processors = {
-    "xnli": XnliProcessor,
-}
+xnli_processors = {"xnli": XnliProcessor}
 
-xnli_output_modes = {
-    "xnli": "classification",
-}
+xnli_output_modes = {"xnli": "classification"}
 
-xnli_tasks_num_labels = {
-    "xnli": 3,
-}
+xnli_tasks_num_labels = {"xnli": 3}

src/transformers/file_utils.py

@@ -55,6 +55,20 @@ try:
 except (ImportError, AssertionError):
     _tf_available = False  # pylint: disable=invalid-name
 
+try:
+    os.environ.setdefault("USE_FAST_TOKENIZERS", "YES")
+    if os.environ["USE_FAST_TOKENIZERS"].upper() in ("1", "ON", "YES"):
+        import tokenizers
+
+        _fast_tokenizers_available = True  # pylint: disable=invalid-name
+        logger.info("Fast Tokenizers version {} available.".format(tokenizers.__version__))
+    else:
+        logger.info("USE_FAST_TOKENIZERS override through env variable, disabling fast Tokenizers")
+        _fast_tokenizers_available = False
+except ImportError:
+    _fast_tokenizers_available = False  # pylint: disable=invalid-name
+
+
 try:
     from torch.hub import _get_torch_home
 
@@ -97,6 +111,10 @@ def is_torch_available():
     return _torch_available
 
 
+def is_fast_tokenizers_available():
+    return _fast_tokenizers_available
+
+
 def is_tf_available():
 
     return _tf_available

src/transformers/modeling_albert.py

@@ -241,8 +241,6 @@ class AlbertAttention(BertSelfAttention):
         context_layer = torch.matmul(attention_probs, value_layer)
 
         context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
-        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
-        reshaped_context_layer = context_layer.view(*new_context_layer_shape)
 
         # Should find a better way to do this
         w = (
@@ -334,9 +332,6 @@ class AlbertTransformer(nn.Module):
             # Index of the hidden group
             group_idx = int(i / (self.config.num_hidden_layers / self.config.num_hidden_groups))
 
-            # Index of the layer inside the group
-            layer_idx = int(i - group_idx * layers_per_group)
-
             layer_group_output = self.albert_layer_groups[group_idx](
                 hidden_states,
                 attention_mask,
@@ -842,7 +837,7 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_bert.py

@@ -810,7 +810,7 @@ class BertModel(BertPreTrainedModel):
         sequence_output = encoder_outputs[0]
         pooled_output = self.pooler(sequence_output)
 
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[
+        outputs = (sequence_output, pooled_output) + encoder_outputs[
             1:
         ]  # add hidden_states and attentions if they are here
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
@@ -895,7 +895,7 @@ class BertForPreTraining(BertPreTrainedModel):
         sequence_output, pooled_output = outputs[:2]
         prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output)
 
-        outputs = (prediction_scores, seq_relationship_score,) + outputs[
+        outputs = (prediction_scores, seq_relationship_score) + outputs[
             2:
         ]  # add hidden states and attention if they are here
 
@@ -1440,7 +1440,7 @@ class BertForQuestionAnswering(BertPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_camembert.py

@@ -32,7 +32,7 @@ from .modeling_roberta import (
 logger = logging.getLogger(__name__)
 
 CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin",
+    "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin"
 }
 
 

src/transformers/modeling_ctrl.py

@@ -490,6 +490,15 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        # only last token for inputs_ids if past is defined in kwargs
+        if "past" in kwargs and kwargs["past"]:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+
+        inputs = {"input_ids": input_ids}
+        inputs.update(kwargs)
+        return inputs
+
     def forward(
         self,
         input_ids=None,

src/transformers/modeling_distilbert.py

@@ -707,7 +707,7 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel):
         start_logits = start_logits.squeeze(-1)  # (bs, max_query_len)
         end_logits = end_logits.squeeze(-1)  # (bs, max_query_len)
 
-        outputs = (start_logits, end_logits,) + distilbert_output[1:]
+        outputs = (start_logits, end_logits) + distilbert_output[1:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_encoder_decoder.py

@@ -325,7 +325,7 @@ class Model2Model(PreTrainedEncoderDecoder):
             encoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
             decoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
             *args,
-            **kwargs
+            **kwargs,
         )
 
         return model

src/transformers/modeling_gpt2.py

@@ -559,6 +559,15 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        # only last token for inputs_ids if past is defined in kwargs
+        if "past" in kwargs and kwargs["past"]:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+
+        inputs = {"input_ids": input_ids}
+        inputs.update(kwargs)
+        return inputs
+
     def forward(
         self,
         input_ids=None,

src/transformers/modeling_mmbt.py

@@ -310,7 +310,7 @@ class MMBTModel(nn.Module):
         sequence_output = encoder_outputs[0]
         pooled_output = self.transformer.pooler(sequence_output)
 
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[
+        outputs = (sequence_output, pooled_output) + encoder_outputs[
             1:
         ]  # add hidden_states and attentions if they are here
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)

src/transformers/modeling_roberta.py

@@ -715,7 +715,7 @@ class RobertaForQuestionAnswering(BertPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_t5.py

@@ -629,7 +629,7 @@ class T5Stack(T5PreTrainedModel):
                 all_attentions = all_attentions + (layer_outputs[1],)  # We keep only self-attention weights for now
 
         hidden_states = self.final_layer_norm(hidden_states)
-        layer_output = self.dropout(hidden_states)
+        hidden_states = self.dropout(hidden_states)
 
         # Add last layer
         if self.output_hidden_states:

src/transformers/modeling_tf_albert.py

@@ -698,7 +698,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel):
         pooled_output = self.pooler(sequence_output[:, 0])
 
         # add hidden_states and attentions if they are here
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[1:]
+        outputs = (sequence_output, pooled_output) + encoder_outputs[1:]
         # sequence_output, pooled_output, (hidden_states), (attentions)
         return outputs
 

src/transformers/modeling_tf_bert.py

@@ -568,7 +568,7 @@ class TFBertMainLayer(tf.keras.layers.Layer):
         sequence_output = encoder_outputs[0]
         pooled_output = self.pooler(sequence_output)
 
-        outputs = (sequence_output, pooled_output,) + encoder_outputs[
+        outputs = (sequence_output, pooled_output) + encoder_outputs[
             1:
         ]  # add hidden_states and attentions if they are here
         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
@@ -766,7 +766,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel):
         prediction_scores = self.mlm(sequence_output, training=kwargs.get("training", False))
         seq_relationship_score = self.nsp(pooled_output)
 
-        outputs = (prediction_scores, seq_relationship_score,) + outputs[
+        outputs = (prediction_scores, seq_relationship_score) + outputs[
             2:
         ]  # add hidden states and attention if they are here
 
@@ -1139,6 +1139,6 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
 
         return outputs  # start_logits, end_logits, (hidden_states), (attentions)

src/transformers/modeling_tf_distilbert.py

@@ -822,5 +822,5 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + distilbert_output[1:]
+        outputs = (start_logits, end_logits) + distilbert_output[1:]
         return outputs  # start_logits, end_logits, (hidden_states), (attentions)

src/transformers/modeling_tf_pytorch_utils.py

@@ -122,7 +122,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
         tf_inputs = tf_model.dummy_inputs
 
     if tf_inputs is not None:
-        tfo = tf_model(tf_inputs, training=False)  # Make sure model is built
+        tf_model(tf_inputs, training=False)  # Make sure model is built
 
     # Adapt state dict - TODO remove this and update the AWS weights files instead
     # Convert old format to new format if needed from a PyTorch state_dict
@@ -187,7 +187,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
     K.batch_set_value(weight_value_tuples)
 
     if tf_inputs is not None:
-        tfo = tf_model(tf_inputs, training=False)  # Make sure restore ops are run
+        tf_model(tf_inputs, training=False)  # Make sure restore ops are run
 
     logger.info("Loaded {:,} parameters in the TF 2.0 model.".format(tf_loaded_numel))
 
@@ -218,7 +218,6 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs
 
     import transformers
 
-    tf_path = os.path.abspath(tf_checkpoint_path)
     logger.info("Loading TensorFlow weights from {}".format(tf_checkpoint_path))
 
     # Instantiate and load the associated TF 2.0 model
@@ -230,7 +229,7 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs
         tf_inputs = tf_model.dummy_inputs
 
     if tf_inputs is not None:
-        tfo = tf_model(tf_inputs, training=False)  # Make sure model is built
+        tf_model(tf_inputs, training=False)  # Make sure model is built
 
     tf_model.load_weights(tf_checkpoint_path, by_name=True)
 

src/transformers/modeling_tf_t5.py

@@ -491,7 +491,7 @@ class TFT5MainLayer(tf.keras.layers.Layer):
                 all_attentions = all_attentions + (layer_outputs[1],)
 
         hidden_states = self.final_layer_norm(hidden_states)
-        layer_output = self.dropout(hidden_states, training=training)
+        hidden_states = self.dropout(hidden_states, training=training)
 
         # Add last layer
         if self.output_hidden_states:

src/transformers/modeling_tf_transfo_xl.py

@@ -30,7 +30,7 @@ from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list
 logger = logging.getLogger(__name__)
 
 TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-tf_model.h5",
+    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-tf_model.h5"
 }
 
 

src/transformers/modeling_tf_transfo_xl_utilities.py

@@ -64,7 +64,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer):
                 else:
                     self.out_projs.append(None)
                 weight = self.add_weight(
-                    shape=(self.vocab_size, self.d_embed,),
+                    shape=(self.vocab_size, self.d_embed),
                     initializer="zeros",
                     trainable=True,
                     name="out_layers_._{}_._weight".format(i),
@@ -86,7 +86,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer):
                 )
                 self.out_projs.append(weight)
                 weight = self.add_weight(
-                    shape=(r_idx - l_idx, d_emb_i,),
+                    shape=(r_idx - l_idx, d_emb_i),
                     initializer="zeros",
                     trainable=True,
                     name="out_layers_._{}_._weight".format(i),
@@ -118,7 +118,6 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer):
         hidden, target = inputs
         head_logprob = 0
         if self.n_clusters == 0:
-            softmax_b = tf.get_variable("bias", [self.config.vocab_size], initializer=tf.zeros_initializer())
             output = self._logit(hidden, self.out_layers[0][0], self.out_layers[0][1], self.out_projs[0])
             if target is not None:
                 loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, logits=output)

src/transformers/modeling_tf_utils.py

@@ -250,7 +250,7 @@ class TFPreTrainedModel(tf.keras.Model):
                 return_unused_kwargs=True,
                 force_download=force_download,
                 resume_download=resume_download,
-                **kwargs
+                **kwargs,
             )
         else:
             model_kwargs = kwargs
@@ -320,7 +320,7 @@ class TFPreTrainedModel(tf.keras.Model):
             # Load from a PyTorch checkpoint
             return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, allow_missing_keys=True)
 
-        ret = model(model.dummy_inputs, training=False)  # build the network with dummy inputs
+        model(model.dummy_inputs, training=False)  # build the network with dummy inputs
 
         assert os.path.isfile(resolved_archive_file), "Error retrieving file {}".format(resolved_archive_file)
         # 'by_name' allow us to do transfer learning by skipping/adding layers
@@ -333,7 +333,7 @@ class TFPreTrainedModel(tf.keras.Model):
                 "If you tried to load a TF 2.0 model from a PyTorch checkpoint, please set from_pt=True. "
             )
 
-        ret = model(model.dummy_inputs, training=False)  # Make sure restore ops are run
+        model(model.dummy_inputs, training=False)  # Make sure restore ops are run
 
         # Check if the models are the same to output loading informations
         with h5py.File(resolved_archive_file, "r") as f:
@@ -515,7 +515,7 @@ class TFSequenceSummary(tf.keras.layers.Layer):
             cls_index = inputs[1] if len(inputs) > 1 else None
             assert len(inputs) <= 2, "Too many inputs."
         else:
-            input_ids = inputs.get("input_ids")
+            hidden_states = inputs.get("hidden_states")
             cls_index = inputs.get("cls_index", None)
 
         if self.summary_type == "last":

src/transformers/modeling_tf_xlm.py

@@ -800,7 +800,7 @@ class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + transformer_outputs[
+        outputs = (start_logits, end_logits) + transformer_outputs[
             1:
         ]  # Keep mems, hidden states, attentions if there are in it
 

src/transformers/modeling_tf_xlnet.py

@@ -1074,7 +1074,7 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + transformer_outputs[
+        outputs = (start_logits, end_logits) + transformer_outputs[
             1:
         ]  # Keep mems, hidden states, attentions if there are in it
 

src/transformers/modeling_transfo_xl.py

@@ -34,7 +34,7 @@ from .modeling_utils import PreTrainedModel
 logger = logging.getLogger(__name__)
 
 TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-pytorch_model.bin",
+    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-pytorch_model.bin"
 }
 
 
@@ -930,3 +930,12 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel):
             return self.out_layer
         else:
             return self.crit.out_layers[-1]
+
+    def prepare_inputs_for_generation(self, input_ids, **model_kwargs):
+        inputs = {"input_ids": input_ids}
+
+        # if past is defined in model kwargs then use it for faster decoding
+        if "past" in model_kwargs and model_kwargs["past"]:
+            inputs["mems"] = model_kwargs["past"]
+
+        return inputs

src/transformers/modeling_utils.py

@@ -355,7 +355,7 @@ class PreTrainedModel(nn.Module):
                 force_download=force_download,
                 resume_download=resume_download,
                 proxies=proxies,
-                **kwargs
+                **kwargs,
             )
         else:
             model_kwargs = kwargs
@@ -539,6 +539,17 @@ class PreTrainedModel(nn.Module):
     def prepare_inputs_for_generation(self, input_ids, **kwargs):
         return {"input_ids": input_ids}
 
+    def _do_output_past(self, outputs):
+        has_output_past = hasattr(self.config, "output_past") and self.config.output_past
+        has_mem_len = hasattr(self.config, "mem_len") and self.config.mem_len
+
+        if has_output_past and not has_mem_len and len(outputs) > 1:
+            return True
+        elif has_mem_len and self.config.mem_len > 0 and len(outputs) > 1:
+            return True
+
+        return False
+
     @torch.no_grad()
     def generate(
         self,
@@ -556,48 +567,89 @@ class PreTrainedModel(nn.Module):
         length_penalty=None,
         num_return_sequences=None,
     ):
-        """ Sequence generator for models with a LM head.
-
-        The method currently supports greedy or penalized greedy decoding, sampling with top-k or nucleus sampling
+        r""" Generates sequences for models with a LM head. The method currently supports greedy or penalized greedy decoding, sampling with top-k or nucleus sampling
         and beam-search.
 
-        Adapted in part from Facebook's XLM beam search code: https://github.com/facebookresearch/XLM
+        Adapted in part from `Facebook's XLM beam search code`_.
 
-        Params:
-            **input_ids**: (`optional`) `torch.LongTensor` of shape (1, sequence_length)
+        .. _`Facebook's XLM beam search code`:
+           https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529
+
+
+        Parameters:
+
+            input_ids: (`optional`) `torch.LongTensor` of shape `(batch_size, sequence_length)`
                 The sequence used as a prompt for the generation. If `None` the method initializes
-                it as an empty `torch.LongTensor` of shape (1,)
-            **max_length**: (`optional`) int
+                it as an empty `torch.LongTensor` of shape `(1,)`.
+
+            max_length: (`optional`) int
                 The max length of the sequence to be generated.  Between 1 and infinity. Default to 20.
-            **do_sample**: (`optional`) bool
-                If set to `False` we use greedy decoding; otherwise sampling. Default to greedy sampling.
-            **num_beams**: (`optional`) int
-                Number of beams for beam search. 1 means no beam serach. Default to 1.
-            **temperature**: (`optional`) float
-                The value used to module the next token probabilities.
-            **top_k**: (`optional`) int
+
+            do_sample: (`optional`) bool
+                If set to `False` greedy decoding is used. Otherwise sampling is used. Default to greedy sampling.
+
+            num_beams: (`optional`) int
+                Number of beams for beam search. Must be between 1 and infinity. 1 means no beam search. Default to 1.
+
+            temperature: (`optional`) float
+                The value used to module the next token probabilities. Must be strictely positive. Default to 1.0.
+
+            top_k: (`optional`) int
                 The number of highest probability vocabulary tokens to keep for top-k-filtering. Between 1 and infinity. Default to 50.
-            **top_p**: (`optional`) float
+
+            top_p: (`optional`) float
                 The cumulative probability of parameter highest probability vocabulary tokens to keep for nucleus sampling. Must be between 0 and 1. Default to 1.
-            **repetition_penalty**: (`optional`) float
-                The parameter for repetition penalty. Between 1.0 and + infinity. 1.0 means no penalty. Default to 1.
-            **bos_token_id**: (`optional`) int
+
+            repetition_penalty: (`optional`) float
+                The parameter for repetition penalty. Between 1.0 and infinity. 1.0 means no penalty. Default to 1.0.
+
+            bos_token_id: (`optional`) int
                 Beginning of sentence token if no prompt is provided. Default to 0.
-            **eos_token_ids**: (`optional`) int or list of int
+
+            eos_token_ids: (`optional`) int or list of int
                 End of sequence token or list of tokens to stop the generation. Default to 0.
-            **length_penalty**: (`optional`) int
-                Exponential penalty to the length. Default to 0.
-            **length_penalty**: (`optional`) float
+            length_penalty: (`optional`) float
                 Exponential penalty to the length. Default to 1.
-            **num_return_sequences**: (`optional`) int
-                The number of independantly computed returned sequences for each element in the batch. Default to 1.
+
+            num_return_sequences: (`optional`) int
+                The number of independently computed returned sequences for each element in the batch. Default to 1.
+
+        Examples::
+
+            tokenizer = AutoTokenizer.from_pretrained('distilgpt2')   # Initialize tokenizer
+            model = AutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
+            outputs = model.generate(max_length=40, bos_token_id=tokenizer.bos_token_id, eos_token_ids=tokenizer.eos_token_id)  # do greedy decoding without beam search
+            print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))
+
+            tokenizer = AutoTokenizer.from_pretrained('openai-gpt')   # Initialize tokenizer
+            model = AutoModelWithLMHead.from_pretrained('openai-gpt')    # Download model and configuration from S3 and cache.
+            input_context = 'The dog'
+            input_ids = torch.tensor(tokenizer.encode(input_context)).unsqueeze(0)  # encode input context
+            outputs = model.generate(input_ids=input_ids, do_sample=True, num_beams=5, num_return_sequences=3, temperature=1.5)  # generate 3 independent sequences using beam search decoding (5 beams) with sampling from initial context 'The dog'
+            for i in range(3): #  3 output sequences were generated
+                print('Generated {}: {}'.format(i, tokenizer.decode(outputs[0][i], skip_special_tokens=True)))
+
+            tokenizer = AutoTokenizer.from_pretrained('distilgpt2')   # Initialize tokenizer
+            model = AutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
+            input_context = 'The dog'
+            input_ids = torch.tensor(tokenizer.encode(input_context)).unsqueeze(0)  # encode input context
+            outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, bos_token_id=tokenizer.bos_token_id, eos_token_ids=tokenizer.eos_token_id, num_beams=3)  # generate sequences using greedy beam search decoding (3 beams)
+            print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))
+
+            tokenizer = AutoTokenizer.from_pretrained('ctrl')   # Initialize tokenizer
+            model = AutoModelWithLMHead.from_pretrained('ctrl')    # Download model and configuration from S3 and cache.
+            input_context = 'Legal My neighbor is'  # "Legal" is one of the control codes for ctrl
+            input_ids = torch.tensor(tokenizer.encode(input_context)).unsqueeze(0)  # encode input context
+            outputs = model.generate(input_ids=input_ids, max_length=50, temperature=0.7, repetition_penalty=1.2)  # generate sequences using using greedy search
+            print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))
+
         """
 
         # We cannot generate if the model does not have a LM head
         if self.get_output_embeddings() is None:
             raise AttributeError(
                 "You tried to generate sequences with a model that does not have a LM Head."
-                "Please use another model class (e.g. `OpenAIGPTLMHeadModel`)"
+                "Please use another model class (e.g. `OpenAIGPTLMHeadModel`, `XLNetLMHeadModel`, `GPT2LMHeadModel`, `CTRLLMHeadModel`, `T5WithLMHeadModel`, `TransfoXLLMHeadModel`)"
             )
 
         max_length = max_length if max_length is not None else self.config.max_length
@@ -625,7 +677,7 @@ class PreTrainedModel(nn.Module):
         assert isinstance(max_length, int) and max_length > 0, "`max_length` should be a strictely positive integer."
         assert isinstance(do_sample, bool), "`do_sample` should be a boolean."
         assert isinstance(num_beams, int) and num_beams > 0, "`num_beams` should be a strictely positive integer."
-        # assert temperature >= 0, "`temperature` should be positive."
+        assert temperature > 0, "`temperature` should be strictely positive."
         assert isinstance(top_k, int) and top_k >= 0, "`top_k` should be a positive integer."
         assert 0 <= top_p <= 1, "`top_p` should be between 0 and 1."
         assert repetition_penalty >= 1.0, "`repetition_penalty` should be >= 1."
@@ -716,23 +768,30 @@ class PreTrainedModel(nn.Module):
         # current position / max lengths / length of generated sentences / unfinished sentences
         unfinished_sents = input_ids.new(batch_size).fill_(1)
 
-        # TODO: add cached compute states
-        pasts = None
+        past = None
 
         while cur_len < max_length:
-            model_inputs = self.prepare_inputs_for_generation(input_ids, pasts=pasts)
+            model_inputs = self.prepare_inputs_for_generation(input_ids, past=past)
             outputs = self(**model_inputs)
             next_token_logits = outputs[0][:, -1, :]
 
+            # if model has past, then set the past variable to speed up decoding
+            if self._do_output_past(outputs):
+                past = outputs[1]
+
             # repetition penalty from CTRL paper (https://arxiv.org/abs/1909.05858)
             if repetition_penalty != 1.0:
                 for i in range(batch_size):
-                    for previous_tokens in set(input_ids[i].tolist()):
-                        next_token_logits[i, previous_tokens] /= repetition_penalty
+                    for previous_token in set(input_ids[i].tolist()):
+                        # if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
+                        if next_token_logits[i, previous_token] < 0:
+                            next_token_logits[i, previous_token] *= repetition_penalty
+                        else:
+                            next_token_logits[i, previous_token] /= repetition_penalty
 
             if do_sample:
                 # Temperature (higher temperature => more likely to sample low probability tokens)
-                if temperature > 0 and temperature != 1.0:
+                if temperature != 1.0:
                     next_token_logits = next_token_logits / temperature
                 # Top-p/top-k filtering
                 next_token_logits = top_k_top_p_filtering(next_token_logits, top_k=top_k, top_p=top_p)
@@ -793,25 +852,33 @@ class PreTrainedModel(nn.Module):
         beam_scores = beam_scores.view(-1)  # shape (batch_size * num_beams,)
 
         # cache compute states
-        pasts = None  # self.prepare_pasts()
+        past = None
 
         # done sentences
         done = [False for _ in range(batch_size)]
 
         while cur_len < max_length:
-            model_inputs = self.prepare_inputs_for_generation(input_ids, pasts=pasts)
-            scores = self(**model_inputs)[0]  # (batch_size * num_beams, cur_len, vocab_size)
-            scores = scores[:, -1, :]  # (batch_size * num_beams, vocab_size)
+            model_inputs = self.prepare_inputs_for_generation(input_ids, past=past)
+            outputs = self(**model_inputs)  # (batch_size * num_beams, cur_len, vocab_size)
+            scores = outputs[0][:, -1, :]  # (batch_size * num_beams, vocab_size)
+
+            # if model has past, then set the past variable to speed up decoding
+            if self._do_output_past(outputs):
+                past = outputs[1]
 
             # repetition penalty (from CTRL paper https://arxiv.org/abs/1909.05858)
             if repetition_penalty != 1.0:
                 for i in range(batch_size * num_beams):
-                    for previous_tokens in set(input_ids[i].tolist()):
-                        scores[i, previous_tokens] /= repetition_penalty
+                    for previous_token in set(input_ids[i].tolist()):
+                        # if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
+                        if scores[i, previous_token] < 0:
+                            scores[i, previous_token] *= repetition_penalty
+                        else:
+                            scores[i, previous_token] /= repetition_penalty
 
             if do_sample:
                 # Temperature (higher temperature => more likely to sample low probability tokens)
-                if temperature > 0 and temperature != 1.0:
+                if temperature != 1.0:
                     scores = scores / temperature
                 # Top-p/top-k filtering
                 scores = top_k_top_p_filtering(
@@ -886,13 +953,22 @@ class PreTrainedModel(nn.Module):
             beam_words = input_ids.new([x[1] for x in next_batch_beam])
             beam_idx = input_ids.new([x[2] for x in next_batch_beam])
 
-            # re-order batch and internal states
+            # re-order batch
             input_ids = input_ids[beam_idx, :]
             input_ids = torch.cat([input_ids, beam_words.unsqueeze(1)], dim=-1)
-            # TODO: Activate cache
-            # for k in cache.keys():
-            #     if k != 'slen':
-            #         cache[k] = (cache[k][0][beam_idx], cache[k][1][beam_idx])
+
+            # re-order internal states
+            if past:
+                reordered_past = []
+                for layer_past in past:
+                    # get the correct batch idx from layer past batch dim
+                    # batch dim of `past` and `mems` is at 2nd position
+                    reordered_layer_past = [layer_past[:, i].unsqueeze(1).clone().detach() for i in beam_idx]
+                    reordered_layer_past = torch.cat(reordered_layer_past, dim=1)
+                    # check that shape matches
+                    assert reordered_layer_past.shape == layer_past.shape
+                    reordered_past.append(reordered_layer_past)
+                past = tuple(reordered_past)
 
             # update current length
             cur_len = cur_len + 1
@@ -958,9 +1034,7 @@ def top_k_top_p_filtering(logits, top_k=0, top_p=1.0, filter_value=-float("Inf")
         sorted_indices_to_remove[..., 0] = 0
 
         # scatter sorted tensors to original indexing
-        indices_to_remove = sorted_indices_to_remove.scatter(
-            dim=1, index=sorted_indices, source=sorted_indices_to_remove
-        )
+        indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
         logits[indices_to_remove] = filter_value
     return logits
 

src/transformers/modeling_xlm.py

@@ -674,7 +674,8 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
         mask_token_id = self.config.mask_token_id
         lang_id = self.config.lang_id
 
-        mask_token = torch.full((1, 1), mask_token_id, dtype=torch.long, device=input_ids.device)
+        effective_batch_size = input_ids.shape[0]
+        mask_token = torch.full((effective_batch_size, 1), mask_token_id, dtype=torch.long, device=input_ids.device)
         input_ids = torch.cat([input_ids, mask_token], dim=1)
         if lang_id is not None:
             langs = torch.full_like(input_ids, lang_id)
@@ -895,10 +896,7 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (
-            start_logits,
-            end_logits,
-        )
+        outputs = (start_logits, end_logits)
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/modeling_xlnet.py

@@ -1010,22 +1010,31 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
 
     def prepare_inputs_for_generation(self, input_ids, **model_kwargs):
         # Add dummy token at the end (no attention on this one)
-        dummy_token = torch.zeros((1, 1), dtype=torch.long, device=input_ids.device)
+
+        effective_batch_size = input_ids.shape[0]
+        dummy_token = torch.zeros((effective_batch_size, 1), dtype=torch.long, device=input_ids.device)
         input_ids = torch.cat([input_ids, dummy_token], dim=1)
 
         # Build permutation mask so that previous tokens don't see last token
+        sequence_length = input_ids.shape[1]
         perm_mask = torch.zeros(
-            (input_ids.shape[0], input_ids.shape[1], input_ids.shape[1]), dtype=torch.float, device=input_ids.device
+            (effective_batch_size, sequence_length, sequence_length), dtype=torch.float, device=input_ids.device
         )
         perm_mask[:, :, -1] = 1.0
 
         # We'll only predict the last token
         target_mapping = torch.zeros(
-            (input_ids.shape[0], 1, input_ids.shape[1]), dtype=torch.float, device=input_ids.device
+            (effective_batch_size, 1, sequence_length), dtype=torch.float, device=input_ids.device
         )
         target_mapping[0, 0, -1] = 1.0
 
-        return {"input_ids": input_ids, "perm_mask": perm_mask, "target_mapping": target_mapping}
+        inputs = {"input_ids": input_ids, "perm_mask": perm_mask, "target_mapping": target_mapping}
+
+        # if past is defined in model kwargs then use it for faster decoding
+        if "past" in model_kwargs and model_kwargs["past"]:
+            inputs["mems"] = model_kwargs["past"]
+
+        return inputs
 
     def forward(
         self,
@@ -1496,7 +1505,7 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

src/transformers/pipelines.py

@@ -643,7 +643,7 @@ class QuestionAnsweringPipeline(Pipeline):
             framework=framework,
             args_parser=QuestionAnsweringArgumentHandler(),
             device=device,
-            **kwargs
+            **kwargs,
         )
 
     @staticmethod

src/transformers/tokenization_albert.py

@@ -87,7 +87,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
 
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens

src/transformers/tokenization_bert.py

@@ -20,11 +20,15 @@ import logging
 import os
 import unicodedata
 
-import tokenizers as tk
-
+from .file_utils import is_fast_tokenizers_available
 from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
 
 
+if is_fast_tokenizers_available():
+    import tokenizers as tk
+else:
+    tk = None
+
 logger = logging.getLogger(__name__)
 
 VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
@@ -169,7 +173,7 @@ class BertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens
@@ -560,10 +564,18 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
 
-        self._tokenizer = tk.Tokenizer(tk.models.WordPiece.from_files(vocab_file, unk_token=unk_token))
+        if is_fast_tokenizers_available():
+            self._tokenizer = tk.Tokenizer(tk.models.WordPiece.from_files(vocab_file, unk_token=unk_token))
+        else:
+            logger.error(
+                "Using fast Tokenizers requires the `tokenizers` library. "
+                "Please install it with `pip install tokenizers`."
+            )
+            raise ImportError()
+
         self._update_special_tokens()
         self._tokenizer.with_pre_tokenizer(
             tk.pre_tokenizers.BertPreTokenizer.new(

src/transformers/tokenization_bert_japanese.py

@@ -113,7 +113,7 @@ class BertJapaneseTokenizer(BertTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens

src/transformers/tokenization_camembert.py

@@ -32,13 +32,11 @@ VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.bpe.model"}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {
-        "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model",
+        "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model"
     }
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "camembert-base": None,
-}
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"camembert-base": None}
 
 
 class CamembertTokenizer(PreTrainedTokenizer):
@@ -76,7 +74,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens

src/transformers/tokenization_ctrl.py

@@ -26,19 +26,14 @@ from .tokenization_utils import PreTrainedTokenizer
 
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
     "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "ctrl": 256,
-}
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"ctrl": 256}
 
 CONTROL_CODES = {
     "Pregnancy": 168629,

src/transformers/tokenization_gpt2.py

@@ -21,17 +21,19 @@ import os
 from functools import lru_cache
 
 import regex as re
-import tokenizers as tk
 
+from .file_utils import is_fast_tokenizers_available
 from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
 
 
+if is_fast_tokenizers_available():
+    import tokenizers as tk
+else:
+    tk = None
+
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {
@@ -272,7 +274,15 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
             bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs
         )
 
-        self._tokenizer = tk.Tokenizer(tk.models.BPE.from_files(vocab_file, merges_file))
+        if is_fast_tokenizers_available():
+            self._tokenizer = tk.Tokenizer(tk.models.BPE.from_files(vocab_file, merges_file))
+        else:
+            logger.error(
+                "Using fast Tokenizers requires the `tokenizers` library. "
+                "Please install it with `pip install tokenizers`."
+            )
+            raise ImportError()
+
         self._update_special_tokens()
         self._tokenizer.with_pre_tokenizer(tk.pre_tokenizers.ByteLevel.new(add_prefix_space=add_prefix_space))
         self._tokenizer.with_decoder(tk.decoders.ByteLevel.new())

src/transformers/tokenization_openai.py

@@ -26,19 +26,14 @@ from .tokenization_utils import PreTrainedTokenizer
 
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-vocab.json"},
     "merges_file": {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt"},
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "openai-gpt": 512,
-}
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"openai-gpt": 512}
 
 
 def get_pairs(word):

src/transformers/tokenization_roberta.py

@@ -22,10 +22,7 @@ from .tokenization_gpt2 import GPT2Tokenizer
 
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {
@@ -95,7 +92,7 @@ class RobertaTokenizer(GPT2Tokenizer):
             cls_token=cls_token,
             pad_token=pad_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens

src/transformers/tokenization_t5.py

@@ -96,7 +96,7 @@ class T5Tokenizer(PreTrainedTokenizer):
             unk_token=unk_token,
             pad_token=pad_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs
+            **kwargs,
         )
 
         try:

src/transformers/tokenization_transfo_xl.py

@@ -40,16 +40,14 @@ VOCAB_FILES_NAMES = {"pretrained_vocab_file": "vocab.bin", "vocab_file": "vocab.
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "pretrained_vocab_file": {
-        "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-vocab.bin",
+        "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-vocab.bin"
     }
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "transfo-xl-wt103": None,
-}
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"transfo-xl-wt103": None}
 
 PRETRAINED_CORPUS_ARCHIVE_MAP = {
-    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-corpus.bin",
+    "transfo-xl-wt103": "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-corpus.bin"
 }
 CORPUS_NAME = "corpus.bin"
 

src/transformers/tokenization_utils.py

@@ -469,6 +469,9 @@ class PreTrainedTokenizer(object):
         tokenizer.init_inputs = init_inputs
         tokenizer.init_kwargs = init_kwargs
 
+        # update unique_added_tokens_encoder with special tokens for correct tokenization
+        tokenizer.unique_added_tokens_encoder.update(set(tokenizer.all_special_tokens))
+
         # Add supplementary tokens.
         if added_tokens_file is not None:
             with open(added_tokens_file, encoding="utf-8") as added_tokens_handle:
@@ -476,6 +479,7 @@ class PreTrainedTokenizer(object):
             added_tok_decoder = {v: k for k, v in added_tok_encoder.items()}
             tokenizer.added_tokens_encoder.update(added_tok_encoder)
             tokenizer.added_tokens_decoder.update(added_tok_decoder)
+            tokenizer.unique_added_tokens_encoder.update(set(tokenizer.added_tokens_encoder.keys()))
 
         return tokenizer
 
@@ -813,7 +817,7 @@ class PreTrainedTokenizer(object):
             truncation_strategy=truncation_strategy,
             pad_to_max_length=pad_to_max_length,
             return_tensors=return_tensors,
-            **kwargs
+            **kwargs,
         )
 
         return encoded_inputs["input_ids"]
@@ -866,7 +870,7 @@ class PreTrainedTokenizer(object):
             return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant
                 or PyTorch torch.Tensor instead of a list of python integers.
             return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True).
-            return_attention_mask: (optional) Set to False to avoir returning attention mask (default True)
+            return_attention_mask: (optional) Set to False to avoid returning attention mask (default True)
             return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False).
             return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False).
             **kwargs: passed to the `self.tokenize()` method
@@ -1491,9 +1495,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer):
         return_overflowing_tokens=False,
         return_special_tokens_mask=False,
     ):
-        encoding_dict = {
-            "input_ids": encoding.ids,
-        }
+        encoding_dict = {"input_ids": encoding.ids}
         if return_token_type_ids:
             encoding_dict["token_type_ids"] = encoding.type_ids
         if return_attention_mask:

src/transformers/tokenization_xlm.py

@@ -29,10 +29,7 @@ from .tokenization_utils import PreTrainedTokenizer
 
 logger = logging.getLogger(__name__)
 
-VOCAB_FILES_NAMES = {
-    "vocab_file": "vocab.json",
-    "merges_file": "merges.txt",
-}
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"}
 
 PRETRAINED_VOCAB_FILES_MAP = {
     "vocab_file": {
@@ -586,7 +583,7 @@ class XLMTokenizer(PreTrainedTokenizer):
             cls_token=cls_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs
+            **kwargs,
         )
 
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens

src/transformers/tokenization_xlm_roberta.py

@@ -83,7 +83,7 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
             cls_token=cls_token,
             pad_token=pad_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens

src/transformers/tokenization_xlnet.py

@@ -34,10 +34,7 @@ PRETRAINED_VOCAB_FILES_MAP = {
     }
 }
 
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "xlnet-base-cased": None,
-    "xlnet-large-cased": None,
-}
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"xlnet-base-cased": None, "xlnet-large-cased": None}
 
 SPIECE_UNDERLINE = "▁"
 
@@ -86,7 +83,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
             cls_token=cls_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs
+            **kwargs,
         )
 
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens

templates/adding_a_new_example_script/utils_xxx.py

@@ -868,8 +868,6 @@ def write_predictions_extended(
         orig_data = json.load(reader)["data"]
 
     qid_to_has_ans = make_qid_to_has_ans(orig_data)
-    has_ans_qids = [k for k, v in qid_to_has_ans.items() if v]
-    no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v]
     exact_raw, f1_raw = get_raw_scores(orig_data, all_predictions)
     out_eval = {}
 

templates/adding_a_new_model/modeling_tf_xxx.py

@@ -527,6 +527,6 @@ class TFXxxForQuestionAnswering(TFXxxPreTrainedModel):
         start_logits = tf.squeeze(start_logits, axis=-1)
         end_logits = tf.squeeze(end_logits, axis=-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
 
         return outputs  # start_logits, end_logits, (hidden_states), (attentions)

templates/adding_a_new_model/modeling_xxx.py

@@ -728,7 +728,7 @@ class XxxForQuestionAnswering(XxxPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (start_logits, end_logits,) + outputs[2:]
+        outputs = (start_logits, end_logits) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

templates/adding_a_new_model/tests/test_modeling_tf_xxx.py

@@ -144,10 +144,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output.numpy(),
-                "pooled_output": pooled_output.numpy(),
-            }
+            result = {"sequence_output": sequence_output.numpy(), "pooled_output": pooled_output.numpy()}
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -159,9 +156,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForMaskedLM(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (prediction_scores,) = model(inputs)
-            result = {
-                "prediction_scores": prediction_scores.numpy(),
-            }
+            result = {"prediction_scores": prediction_scores.numpy()}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -173,9 +168,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForSequenceClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_labels])
 
         def create_and_check_xxx_for_token_classification(
@@ -185,9 +178,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForTokenClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, self.num_labels]
             )
@@ -198,10 +189,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForQuestionAnswering(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             start_logits, end_logits = model(inputs)
-            result = {
-                "start_logits": start_logits.numpy(),
-                "end_logits": end_logits.numpy(),
-            }
+            result = {"start_logits": start_logits.numpy(), "end_logits": end_logits.numpy()}
             self.parent.assertListEqual(list(result["start_logits"].shape), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].shape), [self.batch_size, self.seq_length])
 

templates/adding_a_new_model/tests/test_modeling_xxx.py

@@ -141,10 +141,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
             sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -159,10 +156,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
             loss, prediction_scores = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels
             )
-            result = {
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
+            result = {"loss": loss, "prediction_scores": prediction_scores}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -181,11 +175,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
                 start_positions=sequence_labels,
                 end_positions=sequence_labels,
             )
-            result = {
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -200,10 +190,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels
             )
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 
@@ -217,10 +204,7 @@ class XxxModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels
             )
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
             self.parent.assertListEqual(
                 list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]
             )

templates/adding_a_new_model/tokenization_xxx.py

@@ -48,10 +48,7 @@ PRETRAINED_VOCAB_FILES_MAP = {
 ####################################################
 # Mapping from model shortcut names to max length of inputs
 ####################################################
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "xxx-base-uncased": 512,
-    "xxx-large-uncased": 512,
-}
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"xxx-base-uncased": 512, "xxx-large-uncased": 512}
 
 ####################################################
 # Mapping from model shortcut names to a dictionary of additional
@@ -115,7 +112,7 @@ class XxxTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs
+            **kwargs,
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens

tests/test_modeling_albert.py

@@ -141,10 +141,7 @@ class AlbertModelTest(ModelTesterMixin, unittest.TestCase):
             sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -159,10 +156,7 @@ class AlbertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, prediction_scores = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels
             )
-            result = {
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
+            result = {"loss": loss, "prediction_scores": prediction_scores}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -181,11 +175,7 @@ class AlbertModelTest(ModelTesterMixin, unittest.TestCase):
                 start_positions=sequence_labels,
                 end_positions=sequence_labels,
             )
-            result = {
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -200,10 +190,7 @@ class AlbertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels
             )
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 

tests/test_modeling_bert.py

@@ -180,10 +180,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -219,10 +216,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             )
             sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
 
-            result = {
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -237,10 +231,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, prediction_scores = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels
             )
-            result = {
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
+            result = {"loss": loss, "prediction_scores": prediction_scores}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -276,10 +267,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
                 masked_lm_labels=token_labels,
                 encoder_hidden_states=encoder_hidden_states,
             )
-            result = {
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
+            result = {"loss": loss, "prediction_scores": prediction_scores}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -297,10 +285,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
                 token_type_ids=token_type_ids,
                 next_sentence_label=sequence_labels,
             )
-            result = {
-                "loss": loss,
-                "seq_relationship_score": seq_relationship_score,
-            }
+            result = {"loss": loss, "seq_relationship_score": seq_relationship_score}
             self.parent.assertListEqual(list(result["seq_relationship_score"].size()), [self.batch_size, 2])
             self.check_loss_output(result)
 
@@ -341,11 +326,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
                 start_positions=sequence_labels,
                 end_positions=sequence_labels,
             )
-            result = {
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -360,10 +341,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels
             )
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 
@@ -377,10 +355,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels
             )
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
             self.parent.assertListEqual(
                 list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]
             )
@@ -402,10 +377,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
                 token_type_ids=multiple_choice_token_type_ids,
                 labels=choice_labels,
             )
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_choices])
             self.check_loss_output(result)
 

tests/test_modeling_common.py

@@ -284,7 +284,6 @@ class ModelTesterMixin:
             multihead_outputs = head_mask.grad
 
             attentions = outputs[-1]
-            hidden_states = outputs[-2]
 
             # Remove Nan
             for t in attentions:
@@ -590,7 +589,7 @@ class ModelTesterMixin:
                 inputs_dict["decoder_inputs_embeds"] = wte(decoder_input_ids)
 
             with torch.no_grad():
-                outputs = model(**inputs_dict)
+                model(**inputs_dict)
 
 
 class ConfigTester(object):

tests/test_modeling_ctrl.py

@@ -150,10 +150,7 @@ class CTRLModelTest(ModelTesterMixin, unittest.TestCase):
             model(input_ids, token_type_ids=token_type_ids)
             sequence_output, presents = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output,
-                "presents": presents,
-            }
+            result = {"sequence_output": sequence_output, "presents": presents}
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )

tests/test_modeling_distilbert.py

@@ -138,9 +138,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             (sequence_output,) = model(input_ids, input_mask)
             (sequence_output,) = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output,
-            }
+            result = {"sequence_output": sequence_output}
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -152,10 +150,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             model.to(torch_device)
             model.eval()
             loss, prediction_scores = model(input_ids, attention_mask=input_mask, masked_lm_labels=token_labels)
-            result = {
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
+            result = {"loss": loss, "prediction_scores": prediction_scores}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -170,11 +165,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             loss, start_logits, end_logits = model(
                 input_ids, attention_mask=input_mask, start_positions=sequence_labels, end_positions=sequence_labels
             )
-            result = {
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -187,10 +178,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             model.to(torch_device)
             model.eval()
             loss, logits = model(input_ids, attention_mask=input_mask, labels=sequence_labels)
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
             self.parent.assertListEqual(list(result["logits"].size()), [self.batch_size, self.num_labels])
             self.check_loss_output(result)
 
@@ -203,10 +191,7 @@ class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):
             model.eval()
 
             loss, logits = model(input_ids, attention_mask=input_mask, labels=token_labels)
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
             self.parent.assertListEqual(
                 list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]
             )

tests/test_modeling_gpt2.py

@@ -153,10 +153,7 @@ class GPT2ModelTest(ModelTesterMixin, unittest.TestCase):
             model(input_ids, token_type_ids=token_type_ids)
             sequence_output, presents = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output,
-                "presents": presents,
-            }
+            result = {"sequence_output": sequence_output, "presents": presents}
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )

tests/test_modeling_roberta.py

@@ -138,10 +138,7 @@ class RobertaModelTest(ModelTesterMixin, unittest.TestCase):
             sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids)
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output,
-                "pooled_output": pooled_output,
-            }
+            result = {"sequence_output": sequence_output, "pooled_output": pooled_output}
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -156,10 +153,7 @@ class RobertaModelTest(ModelTesterMixin, unittest.TestCase):
             loss, prediction_scores = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels
             )
-            result = {
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
+            result = {"loss": loss, "prediction_scores": prediction_scores}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -175,10 +169,7 @@ class RobertaModelTest(ModelTesterMixin, unittest.TestCase):
             loss, logits = model(
                 input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels
             )
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
             self.parent.assertListEqual(
                 list(result["logits"].size()), [self.batch_size, self.seq_length, self.num_labels]
             )

tests/test_modeling_t5.py

@@ -136,10 +136,7 @@ class T5ModelTest(ModelTesterMixin, unittest.TestCase):
                 encoder_input_ids=encoder_input_ids, decoder_input_ids=decoder_input_ids
             )
 
-            result = {
-                "encoder_output": encoder_output,
-                "decoder_output": decoder_output,
-            }
+            result = {"encoder_output": encoder_output, "decoder_output": decoder_output}
             self.parent.assertListEqual(
                 list(result["encoder_output"].size()), [self.batch_size, self.encoder_seq_length, self.hidden_size]
             )
@@ -165,10 +162,7 @@ class T5ModelTest(ModelTesterMixin, unittest.TestCase):
                 decoder_lm_labels=decoder_lm_labels,
             )
             loss, prediction_scores = outputs[0], outputs[1]
-            result = {
-                "loss": loss,
-                "prediction_scores": prediction_scores,
-            }
+            result = {"loss": loss, "prediction_scores": prediction_scores}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].size()), [self.batch_size, self.decoder_seq_length, self.vocab_size]
             )

tests/test_modeling_tf_albert.py

@@ -141,10 +141,7 @@ class TFAlbertModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output.numpy(),
-                "pooled_output": pooled_output.numpy(),
-            }
+            result = {"sequence_output": sequence_output.numpy(), "pooled_output": pooled_output.numpy()}
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -156,9 +153,7 @@ class TFAlbertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFAlbertForMaskedLM(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (prediction_scores,) = model(inputs)
-            result = {
-                "prediction_scores": prediction_scores.numpy(),
-            }
+            result = {"prediction_scores": prediction_scores.numpy()}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -170,9 +165,7 @@ class TFAlbertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFAlbertForSequenceClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_labels])
 
         def prepare_config_and_inputs_for_common(self):

tests/test_modeling_tf_bert.py

@@ -150,10 +150,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output, pooled_output = model(input_ids)
 
-            result = {
-                "sequence_output": sequence_output.numpy(),
-                "pooled_output": pooled_output.numpy(),
-            }
+            result = {"sequence_output": sequence_output.numpy(), "pooled_output": pooled_output.numpy()}
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -165,9 +162,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForMaskedLM(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (prediction_scores,) = model(inputs)
-            result = {
-                "prediction_scores": prediction_scores.numpy(),
-            }
+            result = {"prediction_scores": prediction_scores.numpy()}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -178,9 +173,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForNextSentencePrediction(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (seq_relationship_score,) = model(inputs)
-            result = {
-                "seq_relationship_score": seq_relationship_score.numpy(),
-            }
+            result = {"seq_relationship_score": seq_relationship_score.numpy()}
             self.parent.assertListEqual(list(result["seq_relationship_score"].shape), [self.batch_size, 2])
 
         def create_and_check_bert_for_pretraining(
@@ -205,9 +198,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForSequenceClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_labels])
 
         def create_and_check_bert_for_multiple_choice(
@@ -224,9 +215,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
                 "token_type_ids": multiple_choice_token_type_ids,
             }
             (logits,) = model(inputs)
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_choices])
 
         def create_and_check_bert_for_token_classification(
@@ -236,9 +225,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForTokenClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, self.num_labels]
             )
@@ -249,10 +236,7 @@ class TFBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFBertForQuestionAnswering(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             start_logits, end_logits = model(inputs)
-            result = {
-                "start_logits": start_logits.numpy(),
-                "end_logits": end_logits.numpy(),
-            }
+            result = {"start_logits": start_logits.numpy(), "end_logits": end_logits.numpy()}
             self.parent.assertListEqual(list(result["start_logits"].shape), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].shape), [self.batch_size, self.seq_length])
 

tests/test_modeling_tf_common.py

@@ -332,7 +332,7 @@ class TFModelTesterMixin:
                 inputs_dict["encoder_inputs_embeds"] = self._get_embeds(wte, encoder_input_ids)
                 inputs_dict["decoder_inputs_embeds"] = self._get_embeds(wte, decoder_input_ids)
 
-            outputs = model(inputs_dict)
+            model(inputs_dict)
 
 
 def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None):

tests/test_modeling_tf_ctrl.py

@@ -145,9 +145,7 @@ class TFCTRLModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output = model(input_ids)[0]
 
-            result = {
-                "sequence_output": sequence_output.numpy(),
-            }
+            result = {"sequence_output": sequence_output.numpy()}
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -156,9 +154,7 @@ class TFCTRLModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFCTRLLMHeadModel(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             prediction_scores = model(inputs)[0]
-            result = {
-                "prediction_scores": prediction_scores.numpy(),
-            }
+            result = {"prediction_scores": prediction_scores.numpy()}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )

tests/test_modeling_tf_distilbert.py

@@ -142,9 +142,7 @@ class TFDistilBertModelTest(TFModelTesterMixin, unittest.TestCase):
 
             (sequence_output,) = model(inputs)
 
-            result = {
-                "sequence_output": sequence_output.numpy(),
-            }
+            result = {"sequence_output": sequence_output.numpy()}
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -155,9 +153,7 @@ class TFDistilBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFDistilBertForMaskedLM(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask}
             (prediction_scores,) = model(inputs)
-            result = {
-                "prediction_scores": prediction_scores.numpy(),
-            }
+            result = {"prediction_scores": prediction_scores.numpy()}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -168,10 +164,7 @@ class TFDistilBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFDistilBertForQuestionAnswering(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask}
             start_logits, end_logits = model(inputs)
-            result = {
-                "start_logits": start_logits.numpy(),
-                "end_logits": end_logits.numpy(),
-            }
+            result = {"start_logits": start_logits.numpy(), "end_logits": end_logits.numpy()}
             self.parent.assertListEqual(list(result["start_logits"].shape), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].shape), [self.batch_size, self.seq_length])
 
@@ -182,9 +175,7 @@ class TFDistilBertModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFDistilBertForSequenceClassification(config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask}
             (logits,) = model(inputs)
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_labels])
 
         def prepare_config_and_inputs_for_common(self):

tests/test_modeling_tf_gpt2.py

@@ -152,9 +152,7 @@ class TFGPT2ModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output = model(input_ids)[0]
 
-            result = {
-                "sequence_output": sequence_output.numpy(),
-            }
+            result = {"sequence_output": sequence_output.numpy()}
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -163,9 +161,7 @@ class TFGPT2ModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFGPT2LMHeadModel(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             prediction_scores = model(inputs)[0]
-            result = {
-                "prediction_scores": prediction_scores.numpy(),
-            }
+            result = {"prediction_scores": prediction_scores.numpy()}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )

tests/test_modeling_tf_openai_gpt.py

@@ -153,9 +153,7 @@ class TFOpenAIGPTModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output = model(input_ids)[0]
 
-            result = {
-                "sequence_output": sequence_output.numpy(),
-            }
+            result = {"sequence_output": sequence_output.numpy()}
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -164,9 +162,7 @@ class TFOpenAIGPTModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFOpenAIGPTLMHeadModel(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             prediction_scores = model(inputs)[0]
-            result = {
-                "prediction_scores": prediction_scores.numpy(),
-            }
+            result = {"prediction_scores": prediction_scores.numpy()}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )

tests/test_modeling_tf_roberta.py

@@ -138,9 +138,7 @@ class TFRobertaModelTest(TFModelTesterMixin, unittest.TestCase):
 
             sequence_output = model(input_ids)[0]
 
-            result = {
-                "sequence_output": sequence_output.numpy(),
-            }
+            result = {"sequence_output": sequence_output.numpy()}
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -150,9 +148,7 @@ class TFRobertaModelTest(TFModelTesterMixin, unittest.TestCase):
         ):
             model = TFRobertaForMaskedLM(config=config)
             prediction_scores = model([input_ids, input_mask, token_type_ids])[0]
-            result = {
-                "prediction_scores": prediction_scores.numpy(),
-            }
+            result = {"prediction_scores": prediction_scores.numpy()}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )
@@ -164,9 +160,7 @@ class TFRobertaModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFRobertaForTokenClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             (logits,) = model(inputs)
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, self.num_labels]
             )

tests/test_modeling_tf_t5.py

@@ -109,10 +109,7 @@ class TFT5ModelTest(TFModelTesterMixin, unittest.TestCase):
                 input_ids, decoder_attention_mask=input_mask, encoder_input_ids=input_ids
             )
 
-            result = {
-                "encoder_output": encoder_output.numpy(),
-                "decoder_output": decoder_output.numpy(),
-            }
+            result = {"encoder_output": encoder_output.numpy(), "decoder_output": decoder_output.numpy()}
             self.parent.assertListEqual(
                 list(result["encoder_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -128,9 +125,7 @@ class TFT5ModelTest(TFModelTesterMixin, unittest.TestCase):
                 "decoder_attention_mask": input_mask,
             }
             prediction_scores, decoder_output = model(inputs)
-            result = {
-                "prediction_scores": prediction_scores.numpy(),
-            }
+            result = {"prediction_scores": prediction_scores.numpy()}
             self.parent.assertListEqual(
                 list(result["prediction_scores"].shape), [self.batch_size, self.seq_length, self.vocab_size]
             )

tests/test_modeling_tf_xlm.py

@@ -176,9 +176,7 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
             inputs = [input_ids, input_mask]
             outputs = model(inputs)
             sequence_output = outputs[0]
-            result = {
-                "sequence_output": sequence_output.numpy(),
-            }
+            result = {"sequence_output": sequence_output.numpy()}
             self.parent.assertListEqual(
                 list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -201,9 +199,7 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
 
             logits = outputs[0]
 
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
 
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, self.vocab_size]
@@ -224,13 +220,9 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
 
             inputs = {"input_ids": input_ids, "lengths": input_lengths}
 
-            outputs = model(inputs)
             start_logits, end_logits = model(inputs)
 
-            result = {
-                "start_logits": start_logits.numpy(),
-                "end_logits": end_logits.numpy(),
-            }
+            result = {"start_logits": start_logits.numpy(), "end_logits": end_logits.numpy()}
 
             self.parent.assertListEqual(list(result["start_logits"].shape), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].shape), [self.batch_size, self.seq_length])
@@ -252,9 +244,7 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
 
             (logits,) = model(inputs)
 
-            result = {
-                "logits": logits.numpy(),
-            }
+            result = {"logits": logits.numpy()}
 
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.type_sequence_label_size])
 

tests/test_modeling_tf_xlnet.py

@@ -183,10 +183,7 @@ class TFXLNetModelTest(TFModelTesterMixin, unittest.TestCase):
 
             outputs, mems_1 = model(inputs)
 
-            result = {
-                "mems_1": [mem.numpy() for mem in mems_1],
-                "outputs": outputs.numpy(),
-            }
+            result = {"mems_1": [mem.numpy() for mem in mems_1], "outputs": outputs.numpy()}
 
             config.mem_len = 0
             model = TFXLNetModel(config)
@@ -302,10 +299,7 @@ class TFXLNetModelTest(TFModelTesterMixin, unittest.TestCase):
 
             logits, mems_1 = model(input_ids_1)
 
-            result = {
-                "mems_1": [mem.numpy() for mem in mems_1],
-                "logits": logits.numpy(),
-            }
+            result = {"mems_1": [mem.numpy() for mem in mems_1], "logits": logits.numpy()}
 
             self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.type_sequence_label_size])
             self.parent.assertListEqual(
@@ -335,10 +329,7 @@ class TFXLNetModelTest(TFModelTesterMixin, unittest.TestCase):
                 # 'token_type_ids': token_type_ids
             }
             logits, mems_1 = model(inputs)
-            result = {
-                "mems_1": [mem.numpy() for mem in mems_1],
-                "logits": logits.numpy(),
-            }
+            result = {"mems_1": [mem.numpy() for mem in mems_1], "logits": logits.numpy()}
             self.parent.assertListEqual(
                 list(result["logits"].shape), [self.batch_size, self.seq_length, config.num_labels]
             )

tests/test_modeling_xlm.py

@@ -185,9 +185,7 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
             outputs = model(input_ids, langs=token_type_ids)
             outputs = model(input_ids)
             sequence_output = outputs[0]
-            result = {
-                "sequence_output": sequence_output,
-            }
+            result = {"sequence_output": sequence_output}
             self.parent.assertListEqual(
                 list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size]
             )
@@ -209,10 +207,7 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
 
             loss, logits = model(input_ids, token_type_ids=token_type_ids, labels=token_labels)
 
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(
@@ -239,11 +234,7 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
             outputs = model(input_ids, start_positions=sequence_labels, end_positions=sequence_labels)
             loss, start_logits, end_logits = outputs
 
-            result = {
-                "loss": loss,
-                "start_logits": start_logits,
-                "end_logits": end_logits,
-            }
+            result = {"loss": loss, "start_logits": start_logits, "end_logits": end_logits}
             self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length])
             self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length])
             self.check_loss_output(result)
@@ -333,10 +324,7 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
             (logits,) = model(input_ids)
             loss, logits = model(input_ids, labels=sequence_labels)
 
-            result = {
-                "loss": loss,
-                "logits": logits,
-            }
+            result = {"loss": loss, "logits": logits}
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(

tests/test_modeling_xlnet.py

@@ -187,10 +187,7 @@ class XLNetModelTest(ModelTesterMixin, unittest.TestCase):
             _, _ = model(input_ids_1, token_type_ids=segment_ids)
             outputs, mems_1 = model(input_ids_1)
 
-            result = {
-                "mems_1": mems_1,
-                "outputs": outputs,
-            }
+            result = {"mems_1": mems_1, "outputs": outputs}
 
             config.mem_len = 0
             model = XLNetModel(config)
@@ -385,11 +382,7 @@ class XLNetModelTest(ModelTesterMixin, unittest.TestCase):
             logits, mems_1 = model(input_ids_1)
             loss, logits, mems_1 = model(input_ids_1, labels=token_labels)
 
-            result = {
-                "loss": loss,
-                "mems_1": mems_1,
-                "logits": logits,
-            }
+            result = {"loss": loss, "mems_1": mems_1, "logits": logits}
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(
@@ -422,11 +415,7 @@ class XLNetModelTest(ModelTesterMixin, unittest.TestCase):
             logits, mems_1 = model(input_ids_1)
             loss, logits, mems_1 = model(input_ids_1, labels=sequence_labels)
 
-            result = {
-                "loss": loss,
-                "mems_1": mems_1,
-                "logits": logits,
-            }
+            result = {"loss": loss, "mems_1": mems_1, "logits": logits}
 
             self.parent.assertListEqual(list(result["loss"].size()), [])
             self.parent.assertListEqual(

tests/test_tokenization_bert.py

@@ -84,7 +84,7 @@ class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         tokenizer = self.get_tokenizer()
         rust_tokenizer = self.get_rust_tokenizer(add_special_tokens=False)
 
-        sequence = u"UNwant\u00E9d,running"
+        sequence = "UNwant\u00E9d,running"
 
         tokens = tokenizer.tokenize(sequence)
         rust_tokens = rust_tokenizer.tokenize(sequence)

tests/test_tokenization_common.py

@@ -42,15 +42,7 @@ class TokenizerTesterMixin:
 
     def test_tokenizers_common_properties(self):
         tokenizer = self.get_tokenizer()
-        attributes_list = [
-            "bos_token",
-            "eos_token",
-            "unk_token",
-            "sep_token",
-            "pad_token",
-            "cls_token",
-            "mask_token",
-        ]
+        attributes_list = ["bos_token", "eos_token", "unk_token", "sep_token", "pad_token", "cls_token", "mask_token"]
         for attr in attributes_list:
             self.assertTrue(hasattr(tokenizer, attr))
             self.assertTrue(hasattr(tokenizer, attr + "_id"))
@@ -163,7 +155,6 @@ class TokenizerTesterMixin:
         self.assertEqual(all_size_2, all_size + len(new_toks))
 
         tokens = tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l", add_special_tokens=False)
-        out_string = tokenizer.decode(tokens)
 
         self.assertGreaterEqual(len(tokens), 4)
         self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
@@ -182,7 +173,6 @@ class TokenizerTesterMixin:
         tokens = tokenizer.encode(
             ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l", add_special_tokens=False
         )
-        out_string = tokenizer.decode(tokens)
 
         self.assertGreaterEqual(len(tokens), 6)
         self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
@@ -282,7 +272,7 @@ class TokenizerTesterMixin:
         num_added_tokens = tokenizer.num_added_tokens()
         total_length = len(sequence) + num_added_tokens
         information = tokenizer.encode_plus(
-            seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride, return_overflowing_tokens=True,
+            seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride, return_overflowing_tokens=True
         )
 
         truncated_sequence = information["input_ids"]
@@ -305,7 +295,7 @@ class TokenizerTesterMixin:
 
         sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)
         truncated_second_sequence = tokenizer.build_inputs_with_special_tokens(
-            tokenizer.encode(seq_0, add_special_tokens=False), tokenizer.encode(seq_1, add_special_tokens=False)[:-2],
+            tokenizer.encode(seq_0, add_special_tokens=False), tokenizer.encode(seq_1, add_special_tokens=False)[:-2]
         )
 
         information = tokenizer.encode_plus(

tests/test_tokenization_gpt2.py

@@ -96,7 +96,7 @@ class GPT2TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         tokenizer = self.get_tokenizer()
         rust_tokenizer = self.get_rust_tokenizer(add_special_tokens=False, add_prefix_space=True)
 
-        sequence = u"lower newer"
+        sequence = "lower newer"
 
         # Testing tokenization
         tokens = tokenizer.tokenize(sequence, add_prefix_space=True)

tests/test_tokenization_transfo_xl.py

@@ -35,19 +35,7 @@ class TransfoXLTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
     def setUp(self):
         super(TransfoXLTokenizationTest, self).setUp()
 
-        vocab_tokens = [
-            "<unk>",
-            "[CLS]",
-            "[SEP]",
-            "want",
-            "unwanted",
-            "wa",
-            "un",
-            "running",
-            ",",
-            "low",
-            "l",
-        ]
+        vocab_tokens = ["<unk>", "[CLS]", "[SEP]", "want", "unwanted", "wa", "un", "running", ",", "low", "l"]
         self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
         with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))