code formating

Cli tweaks

CONTRIBUTING.md

@@ -121,7 +121,7 @@ Follow these steps to start contributing:
 4. Set up a development environment by running the following command in a virtual environment:
 
    ```bash
-   $ pip install -e .[dev]
+   $ pip install -e ".[dev]"
    ```
 
    (If transformers was already installed in the virtual environment, remove

README.md

@@ -120,14 +120,14 @@ Depending on which framework is installed (TensorFlow 2.0 and/or PyTorch), the i
 Here's the easiest way to run tests for the library:
 
 ```bash
-pip install -e .[testing]
+pip install -e ".[testing]"
 make test
 ```
 
 and for the examples:
 
 ```bash
-pip install -e .[testing]
+pip install -e ".[testing]"
 pip install -r examples/requirements.txt
 make test-examples
 ```
@@ -499,7 +499,7 @@ model = AutoModel.from_pretrained("username/pretrained_model")
 
 Finally, list all your files on S3:
 ```shell
-transformers-cli ls
+transformers-cli s3 ls
 # List all your S3 objects.
 ```
 

docs/README.md

@@ -4,7 +4,7 @@ To generate the documentation, you first have to build it. Several packages are
 you can install them with the following command, at the root of the code repository:
 
 ```bash
-pip install -e .[docs]
+pip install -e ".[docs]"
 ```
 
 ## Packages installed

docs/source/model_sharing.md

@@ -34,7 +34,7 @@ model = AutoModel.from_pretrained("username/pretrained_model")
 
 Finally, list all your files on S3:
 ```shell
-transformers-cli ls
+transformers-cli s3 ls
 # List all your S3 objects.
 ```
 

examples/contrib/run_openai_gpt.py

@@ -44,13 +44,10 @@ from transformers import (
     AdamW,
     OpenAIGPTDoubleHeadsModel,
     OpenAIGPTTokenizer,
-    cached_path,
     get_linear_schedule_with_warmup,
 )
 
 
-ROCSTORIES_URL = "https://s3.amazonaws.com/datasets.huggingface.co/ROCStories.tar.gz"
-
 logging.basicConfig(
     format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
 )
@@ -182,9 +179,6 @@ def main():
     model.to(device)
 
     # Load and encode the datasets
-    if not args.train_dataset and not args.eval_dataset:
-        roc_stories = cached_path(ROCSTORIES_URL)
-
     def tokenize_and_encode(obj):
         """ Tokenize and encode a nested object """
         if isinstance(obj, str):

examples/contrib/run_transfo_xl.py

@@ -28,7 +28,7 @@ import time
 
 import torch
 
-from transformers import TransfoXLCorpus, TransfoXLLMHeadModel, TransfoXLTokenizer
+from transformers import TransfoXLCorpus, TransfoXLLMHeadModel
 
 
 logging.basicConfig(
@@ -73,9 +73,7 @@ def main():
     # The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
     # and tokenizing the dataset
     # The pre-processed corpus is a convertion (using the conversion script )
-    tokenizer = TransfoXLTokenizer.from_pretrained(args.model_name)
     corpus = TransfoXLCorpus.from_pretrained(args.model_name)
-    ntokens = len(corpus.vocab)
 
     va_iter = corpus.get_iterator("valid", args.batch_size, args.tgt_len, device=device, ext_len=args.ext_len)
     te_iter = corpus.get_iterator("test", args.batch_size, args.tgt_len, device=device, ext_len=args.ext_len)

examples/distillation/run_squad_w_distillation.py

@@ -56,7 +56,6 @@ from ..utils_squad import (
     write_predictions,
     write_predictions_extended,
 )
-
 # The follwing import is the official SQuAD evaluation script (2.0).
 # You can remove it from the dependencies if you are using this script outside of the library
 # We've added it here for automated tests (see examples/test_examples.py file)

examples/run_lm_finetuning.py

@@ -84,7 +84,7 @@ class TextDataset(Dataset):
         assert os.path.isfile(file_path)
         directory, filename = os.path.split(file_path)
         cached_features_file = os.path.join(
-            directory, args.model_name_or_path + "_cached_lm_" + str(block_size) + "_" + filename
+            directory, args.model_name_or_path + "_cached_lm_" + str(block_size) + "_" + filename + ".bin"
         )
 
         if os.path.exists(cached_features_file) and not args.overwrite_cache:
@@ -625,6 +625,12 @@ def main():
         args.config_name if args.config_name else args.model_name_or_path,
         cache_dir=args.cache_dir if args.cache_dir else None,
     )
+
+    # Desactivate past output for now (reduce memory)
+    # we use it only for GPT/GPT2 generation or specific XLNet/Transformer-XL trainings (not implemented currently)
+    if hasattr(config, "output_past"):
+        config.output_past = False
+
     tokenizer = tokenizer_class.from_pretrained(
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,

examples/run_multiple_choice.py

@@ -141,7 +141,7 @@ def train(args, train_dataset, model, tokenizer):
 
     global_step = 0
     tr_loss, logging_loss = 0.0, 0.0
-    best_dev_acc, best_dev_loss = 0.0, 99999999999.0
+    best_dev_acc = 0.0
     best_steps = 0
     model.zero_grad()
     train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
@@ -193,7 +193,6 @@ def train(args, train_dataset, model, tokenizer):
                             tb_writer.add_scalar("eval_{}".format(key), value, global_step)
                         if results["eval_acc"] > best_dev_acc:
                             best_dev_acc = results["eval_acc"]
-                            best_dev_loss = results["eval_loss"]
                             best_steps = global_step
                             if args.do_test:
                                 results_test = evaluate(args, model, tokenizer, test=True)

examples/summarization/modeling_bertabs.py

@@ -446,8 +446,6 @@ class MultiHeadedAttention(nn.Module):
         batch_size = key.size(0)
         dim_per_head = self.dim_per_head
         head_count = self.head_count
-        key_len = key.size(1)
-        query_len = query.size(1)
 
         def shape(x):
             """  projection """
@@ -504,9 +502,6 @@ class MultiHeadedAttention(nn.Module):
 
         query = shape(query)
 
-        key_len = key.size(2)
-        query_len = query.size(2)
-
         # 2) Calculate and scale scores.
         query = query / math.sqrt(dim_per_head)
         scores = torch.matmul(query, key.transpose(2, 3))

setup.cfg

@@ -25,5 +25,5 @@ multi_line_output = 3
 use_parentheses = True
 
 [flake8]
-ignore = E203, E501, F841, W503
+ignore = E203, E501, W503
 max-line-length = 119

setup.py

@@ -86,7 +86,7 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.0.10",
+        "tokenizers == 0.0.11",
         # accessing files from S3 directly
         "boto3",
         # filesystem locks e.g. to prevent parallel downloads

src/transformers/__init__.py

@@ -31,7 +31,6 @@ from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, Open
 from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig
 from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config
 from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
-
 # Configurations
 from .configuration_utils import PretrainedConfig
 from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig
@@ -56,7 +55,6 @@ from .data import (
     xnli_processors,
     xnli_tasks_num_labels,
 )
-
 # Files and general utilities
 from .file_utils import (
     CONFIG_NAME,
@@ -73,10 +71,8 @@ from .file_utils import (
     is_tf_available,
     is_torch_available,
 )
-
 # Model Cards
 from .modelcard import ModelCard
-
 # TF 2.0 <=> PyTorch conversion utilities
 from .modeling_tf_pytorch_utils import (
     convert_tf_weight_name_to_pt_weight_name,
@@ -87,7 +83,6 @@ from .modeling_tf_pytorch_utils import (
     load_tf2_model_in_pytorch_model,
     load_tf2_weights_in_pytorch_model,
 )
-
 # Pipelines
 from .pipelines import (
     CsvPipelineDataFormat,
@@ -113,7 +108,6 @@ from .tokenization_openai import OpenAIGPTTokenizer
 from .tokenization_roberta import RobertaTokenizer
 from .tokenization_t5 import T5Tokenizer
 from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer
-
 # Tokenizers
 from .tokenization_utils import PreTrainedTokenizer
 from .tokenization_xlm import XLMTokenizer

src/transformers/__main__.py

@@ -1,42 +0,0 @@
-# coding: utf8
-
-
-def main():
-    import sys
-
-    if len(sys.argv) < 2 or sys.argv[1] not in ["convert", "train", "predict", "serve"]:
-        print(
-            "First argument to `transformers` command line interface should be one of: \n"
-            ">> convert serve train predict"
-        )
-    if sys.argv[1] == "convert":
-        from transformers.commands import convert
-
-        convert(sys.argv)
-    elif sys.argv[1] == "train":
-        from transformers.commands import train
-
-        train(sys.argv)
-    elif sys.argv[1] == "serve":
-        pass
-        # from argparse import ArgumentParser
-        # from transformers.commands.serving import ServeCommand
-        # parser = ArgumentParser('Transformers CLI tool', usage='transformers serve <command> [<args>]')
-        # commands_parser = parser.add_subparsers(help='transformers-cli command helpers')
-
-        # # Register commands
-        # ServeCommand.register_subcommand(commands_parser)
-
-        # # Let's go
-        # args = parser.parse_args()
-
-        # if not hasattr(args, 'func'):
-        #     parser.print_help()
-        #     exit(1)
-        # # Run
-        # service = args.func(args)
-        # service.run()
-
-
-if __name__ == "__main__":
-    main()

src/transformers/commands/convert.py

@@ -25,7 +25,7 @@ class ConvertCommand(BaseTransformersCLICommand):
         train_parser = parser.add_parser(
             "convert",
             help="CLI tool to run convert model from original "
-            "author checkpoints to Transformesr PyTorch checkpoints.",
+            "author checkpoints to Transformers PyTorch checkpoints.",
         )
         train_parser.add_argument("--model_type", type=str, required=True, help="Model's type.")
         train_parser.add_argument(

src/transformers/commands/serving.py

@@ -109,8 +109,8 @@ class ServeCommand(BaseTransformersCLICommand):
         if not _serve_dependancies_installed:
             raise RuntimeError(
                 "Using serve command requires FastAPI and unicorn. "
-                "Please install transformers with [serving]: pip install transformers[serving]."
-                "Or install FastAPI and unicorn separatly."
+                'Please install transformers with [serving]: pip install "transformers[serving]".'
+                "Or install FastAPI and unicorn separately."
             )
         else:
             logger.info("Serving model over {}:{}".format(host, port))

src/transformers/commands/user.py

@@ -9,17 +9,26 @@ from transformers.commands import BaseTransformersCLICommand
 from transformers.hf_api import HfApi, HfFolder
 
 
+UPLOAD_MAX_FILES = 15
+
+
 class UserCommands(BaseTransformersCLICommand):
     @staticmethod
     def register_subcommand(parser: ArgumentParser):
-        login_parser = parser.add_parser("login")
+        login_parser = parser.add_parser("login", help="Log in using the same credentials as on huggingface.co")
         login_parser.set_defaults(func=lambda args: LoginCommand(args))
-        whoami_parser = parser.add_parser("whoami")
+        whoami_parser = parser.add_parser("whoami", help="Find out which huggingface.co account you are logged in as.")
         whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args))
-        logout_parser = parser.add_parser("logout")
+        logout_parser = parser.add_parser("logout", help="Log out")
         logout_parser.set_defaults(func=lambda args: LogoutCommand(args))
-        list_parser = parser.add_parser("ls")
-        list_parser.set_defaults(func=lambda args: ListObjsCommand(args))
+        # s3
+        s3_parser = parser.add_parser("s3", help="{ls, rm} Commands to interact with the files you upload on S3.")
+        s3_subparsers = s3_parser.add_subparsers(help="s3 related commands")
+        ls_parser = s3_subparsers.add_parser("ls")
+        ls_parser.set_defaults(func=lambda args: ListObjsCommand(args))
+        rm_parser = s3_subparsers.add_parser("rm")
+        rm_parser.add_argument("filename", type=str, help="individual object filename to delete from S3.")
+        rm_parser.set_defaults(func=lambda args: DeleteObjCommand(args))
         # upload
         upload_parser = parser.add_parser("upload")
         upload_parser.add_argument("path", type=str, help="Local path of the folder or individual file to upload.")
@@ -131,13 +140,27 @@ class ListObjsCommand(BaseUserCommand):
         print(self.tabulate(rows, headers=["Filename", "LastModified", "ETag", "Size"]))
 
 
+class DeleteObjCommand(BaseUserCommand):
+    def run(self):
+        token = HfFolder.get_token()
+        if token is None:
+            print("Not logged in")
+            exit(1)
+        try:
+            self._api.delete_obj(token, filename=self.args.filename)
+        except HTTPError as e:
+            print(e)
+            exit(1)
+        print("Done")
+
+
 class UploadCommand(BaseUserCommand):
     def walk_dir(self, rel_path):
         """
         Recursively list all files in a folder.
         """
         entries: List[os.DirEntry] = list(os.scandir(rel_path))
-        files = [(os.path.join(os.getcwd(), f.path), f.path) for f in entries if f.is_file()]  # filepath  # filename
+        files = [(os.path.join(os.getcwd(), f.path), f.path) for f in entries if f.is_file()]  # (filepath, filename)
         for f in entries:
             if f.is_dir():
                 files += self.walk_dir(f.path)
@@ -160,6 +183,14 @@ class UploadCommand(BaseUserCommand):
         else:
             raise ValueError("Not a valid file or directory: {}".format(local_path))
 
+        if len(files) > UPLOAD_MAX_FILES:
+            print(
+                "About to upload {} files to S3. This is probably wrong. Please filter files before uploading.".format(
+                    ANSI.bold(len(files))
+                )
+            )
+            exit(1)
+
         for filepath, filename in files:
             print("About to upload file {} to S3 under filename {}".format(ANSI.bold(filepath), ANSI.bold(filename)))
 

src/transformers/data/metrics/__init__.py

@@ -19,7 +19,7 @@ try:
     from sklearn.metrics import matthews_corrcoef, f1_score
 
     _has_sklearn = True
-except (AttributeError, ImportError) as e:
+except (AttributeError, ImportError):
     _has_sklearn = False
 
 

src/transformers/hf_api.py

@@ -79,7 +79,7 @@ class HfApi:
         r = requests.post(path, headers={"authorization": "Bearer {}".format(token)})
         r.raise_for_status()
 
-    def presign(self, token: str, filename) -> PresignedUrl:
+    def presign(self, token: str, filename: str) -> PresignedUrl:
         """
         Call HF API to get a presigned url to upload `filename` to S3.
         """
@@ -89,7 +89,7 @@ class HfApi:
         d = r.json()
         return PresignedUrl(**d)
 
-    def presign_and_upload(self, token: str, filename, filepath) -> str:
+    def presign_and_upload(self, token: str, filename: str, filepath: str) -> str:
         """
         Get a presigned url, then upload file to S3.
 
@@ -111,7 +111,7 @@ class HfApi:
             pf.close()
         return urls.access
 
-    def list_objs(self, token) -> List[S3Obj]:
+    def list_objs(self, token: str) -> List[S3Obj]:
         """
         Call HF API to list all stored files for user.
         """
@@ -121,6 +121,14 @@ class HfApi:
         d = r.json()
         return [S3Obj(**x) for x in d]
 
+    def delete_obj(self, token: str, filename: str):
+        """
+        Call HF API to delete a file stored by user
+        """
+        path = "{}/api/deleteObj".format(self.endpoint)
+        r = requests.delete(path, headers={"authorization": "Bearer {}".format(token)}, json={"filename": filename})
+        r.raise_for_status()
+
 
 class TqdmProgressFileReader:
     """

src/transformers/modeling_albert.py

@@ -241,8 +241,6 @@ class AlbertAttention(BertSelfAttention):
         context_layer = torch.matmul(attention_probs, value_layer)
 
         context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
-        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
-        reshaped_context_layer = context_layer.view(*new_context_layer_shape)
 
         # Should find a better way to do this
         w = (
@@ -334,9 +332,6 @@ class AlbertTransformer(nn.Module):
             # Index of the hidden group
             group_idx = int(i / (self.config.num_hidden_layers / self.config.num_hidden_groups))
 
-            # Index of the layer inside the group
-            layer_idx = int(i - group_idx * layers_per_group)
-
             layer_group_output = self.albert_layer_groups[group_idx](
                 hidden_states,
                 attention_mask,

src/transformers/modeling_ctrl.py

@@ -490,6 +490,15 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        # only last token for inputs_ids if past is defined in kwargs
+        if "past" in kwargs and kwargs["past"]:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+
+        inputs = {"input_ids": input_ids}
+        inputs.update(kwargs)
+        return inputs
+
     def forward(
         self,
         input_ids=None,

src/transformers/modeling_gpt2.py

@@ -103,6 +103,7 @@ class Attention(nn.Module):
     def __init__(self, nx, n_ctx, config, scale=False):
         super(Attention, self).__init__()
         self.output_attentions = config.output_attentions
+        self.output_past = config.output_past
 
         n_state = nx  # in Attention: n_state=768 (nx=n_embd)
         # [switch nx => n_state from Block to Attention to keep identical to TF implem]
@@ -159,10 +160,11 @@ class Attention(nn.Module):
         if head_mask is not None:
             w = w * head_mask
 
-        outputs = [torch.matmul(w, v)]
+        x = torch.matmul(w, v)
         if self.output_attentions:
-            outputs.append(w)
-        return outputs
+            return (x, w)
+        else:
+            return (x,)
 
     def merge_heads(self, x):
         x = x.permute(0, 2, 1, 3).contiguous()
@@ -187,7 +189,8 @@ class Attention(nn.Module):
             past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1]  # transpose back cf below
             key = torch.cat((past_key, key), dim=-1)
             value = torch.cat((past_value, value), dim=-2)
-        present = torch.stack((key.transpose(-2, -1), value))  # transpose to have same shapes for stacking
+        if self.output_past:
+            present = torch.stack((key.transpose(-2, -1), value))  # transpose to have same shapes for stacking
 
         attn_outputs = self._attn(query, key, value, attention_mask, head_mask)
         a = attn_outputs[0]
@@ -196,8 +199,11 @@ class Attention(nn.Module):
         a = self.c_proj(a)
         a = self.resid_dropout(a)
 
-        outputs = [a, present] + attn_outputs[1:]
-        return outputs  # a, present, (attentions)
+        if self.output_past:
+            outputs = (a, present) + attn_outputs[1:]
+        else:
+            outputs = (a,) + attn_outputs[1:]
+        return outputs  # a, (present), (attentions)
 
 
 class MLP(nn.Module):
@@ -228,14 +234,14 @@ class Block(nn.Module):
         output_attn = self.attn(
             self.ln_1(x), layer_past=layer_past, attention_mask=attention_mask, head_mask=head_mask
         )
-        a = output_attn[0]  # output_attn: a, present, (attentions)
+        a = output_attn[0]  # output_attn: a, (present), (attentions)
 
         x = x + a
         m = self.mlp(self.ln_2(x))
         x = x + m
 
-        outputs = [x] + output_attn[1:]
-        return outputs  # x, present, (attentions)
+        outputs = (x,) + output_attn[1:]
+        return outputs  # x, (present), (attentions)
 
 
 class GPT2PreTrainedModel(PreTrainedModel):
@@ -465,7 +471,7 @@ class GPT2Model(GPT2PreTrainedModel):
         output_shape = input_shape + (hidden_states.size(-1),)
 
         presents = ()
-        all_attentions = []
+        all_attentions = ()
         all_hidden_states = ()
         for i, (block, layer_past) in enumerate(zip(self.h, past)):
             if self.output_hidden_states:
@@ -475,12 +481,12 @@ class GPT2Model(GPT2PreTrainedModel):
                 hidden_states, layer_past=layer_past, attention_mask=attention_mask, head_mask=head_mask[i]
             )
 
-            hidden_states, present = outputs[:2]
+            hidden_states = outputs[0]
             if self.output_past:
-                presents = presents + (present,)
+                presents = presents + (outputs[1],)
 
             if self.output_attentions:
-                all_attentions.append(outputs[2])
+                all_attentions = all_attentions + (outputs[2 if self.output_past else 1],)
 
         hidden_states = self.ln_f(hidden_states)
 
@@ -559,6 +565,15 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        # only last token for inputs_ids if past is defined in kwargs
+        if "past" in kwargs and kwargs["past"]:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+
+        inputs = {"input_ids": input_ids}
+        inputs.update(kwargs)
+        return inputs
+
     def forward(
         self,
         input_ids=None,

src/transformers/modeling_t5.py

@@ -629,7 +629,7 @@ class T5Stack(T5PreTrainedModel):
                 all_attentions = all_attentions + (layer_outputs[1],)  # We keep only self-attention weights for now
 
         hidden_states = self.final_layer_norm(hidden_states)
-        layer_output = self.dropout(hidden_states)
+        hidden_states = self.dropout(hidden_states)
 
         # Add last layer
         if self.output_hidden_states:

src/transformers/modeling_tf_pytorch_utils.py

@@ -122,7 +122,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
         tf_inputs = tf_model.dummy_inputs
 
     if tf_inputs is not None:
-        tfo = tf_model(tf_inputs, training=False)  # Make sure model is built
+        tf_model(tf_inputs, training=False)  # Make sure model is built
 
     # Adapt state dict - TODO remove this and update the AWS weights files instead
     # Convert old format to new format if needed from a PyTorch state_dict
@@ -187,7 +187,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
     K.batch_set_value(weight_value_tuples)
 
     if tf_inputs is not None:
-        tfo = tf_model(tf_inputs, training=False)  # Make sure restore ops are run
+        tf_model(tf_inputs, training=False)  # Make sure restore ops are run
 
     logger.info("Loaded {:,} parameters in the TF 2.0 model.".format(tf_loaded_numel))
 
@@ -218,7 +218,6 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs
 
     import transformers
 
-    tf_path = os.path.abspath(tf_checkpoint_path)
     logger.info("Loading TensorFlow weights from {}".format(tf_checkpoint_path))
 
     # Instantiate and load the associated TF 2.0 model
@@ -230,7 +229,7 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs
         tf_inputs = tf_model.dummy_inputs
 
     if tf_inputs is not None:
-        tfo = tf_model(tf_inputs, training=False)  # Make sure model is built
+        tf_model(tf_inputs, training=False)  # Make sure model is built
 
     tf_model.load_weights(tf_checkpoint_path, by_name=True)
 

src/transformers/modeling_tf_t5.py

@@ -491,7 +491,7 @@ class TFT5MainLayer(tf.keras.layers.Layer):
                 all_attentions = all_attentions + (layer_outputs[1],)
 
         hidden_states = self.final_layer_norm(hidden_states)
-        layer_output = self.dropout(hidden_states, training=training)
+        hidden_states = self.dropout(hidden_states, training=training)
 
         # Add last layer
         if self.output_hidden_states:

src/transformers/modeling_tf_transfo_xl_utilities.py

@@ -118,7 +118,6 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer):
         hidden, target = inputs
         head_logprob = 0
         if self.n_clusters == 0:
-            softmax_b = tf.get_variable("bias", [self.config.vocab_size], initializer=tf.zeros_initializer())
             output = self._logit(hidden, self.out_layers[0][0], self.out_layers[0][1], self.out_projs[0])
             if target is not None:
                 loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, logits=output)

src/transformers/modeling_tf_utils.py

@@ -320,7 +320,7 @@ class TFPreTrainedModel(tf.keras.Model):
             # Load from a PyTorch checkpoint
             return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, allow_missing_keys=True)
 
-        ret = model(model.dummy_inputs, training=False)  # build the network with dummy inputs
+        model(model.dummy_inputs, training=False)  # build the network with dummy inputs
 
         assert os.path.isfile(resolved_archive_file), "Error retrieving file {}".format(resolved_archive_file)
         # 'by_name' allow us to do transfer learning by skipping/adding layers
@@ -333,7 +333,7 @@ class TFPreTrainedModel(tf.keras.Model):
                 "If you tried to load a TF 2.0 model from a PyTorch checkpoint, please set from_pt=True. "
             )
 
-        ret = model(model.dummy_inputs, training=False)  # Make sure restore ops are run
+        model(model.dummy_inputs, training=False)  # Make sure restore ops are run
 
         # Check if the models are the same to output loading informations
         with h5py.File(resolved_archive_file, "r") as f:
@@ -515,7 +515,7 @@ class TFSequenceSummary(tf.keras.layers.Layer):
             cls_index = inputs[1] if len(inputs) > 1 else None
             assert len(inputs) <= 2, "Too many inputs."
         else:
-            input_ids = inputs.get("input_ids")
+            hidden_states = inputs.get("hidden_states")
             cls_index = inputs.get("cls_index", None)
 
         if self.summary_type == "last":

src/transformers/modeling_transfo_xl.py

@@ -930,3 +930,12 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel):
             return self.out_layer
         else:
             return self.crit.out_layers[-1]
+
+    def prepare_inputs_for_generation(self, input_ids, **model_kwargs):
+        inputs = {"input_ids": input_ids}
+
+        # if past is defined in model kwargs then use it for faster decoding
+        if "past" in model_kwargs and model_kwargs["past"]:
+            inputs["mems"] = model_kwargs["past"]
+
+        return inputs

src/transformers/modeling_utils.py

@@ -539,6 +539,17 @@ class PreTrainedModel(nn.Module):
     def prepare_inputs_for_generation(self, input_ids, **kwargs):
         return {"input_ids": input_ids}
 
+    def _do_output_past(self, outputs):
+        has_output_past = hasattr(self.config, "output_past") and self.config.output_past
+        has_mem_len = hasattr(self.config, "mem_len") and self.config.mem_len
+
+        if has_output_past and not has_mem_len and len(outputs) > 1:
+            return True
+        elif has_mem_len and self.config.mem_len > 0 and len(outputs) > 1:
+            return True
+
+        return False
+
     @torch.no_grad()
     def generate(
         self,
@@ -556,48 +567,89 @@ class PreTrainedModel(nn.Module):
         length_penalty=None,
         num_return_sequences=None,
     ):
-        """ Sequence generator for models with a LM head.
-
-        The method currently supports greedy or penalized greedy decoding, sampling with top-k or nucleus sampling
+        r""" Generates sequences for models with a LM head. The method currently supports greedy or penalized greedy decoding, sampling with top-k or nucleus sampling
         and beam-search.
 
-        Adapted in part from Facebook's XLM beam search code: https://github.com/facebookresearch/XLM
+        Adapted in part from `Facebook's XLM beam search code`_.
 
-        Params:
-            **input_ids**: (`optional`) `torch.LongTensor` of shape (1, sequence_length)
+        .. _`Facebook's XLM beam search code`:
+           https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529
+
+
+        Parameters:
+
+            input_ids: (`optional`) `torch.LongTensor` of shape `(batch_size, sequence_length)`
                 The sequence used as a prompt for the generation. If `None` the method initializes
-                it as an empty `torch.LongTensor` of shape (1,)
-            **max_length**: (`optional`) int
+                it as an empty `torch.LongTensor` of shape `(1,)`.
+
+            max_length: (`optional`) int
                 The max length of the sequence to be generated.  Between 1 and infinity. Default to 20.
-            **do_sample**: (`optional`) bool
-                If set to `False` we use greedy decoding; otherwise sampling. Default to greedy sampling.
-            **num_beams**: (`optional`) int
-                Number of beams for beam search. 1 means no beam serach. Default to 1.
-            **temperature**: (`optional`) float
-                The value used to module the next token probabilities.
-            **top_k**: (`optional`) int
+
+            do_sample: (`optional`) bool
+                If set to `False` greedy decoding is used. Otherwise sampling is used. Default to greedy sampling.
+
+            num_beams: (`optional`) int
+                Number of beams for beam search. Must be between 1 and infinity. 1 means no beam search. Default to 1.
+
+            temperature: (`optional`) float
+                The value used to module the next token probabilities. Must be strictely positive. Default to 1.0.
+
+            top_k: (`optional`) int
                 The number of highest probability vocabulary tokens to keep for top-k-filtering. Between 1 and infinity. Default to 50.
-            **top_p**: (`optional`) float
+
+            top_p: (`optional`) float
                 The cumulative probability of parameter highest probability vocabulary tokens to keep for nucleus sampling. Must be between 0 and 1. Default to 1.
-            **repetition_penalty**: (`optional`) float
-                The parameter for repetition penalty. Between 1.0 and + infinity. 1.0 means no penalty. Default to 1.
-            **bos_token_id**: (`optional`) int
+
+            repetition_penalty: (`optional`) float
+                The parameter for repetition penalty. Between 1.0 and infinity. 1.0 means no penalty. Default to 1.0.
+
+            bos_token_id: (`optional`) int
                 Beginning of sentence token if no prompt is provided. Default to 0.
-            **eos_token_ids**: (`optional`) int or list of int
+
+            eos_token_ids: (`optional`) int or list of int
                 End of sequence token or list of tokens to stop the generation. Default to 0.
-            **length_penalty**: (`optional`) int
-                Exponential penalty to the length. Default to 0.
-            **length_penalty**: (`optional`) float
+            length_penalty: (`optional`) float
                 Exponential penalty to the length. Default to 1.
-            **num_return_sequences**: (`optional`) int
-                The number of independantly computed returned sequences for each element in the batch. Default to 1.
+
+            num_return_sequences: (`optional`) int
+                The number of independently computed returned sequences for each element in the batch. Default to 1.
+
+        Examples::
+
+            tokenizer = AutoTokenizer.from_pretrained('distilgpt2')   # Initialize tokenizer
+            model = AutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
+            outputs = model.generate(max_length=40, bos_token_id=tokenizer.bos_token_id, eos_token_ids=tokenizer.eos_token_id)  # do greedy decoding without beam search
+            print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))
+
+            tokenizer = AutoTokenizer.from_pretrained('openai-gpt')   # Initialize tokenizer
+            model = AutoModelWithLMHead.from_pretrained('openai-gpt')    # Download model and configuration from S3 and cache.
+            input_context = 'The dog'
+            input_ids = torch.tensor(tokenizer.encode(input_context)).unsqueeze(0)  # encode input context
+            outputs = model.generate(input_ids=input_ids, do_sample=True, num_beams=5, num_return_sequences=3, temperature=1.5)  # generate 3 independent sequences using beam search decoding (5 beams) with sampling from initial context 'The dog'
+            for i in range(3): #  3 output sequences were generated
+                print('Generated {}: {}'.format(i, tokenizer.decode(outputs[0][i], skip_special_tokens=True)))
+
+            tokenizer = AutoTokenizer.from_pretrained('distilgpt2')   # Initialize tokenizer
+            model = AutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
+            input_context = 'The dog'
+            input_ids = torch.tensor(tokenizer.encode(input_context)).unsqueeze(0)  # encode input context
+            outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, bos_token_id=tokenizer.bos_token_id, eos_token_ids=tokenizer.eos_token_id, num_beams=3)  # generate sequences using greedy beam search decoding (3 beams)
+            print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))
+
+            tokenizer = AutoTokenizer.from_pretrained('ctrl')   # Initialize tokenizer
+            model = AutoModelWithLMHead.from_pretrained('ctrl')    # Download model and configuration from S3 and cache.
+            input_context = 'Legal My neighbor is'  # "Legal" is one of the control codes for ctrl
+            input_ids = torch.tensor(tokenizer.encode(input_context)).unsqueeze(0)  # encode input context
+            outputs = model.generate(input_ids=input_ids, max_length=50, temperature=0.7, repetition_penalty=1.2)  # generate sequences using using greedy search
+            print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))
+
         """
 
         # We cannot generate if the model does not have a LM head
         if self.get_output_embeddings() is None:
             raise AttributeError(
                 "You tried to generate sequences with a model that does not have a LM Head."
-                "Please use another model class (e.g. `OpenAIGPTLMHeadModel`)"
+                "Please use another model class (e.g. `OpenAIGPTLMHeadModel`, `XLNetLMHeadModel`, `GPT2LMHeadModel`, `CTRLLMHeadModel`, `T5WithLMHeadModel`, `TransfoXLLMHeadModel`)"
             )
 
         max_length = max_length if max_length is not None else self.config.max_length
@@ -625,7 +677,7 @@ class PreTrainedModel(nn.Module):
         assert isinstance(max_length, int) and max_length > 0, "`max_length` should be a strictely positive integer."
         assert isinstance(do_sample, bool), "`do_sample` should be a boolean."
         assert isinstance(num_beams, int) and num_beams > 0, "`num_beams` should be a strictely positive integer."
-        # assert temperature >= 0, "`temperature` should be positive."
+        assert temperature > 0, "`temperature` should be strictely positive."
         assert isinstance(top_k, int) and top_k >= 0, "`top_k` should be a positive integer."
         assert 0 <= top_p <= 1, "`top_p` should be between 0 and 1."
         assert repetition_penalty >= 1.0, "`repetition_penalty` should be >= 1."
@@ -716,23 +768,30 @@ class PreTrainedModel(nn.Module):
         # current position / max lengths / length of generated sentences / unfinished sentences
         unfinished_sents = input_ids.new(batch_size).fill_(1)
 
-        # TODO: add cached compute states
-        pasts = None
+        past = None
 
         while cur_len < max_length:
-            model_inputs = self.prepare_inputs_for_generation(input_ids, pasts=pasts)
+            model_inputs = self.prepare_inputs_for_generation(input_ids, past=past)
             outputs = self(**model_inputs)
             next_token_logits = outputs[0][:, -1, :]
 
+            # if model has past, then set the past variable to speed up decoding
+            if self._do_output_past(outputs):
+                past = outputs[1]
+
             # repetition penalty from CTRL paper (https://arxiv.org/abs/1909.05858)
             if repetition_penalty != 1.0:
                 for i in range(batch_size):
-                    for previous_tokens in set(input_ids[i].tolist()):
-                        next_token_logits[i, previous_tokens] /= repetition_penalty
+                    for previous_token in set(input_ids[i].tolist()):
+                        # if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
+                        if next_token_logits[i, previous_token] < 0:
+                            next_token_logits[i, previous_token] *= repetition_penalty
+                        else:
+                            next_token_logits[i, previous_token] /= repetition_penalty
 
             if do_sample:
                 # Temperature (higher temperature => more likely to sample low probability tokens)
-                if temperature > 0 and temperature != 1.0:
+                if temperature != 1.0:
                     next_token_logits = next_token_logits / temperature
                 # Top-p/top-k filtering
                 next_token_logits = top_k_top_p_filtering(next_token_logits, top_k=top_k, top_p=top_p)
@@ -793,25 +852,33 @@ class PreTrainedModel(nn.Module):
         beam_scores = beam_scores.view(-1)  # shape (batch_size * num_beams,)
 
         # cache compute states
-        pasts = None  # self.prepare_pasts()
+        past = None
 
         # done sentences
         done = [False for _ in range(batch_size)]
 
         while cur_len < max_length:
-            model_inputs = self.prepare_inputs_for_generation(input_ids, pasts=pasts)
-            scores = self(**model_inputs)[0]  # (batch_size * num_beams, cur_len, vocab_size)
-            scores = scores[:, -1, :]  # (batch_size * num_beams, vocab_size)
+            model_inputs = self.prepare_inputs_for_generation(input_ids, past=past)
+            outputs = self(**model_inputs)  # (batch_size * num_beams, cur_len, vocab_size)
+            scores = outputs[0][:, -1, :]  # (batch_size * num_beams, vocab_size)
+
+            # if model has past, then set the past variable to speed up decoding
+            if self._do_output_past(outputs):
+                past = outputs[1]
 
             # repetition penalty (from CTRL paper https://arxiv.org/abs/1909.05858)
             if repetition_penalty != 1.0:
                 for i in range(batch_size * num_beams):
-                    for previous_tokens in set(input_ids[i].tolist()):
-                        scores[i, previous_tokens] /= repetition_penalty
+                    for previous_token in set(input_ids[i].tolist()):
+                        # if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
+                        if scores[i, previous_token] < 0:
+                            scores[i, previous_token] *= repetition_penalty
+                        else:
+                            scores[i, previous_token] /= repetition_penalty
 
             if do_sample:
                 # Temperature (higher temperature => more likely to sample low probability tokens)
-                if temperature > 0 and temperature != 1.0:
+                if temperature != 1.0:
                     scores = scores / temperature
                 # Top-p/top-k filtering
                 scores = top_k_top_p_filtering(
@@ -886,13 +953,22 @@ class PreTrainedModel(nn.Module):
             beam_words = input_ids.new([x[1] for x in next_batch_beam])
             beam_idx = input_ids.new([x[2] for x in next_batch_beam])
 
-            # re-order batch and internal states
+            # re-order batch
             input_ids = input_ids[beam_idx, :]
             input_ids = torch.cat([input_ids, beam_words.unsqueeze(1)], dim=-1)
-            # TODO: Activate cache
-            # for k in cache.keys():
-            #     if k != 'slen':
-            #         cache[k] = (cache[k][0][beam_idx], cache[k][1][beam_idx])
+
+            # re-order internal states
+            if past:
+                reordered_past = []
+                for layer_past in past:
+                    # get the correct batch idx from layer past batch dim
+                    # batch dim of `past` and `mems` is at 2nd position
+                    reordered_layer_past = [layer_past[:, i].unsqueeze(1).clone().detach() for i in beam_idx]
+                    reordered_layer_past = torch.cat(reordered_layer_past, dim=1)
+                    # check that shape matches
+                    assert reordered_layer_past.shape == layer_past.shape
+                    reordered_past.append(reordered_layer_past)
+                past = tuple(reordered_past)
 
             # update current length
             cur_len = cur_len + 1
@@ -958,9 +1034,7 @@ def top_k_top_p_filtering(logits, top_k=0, top_p=1.0, filter_value=-float("Inf")
         sorted_indices_to_remove[..., 0] = 0
 
         # scatter sorted tensors to original indexing
-        indices_to_remove = sorted_indices_to_remove.scatter(
-            dim=1, index=sorted_indices, source=sorted_indices_to_remove
-        )
+        indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
         logits[indices_to_remove] = filter_value
     return logits
 

src/transformers/modeling_xlm.py

@@ -674,7 +674,8 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
         mask_token_id = self.config.mask_token_id
         lang_id = self.config.lang_id
 
-        mask_token = torch.full((1, 1), mask_token_id, dtype=torch.long, device=input_ids.device)
+        effective_batch_size = input_ids.shape[0]
+        mask_token = torch.full((effective_batch_size, 1), mask_token_id, dtype=torch.long, device=input_ids.device)
         input_ids = torch.cat([input_ids, mask_token], dim=1)
         if lang_id is not None:
             langs = torch.full_like(input_ids, lang_id)

src/transformers/modeling_xlnet.py

@@ -1010,22 +1010,31 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
 
     def prepare_inputs_for_generation(self, input_ids, **model_kwargs):
         # Add dummy token at the end (no attention on this one)
-        dummy_token = torch.zeros((1, 1), dtype=torch.long, device=input_ids.device)
+
+        effective_batch_size = input_ids.shape[0]
+        dummy_token = torch.zeros((effective_batch_size, 1), dtype=torch.long, device=input_ids.device)
         input_ids = torch.cat([input_ids, dummy_token], dim=1)
 
         # Build permutation mask so that previous tokens don't see last token
+        sequence_length = input_ids.shape[1]
         perm_mask = torch.zeros(
-            (input_ids.shape[0], input_ids.shape[1], input_ids.shape[1]), dtype=torch.float, device=input_ids.device
+            (effective_batch_size, sequence_length, sequence_length), dtype=torch.float, device=input_ids.device
         )
         perm_mask[:, :, -1] = 1.0
 
         # We'll only predict the last token
         target_mapping = torch.zeros(
-            (input_ids.shape[0], 1, input_ids.shape[1]), dtype=torch.float, device=input_ids.device
+            (effective_batch_size, 1, sequence_length), dtype=torch.float, device=input_ids.device
         )
         target_mapping[0, 0, -1] = 1.0
 
-        return {"input_ids": input_ids, "perm_mask": perm_mask, "target_mapping": target_mapping}
+        inputs = {"input_ids": input_ids, "perm_mask": perm_mask, "target_mapping": target_mapping}
+
+        # if past is defined in model kwargs then use it for faster decoding
+        if "past" in model_kwargs and model_kwargs["past"]:
+            inputs["mems"] = model_kwargs["past"]
+
+        return inputs
 
     def forward(
         self,

src/transformers/tokenization_utils.py

@@ -469,6 +469,9 @@ class PreTrainedTokenizer(object):
         tokenizer.init_inputs = init_inputs
         tokenizer.init_kwargs = init_kwargs
 
+        # update unique_added_tokens_encoder with special tokens for correct tokenization
+        tokenizer.unique_added_tokens_encoder.update(set(tokenizer.all_special_tokens))
+
         # Add supplementary tokens.
         if added_tokens_file is not None:
             with open(added_tokens_file, encoding="utf-8") as added_tokens_handle:
@@ -476,6 +479,7 @@ class PreTrainedTokenizer(object):
             added_tok_decoder = {v: k for k, v in added_tok_encoder.items()}
             tokenizer.added_tokens_encoder.update(added_tok_encoder)
             tokenizer.added_tokens_decoder.update(added_tok_decoder)
+            tokenizer.unique_added_tokens_encoder.update(set(tokenizer.added_tokens_encoder.keys()))
 
         return tokenizer
 
@@ -866,7 +870,7 @@ class PreTrainedTokenizer(object):
             return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant
                 or PyTorch torch.Tensor instead of a list of python integers.
             return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True).
-            return_attention_mask: (optional) Set to False to avoir returning attention mask (default True)
+            return_attention_mask: (optional) Set to False to avoid returning attention mask (default True)
             return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False).
             return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False).
             **kwargs: passed to the `self.tokenize()` method

templates/adding_a_new_example_script/run_xxx.py

@@ -52,7 +52,6 @@ from utils_squad import (
     write_predictions,
     write_predictions_extended,
 )
-
 # The follwing import is the official SQuAD evaluation script (2.0).
 # You can remove it from the dependencies if you are using this script outside of the library
 # We've added it here for automated tests (see examples/test_examples.py file)

templates/adding_a_new_example_script/utils_xxx.py

@@ -21,7 +21,6 @@ import logging
 import math
 
 from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
-
 # Required by XLNet evaluation method to compute optimal threshold (see write_predictions_extended() method)
 from utils_squad_evaluate import find_all_best_thresh_v2, get_raw_scores, make_qid_to_has_ans
 
@@ -868,8 +867,6 @@ def write_predictions_extended(
         orig_data = json.load(reader)["data"]
 
     qid_to_has_ans = make_qid_to_has_ans(orig_data)
-    has_ans_qids = [k for k, v in qid_to_has_ans.items() if v]
-    no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v]
     exact_raw, f1_raw = get_raw_scores(orig_data, all_predictions)
     out_eval = {}
 

templates/adding_a_new_model/README.md

@@ -56,7 +56,7 @@ You can then finish the addition step by adding imports for your classes in the
 - [ ] add your PyTorch and TF 2.0 model respectively in `modeling_auto.py` and `modeling_tf_auto.py`
 - [ ] add your tokenizer in `tokenization_auto.py`
 - [ ] add your models and tokenizer to `pipeline.py`
-- [ ] add a link to your conversion script in the main conversion utility (currently in `__main__` but will be moved to the `commands` subfolder in the near future)
+- [ ] add a link to your conversion script in the main conversion utility (in `commands/convert.py`)
 - [ ] edit the PyTorch to TF 2.0 conversion script to add your model in the `convert_pytorch_checkpoint_to_tf2.py` file
 - [ ] add a mention of your model in the doc: `README.md` and the documentation itself at `docs/source/pretrained_models.rst`.
 - [ ] upload the pretrained weigths, configurations and vocabulary files.

tests/test_hf_api.py

@@ -60,6 +60,11 @@ class HfApiEndpointsTest(HfApiCommonTest):
         """
         cls._token = cls._api.login(username=USER, password=PASS)
 
+    @classmethod
+    def tearDownClass(cls):
+        for FILE_KEY, FILE_PATH in FILES:
+            cls._api.delete_obj(token=cls._token, filename=FILE_KEY)
+
     def test_whoami(self):
         user = self._api.whoami(token=self._token)
         self.assertEqual(user, USER)

tests/test_modeling_common.py

@@ -284,7 +284,6 @@ class ModelTesterMixin:
             multihead_outputs = head_mask.grad
 
             attentions = outputs[-1]
-            hidden_states = outputs[-2]
 
             # Remove Nan
             for t in attentions:
@@ -590,7 +589,7 @@ class ModelTesterMixin:
                 inputs_dict["decoder_inputs_embeds"] = wte(decoder_input_ids)
 
             with torch.no_grad():
-                outputs = model(**inputs_dict)
+                model(**inputs_dict)
 
 
 class ConfigTester(object):

tests/test_modeling_tf_common.py

@@ -332,7 +332,7 @@ class TFModelTesterMixin:
                 inputs_dict["encoder_inputs_embeds"] = self._get_embeds(wte, encoder_input_ids)
                 inputs_dict["decoder_inputs_embeds"] = self._get_embeds(wte, decoder_input_ids)
 
-            outputs = model(inputs_dict)
+            model(inputs_dict)
 
 
 def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None):

tests/test_modeling_tf_xlm.py

@@ -224,7 +224,6 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase):
 
             inputs = {"input_ids": input_ids, "lengths": input_lengths}
 
-            outputs = model(inputs)
             start_logits, end_logits = model(inputs)
 
             result = {

tests/test_tokenization_common.py

@@ -163,7 +163,6 @@ class TokenizerTesterMixin:
         self.assertEqual(all_size_2, all_size + len(new_toks))
 
         tokens = tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l", add_special_tokens=False)
-        out_string = tokenizer.decode(tokens)
 
         self.assertGreaterEqual(len(tokens), 4)
         self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
@@ -182,7 +181,6 @@ class TokenizerTesterMixin:
         tokens = tokenizer.encode(
             ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l", add_special_tokens=False
         )
-        out_string = tokenizer.decode(tokens)
 
         self.assertGreaterEqual(len(tokens), 6)
         self.assertGreater(tokens[0], tokenizer.vocab_size - 1)