boom boom

.circleci/config.yml

@@ -77,7 +77,6 @@ jobs:
                       - v0.3-torch_and_tf-{{ checksum "setup.py" }}
                       - v0.3-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install git+https://github.com/huggingface/nlp
             - run: pip install .[sklearn,tf-cpu,torch,testing]
             - run: pip install codecov pytest-cov
             - save_cache:
@@ -104,7 +103,6 @@ jobs:
                       - v0.3-torch-{{ checksum "setup.py" }}
                       - v0.3-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install git+https://github.com/huggingface/nlp
             - run: pip install .[sklearn,torch,testing]
             - save_cache:
                   key: v0.3-torch-{{ checksum "setup.py" }}
@@ -129,7 +127,6 @@ jobs:
                       - v0.3-tf-{{ checksum "setup.py" }}
                       - v0.3-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install git+https://github.com/huggingface/nlp
             - run: pip install .[sklearn,tf-cpu,testing]
             - save_cache:
                   key: v0.3-tf-{{ checksum "setup.py" }}
@@ -238,7 +235,8 @@ jobs:
                       - v0.3-code_quality-{{ checksum "setup.py" }}
                       - v0.3-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install isort
+            # we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
+            - run: pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
             - run: pip install .[tf,torch,quality]
             - save_cache:
                   key: v0.3-code_quality-{{ checksum "setup.py" }}

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,2 +0,0 @@
-<!-- This line specifies which issue to close after the pull request is merged. -->
-Fixes #{issue number}

.github/workflows/self-push.yml

@@ -45,8 +45,7 @@ jobs:
         source .env/bin/activate
         pip install --upgrade pip
         pip install torch!=1.6.0
-        pip install .[sklearn,testing,onnxruntime]
-        pip install git+https://github.com/huggingface/nlp
+        pip install .[sklearn,testing]
 
     - name: Are GPUs recognized by our DL frameworks
       run: |

.github/workflows/self-scheduled.yml

@@ -42,7 +42,7 @@ jobs:
         source .env/bin/activate
         pip install --upgrade pip
         pip install torch!=1.6.0
-        pip install .[sklearn,testing,onnxruntime]
+        pip install .[sklearn,testing]
 
     - name: Are GPUs recognized by our DL frameworks
       run: |
@@ -58,7 +58,6 @@ jobs:
       run: |
         source .env/bin/activate
         python -m pytest -n 1 --dist=loadfile -s ./tests/
-
     - name: Run examples tests on GPU
       env:
         TF_FORCE_GPU_ALLOW_GROWTH: "true"

CONTRIBUTING.md

@@ -134,6 +134,12 @@ Follow these steps to start contributing:
    it with `pip uninstall transformers` before reinstalling it in editable
    mode with the `-e` flag.)
 
+   Right now, we need an unreleased version of `isort` to avoid a
+   [bug](https://github.com/timothycrosley/isort/pull/1000):
+
+   ```bash
+   $ pip install -U git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
+   ```
 5. Develop the features on your branch.
 
    As you work on the features, you should make sure that the test suite

Makefile

@@ -4,7 +4,7 @@
 
 quality:
 	black --check --line-length 119 --target-version py35 examples templates tests src utils
-	isort --check-only examples templates tests src utils
+	isort --check-only --recursive examples templates tests src utils
 	flake8 examples templates tests src utils
 	python utils/check_repo.py
 
@@ -12,7 +12,7 @@ quality:
 
 style:
 	black --line-length 119 --target-version py35 examples templates tests src utils
-	isort examples templates tests src utils
+	isort --recursive examples templates tests src utils
 
 # Run tests for the library
 

README.md

@@ -698,11 +698,11 @@ for batch in train_data:
 
 ## Citation
 
-We now have a [paper](https://arxiv.org/abs/1910.03771) you can cite for the 🤗 Transformers library:
+We now have a paper you can cite for the 🤗 Transformers library:
 ```bibtex
 @article{Wolf2019HuggingFacesTS,
   title={HuggingFace's Transformers: State-of-the-art Natural Language Processing},
-  author={Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Joe Davison and Sam Shleifer and Patrick von Platen and Clara Ma and Yacine Jernite and Julien Plu and Canwen Xu and Teven Le Scao and Sylvain Gugger and Mariama Drame and Quentin Lhoest and Alexander M. Rush},
+  author={Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and R'emi Louf and Morgan Funtowicz and Jamie Brew},
   journal={ArXiv},
   year={2019},
   volume={abs/1910.03771}

docs/source/model_doc/bart.rst

@@ -32,7 +32,7 @@ BartForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.BartForConditionalGeneration
-    :members: forward
+    :members: generate, forward
 
 
 BartConfig

docs/source/model_doc/pegasus.rst

@@ -48,7 +48,6 @@ Usage Example
 .. code-block:: python
 
     from transformers import PegasusForConditionalGeneration, PegasusTokenizer
-    import torch
     src_text = [
         """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."""
     ]

docs/source/model_summary.rst

@@ -14,7 +14,6 @@ Each one of the models in the library falls into one of the following categories
   * :ref:`autoencoding-models`
   * :ref:`seq-to-seq-models`
   * :ref:`multimodal-models`
-  * :ref:`retrieval-based-models`
 
 Autoregressive models are pretrained on the classic language modeling task: guess the next token having read all the
 previous ones. They correspond to the decoder of the original transformer model, and a mask is used on top of the full
@@ -606,40 +605,6 @@ The pretrained model only works for classification.
     More information in this :doc:`model documentation </model_doc/mmbt.html>`.
     TODO: write this page
 
-.. _retrieval-based-models:
-
-Retrieval-based models
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Some models use documents retrieval during (pre)training and inference for open-domain question answering, for example.
-
-
-DPR
-----------------------------------------------
-
-.. raw:: html
-
-   <a href="https://huggingface.co/models?filter=dpr">
-       <img alt="Models" src="https://img.shields.io/badge/All_model_pages-dpr-blueviolet">
-   </a>
-   <a href="model_doc/ctrl.dpr">
-       <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-dpr-blueviolet">
-   </a>
-
-`Dense Passage Retrieval for Open-Domain Question Answering <https://arxiv.org/abs/2004.04906>`_,
-Vladimir Karpukhin et al.
-
-Dense Passage Retrieval (DPR) - is a set of tools and models for state-of-the-art open-domain question-answering research.
-
-
-DPR consists in three models:
-
-  * Question encoder: encode questions as vectors
-  * Context encoder: encode contexts as vectors
-  * Reader: extract the answer of the questions inside retrieved contexts, along with a relevance score (high if the inferred span actually answers the question).
-
-DPR's pipeline (not implemented yet) uses a retrieval step to find the top k contexts given a certain question, and then it calls the reader with the question and the retrieved documents to get the answer.
-
 More technical aspects
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/source/serialization.rst

@@ -130,12 +130,13 @@ Pytorch's two modules `JIT and TRACE <https://pytorch.org/docs/stable/jit.html>`
 their model to be re-used in other programs, such as efficiency-oriented C++ programs.
 
 We have provided an interface that allows the export of 🤗 Transformers models to TorchScript so that they can
-be reused in a different environment than a Pytorch-based python program. Here we explain how to export and use our models using TorchScript.
+be reused in a different environment than a Pytorch-based python program. Here we explain how to use our models so that
+they can be exported, and what to be mindful of when using these models with TorchScript.
 
-Exporting a model requires two things:
+Exporting a model needs two things:
 
-* a forward pass with dummy inputs.
-* model instantiation with the ``torchscript`` flag.
+* dummy inputs to execute a model forward pass.
+* the model needs to be instantiated with the ``torchscript`` flag.
 
 These necessities imply several things developers should be careful about. These are detailed below.
 
@@ -146,8 +147,8 @@ Implications
 TorchScript flag and tied weights
 ------------------------------------------------
 This flag is necessary because most of the language models in this repository have tied weights between their
-``Embedding`` layer and their ``Decoding`` layer. TorchScript does not allow the export of models that have tied weights, therefore
-it is necessary to untie and clone the weights beforehand.
+``Embedding`` layer and their ``Decoding`` layer. TorchScript does not allow the export of models that have tied weights,
+it is therefore necessary to untie the weights beforehand.
 
 This implies that models instantiated with the ``torchscript`` flag have their ``Embedding`` layer and ``Decoding`` layer
 separate, which means that they should not be trained down the line. Training would de-synchronize the two layers,
@@ -180,7 +181,7 @@ when exporting varying sequence-length models.
 Using TorchScript in Python
 -------------------------------------------------
 
-Below is an example, showing how to save, load models as well as how to use the trace for inference.
+Below are examples of using the Python to save, load models as well as how to use the trace for inference.
 
 Saving a model
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -236,10 +237,10 @@ We are re-using the previously initialised ``dummy_input``.
 
 .. code-block:: python
 
-    loaded_model = torch.jit.load("traced_bert.pt")
+    loaded_model = torch.jit.load("traced_model.pt")
     loaded_model.eval()
 
-    all_encoder_layers, pooled_output = loaded_model(*dummy_input)
+    all_encoder_layers, pooled_output = loaded_model(dummy_input)
 
 Using a traced model for inference
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

examples/adversarial/utils_hans.py

@@ -20,8 +20,8 @@ from dataclasses import dataclass
 from typing import List, Optional, Union
 
 import tqdm
-
 from filelock import FileLock
+
 from transformers import (
     BartTokenizer,
     BartTokenizerFast,
@@ -112,10 +112,7 @@ if is_torch_available():
             cached_features_file = os.path.join(
                 data_dir,
                 "cached_{}_{}_{}_{}".format(
-                    "dev" if evaluate else "train",
-                    tokenizer.__class__.__name__,
-                    str(max_seq_length),
-                    task,
+                    "dev" if evaluate else "train", tokenizer.__class__.__name__, str(max_seq_length), task,
                 ),
             )
             label_list = processor.get_labels()
@@ -281,10 +278,7 @@ class HansProcessor(DataProcessor):
 
 
 def hans_convert_examples_to_features(
-    examples: List[InputExample],
-    label_list: List[str],
-    max_length: int,
-    tokenizer: PreTrainedTokenizer,
+    examples: List[InputExample], label_list: List[str], max_length: int, tokenizer: PreTrainedTokenizer,
 ):
     """
     Loads a data file into a list of ``InputFeatures``

examples/benchmarking/plot_csv_file.py

@@ -20,9 +20,7 @@ class PlotArguments:
     Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
     """
 
-    csv_file: str = field(
-        metadata={"help": "The csv file to plot."},
-    )
+    csv_file: str = field(metadata={"help": "The csv file to plot."},)
     plot_along_batch: bool = field(
         default=False,
         metadata={"help": "Whether to plot along batch size or sequence lengh. Defaults to sequence length."},
@@ -32,8 +30,7 @@ class PlotArguments:
         metadata={"help": "Whether the csv file has time results or memory results. Defaults to memory results."},
     )
     no_log_scale: bool = field(
-        default=False,
-        metadata={"help": "Disable logarithmic scale when plotting"},
+        default=False, metadata={"help": "Disable logarithmic scale when plotting"},
     )
     is_train: bool = field(
         default=False,
@@ -42,8 +39,7 @@ class PlotArguments:
         },
     )
     figure_png_file: Optional[str] = field(
-        default=None,
-        metadata={"help": "Filename under which the plot will be saved. If unused no plot is saved."},
+        default=None, metadata={"help": "Filename under which the plot will be saved. If unused no plot is saved."},
     )
     short_model_names: Optional[List[str]] = list_field(
         default=None, metadata={"help": "List of model names that are used instead of the ones in the csv file."}

examples/bert-loses-patience/pabee/modeling_pabee_albert.py

@@ -101,30 +101,30 @@ class AlbertModelWithPabee(AlbertModel):
         regression=False,
     ):
         r"""
-        Return:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.AlbertConfig`) and inputs:
-            last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
-                Sequence of hidden-states at the output of the last layer of the model.
-            pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
-                Last layer hidden-state of the first token of the sequence (classification token)
-                further processed by a Linear layer and a Tanh activation function. The Linear
-                layer weights are trained from the next sentence prediction (classification)
-                objective during pre-training.
+    Return:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.AlbertConfig`) and inputs:
+        last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during pre-training.
 
-                This output is usually *not* a good summary
-                of the semantic content of the input, you're often better with averaging or pooling
-                the sequence of hidden-states for the whole input sequence.
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+            This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
         """
 
         if input_ids is not None and inputs_embeds is not None:
@@ -157,10 +157,7 @@ class AlbertModelWithPabee(AlbertModel):
             res = []
             for i in range(self.config.num_hidden_layers):
                 encoder_outputs = self.encoder.adaptive_forward(
-                    encoder_outputs,
-                    current_layer=i,
-                    attention_mask=extended_attention_mask,
-                    head_mask=head_mask,
+                    encoder_outputs, current_layer=i, attention_mask=extended_attention_mask, head_mask=head_mask,
                 )
 
                 pooled_output = self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]))
@@ -177,10 +174,7 @@ class AlbertModelWithPabee(AlbertModel):
             for i in range(self.config.num_hidden_layers):
                 calculated_layer_num += 1
                 encoder_outputs = self.encoder.adaptive_forward(
-                    encoder_outputs,
-                    current_layer=i,
-                    attention_mask=extended_attention_mask,
-                    head_mask=head_mask,
+                    encoder_outputs, current_layer=i, attention_mask=extended_attention_mask, head_mask=head_mask,
                 )
 
                 pooled_output = self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]))
@@ -242,42 +236,42 @@ class AlbertForSequenceClassificationWithPabee(AlbertPreTrainedModel):
         labels=None,
     ):
         r"""
-            labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
-                Labels for computing the sequence classification/regression loss.
-                Indices should be in ``[0, ..., config.num_labels - 1]``.
-                If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
-                If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+            If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
+            If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
 
-        Returns:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.AlbertConfig`) and inputs:
-            loss: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
-                Classification (or regression if config.num_labels==1) loss.
-            logits ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
-                Classification (or regression if config.num_labels==1) scores (before SoftMax).
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.AlbertConfig`) and inputs:
+        loss: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification (or regression if config.num_labels==1) loss.
+        logits ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
 
-            Examples::
+        Examples::
 
-                from transformers import AlbertTokenizer
-                from pabee import AlbertForSequenceClassificationWithPabee
-                import torch
+            from transformers import AlbertTokenizer
+            from pabee import AlbertForSequenceClassificationWithPabee
+            import torch
 
-                tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
-                model = AlbertForSequenceClassificationWithPabee.from_pretrained('albert-base-v2')
-                input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
-                labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
-                outputs = model(input_ids, labels=labels)
-                loss, logits = outputs[:2]
+            tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+            model = AlbertForSequenceClassificationWithPabee.from_pretrained('albert-base-v2')
+            input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+            labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+            outputs = model(input_ids, labels=labels)
+            loss, logits = outputs[:2]
 
         """
 

examples/bert-loses-patience/pabee/modeling_pabee_bert.py

@@ -108,30 +108,30 @@ class BertModelWithPabee(BertModel):
         regression=False,
     ):
         r"""
-        Return:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
-            last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
-                Sequence of hidden-states at the output of the last layer of the model.
-            pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
-                Last layer hidden-state of the first token of the sequence (classification token)
-                further processed by a Linear layer and a Tanh activation function. The Linear
-                layer weights are trained from the next sentence prediction (classification)
-                objective during pre-training.
+    Return:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
+        last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during pre-training.
 
-                This output is usually *not* a good summary
-                of the semantic content of the input, you're often better with averaging or pooling
-                the sequence of hidden-states for the whole input sequence.
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+            This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
         """
 
         if input_ids is not None and inputs_embeds is not None:
@@ -266,44 +266,44 @@ class BertForSequenceClassificationWithPabee(BertPreTrainedModel):
         labels=None,
     ):
         r"""
-            labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
-                Labels for computing the sequence classification/regression loss.
-                Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
-                If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
-                If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
+            If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
+            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
 
-        Returns:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
-            loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
-                Classification (or regression if config.num_labels==1) loss.
-            logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
-                Classification (or regression if config.num_labels==1) scores (before SoftMax).
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
+            Classification (or regression if config.num_labels==1) loss.
+        logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
 
-        Examples::
+    Examples::
 
-            from transformers import BertTokenizer, BertForSequenceClassification
-            from pabee import BertForSequenceClassificationWithPabee
-            import torch
+        from transformers import BertTokenizer, BertForSequenceClassification
+        from pabee import BertForSequenceClassificationWithPabee
+        import torch
 
-            tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-            model = BertForSequenceClassificationWithPabee.from_pretrained('bert-base-uncased')
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertForSequenceClassificationWithPabee.from_pretrained('bert-base-uncased')
 
-            input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
-            labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
-            outputs = model(input_ids, labels=labels)
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
 
-            loss, logits = outputs[:2]
+        loss, logits = outputs[:2]
 
         """
 

examples/bert-loses-patience/run_glue_with_pabee.py

@@ -120,10 +120,7 @@ def train(args, train_dataset, model, tokenizer):
     # Distributed training (should be after apex fp16 initialization)
     if args.local_rank != -1:
         model = torch.nn.parallel.DistributedDataParallel(
-            model,
-            device_ids=[args.local_rank],
-            output_device=args.local_rank,
-            find_unused_parameters=True,
+            model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True,
         )
 
     # Train!
@@ -154,17 +151,13 @@ def train(args, train_dataset, model, tokenizer):
         logger.info("  Continuing training from epoch %d", epochs_trained)
         logger.info("  Continuing training from global step %d", global_step)
         logger.info(
-            "  Will skip the first %d steps in the first epoch",
-            steps_trained_in_current_epoch,
+            "  Will skip the first %d steps in the first epoch", steps_trained_in_current_epoch,
         )
 
     tr_loss, logging_loss = 0.0, 0.0
     model.zero_grad()
     train_iterator = trange(
-        epochs_trained,
-        int(args.num_train_epochs),
-        desc="Epoch",
-        disable=args.local_rank not in [-1, 0],
+        epochs_trained, int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0],
     )
     set_seed(args)  # Added here for reproductibility
     for _ in train_iterator:
@@ -379,11 +372,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False):
             processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
         )
         features = convert_examples_to_features(
-            examples,
-            tokenizer,
-            label_list=label_list,
-            max_length=args.max_seq_length,
-            output_mode=output_mode,
+            examples, tokenizer, label_list=label_list, max_length=args.max_seq_length, output_mode=output_mode,
         )
         if args.local_rank in [-1, 0]:
             logger.info("Saving features into cached file %s", cached_features_file)
@@ -445,24 +434,15 @@ def main():
         help="The output directory where the model predictions and checkpoints will be written.",
     )
     parser.add_argument(
-        "--patience",
-        default="0",
-        type=str,
-        required=False,
+        "--patience", default="0", type=str, required=False,
     )
     parser.add_argument(
-        "--regression_threshold",
-        default=0,
-        type=float,
-        required=False,
+        "--regression_threshold", default=0, type=float, required=False,
     )
 
     # Other parameters
     parser.add_argument(
-        "--config_name",
-        default="",
-        type=str,
-        help="Pretrained config name or path if not the same as model_name",
+        "--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name",
     )
     parser.add_argument(
         "--tokenizer_name",
@@ -486,27 +466,17 @@ def main():
     parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
     parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.")
     parser.add_argument(
-        "--evaluate_during_training",
-        action="store_true",
-        help="Run evaluation during training at each logging step.",
+        "--evaluate_during_training", action="store_true", help="Run evaluation during training at each logging step.",
     )
     parser.add_argument(
-        "--do_lower_case",
-        action="store_true",
-        help="Set this flag if you are using an uncased model.",
+        "--do_lower_case", action="store_true", help="Set this flag if you are using an uncased model.",
     )
 
     parser.add_argument(
-        "--per_gpu_train_batch_size",
-        default=8,
-        type=int,
-        help="Batch size per GPU/CPU for training.",
+        "--per_gpu_train_batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.",
     )
     parser.add_argument(
-        "--per_gpu_eval_batch_size",
-        default=1,
-        type=int,
-        help="Batch size per GPU/CPU for evaluation.",
+        "--per_gpu_eval_batch_size", default=1, type=int, help="Batch size per GPU/CPU for evaluation.",
     )
     parser.add_argument(
         "--gradient_accumulation_steps",
@@ -515,19 +485,13 @@ def main():
         help="Number of updates steps to accumulate before performing a backward/update pass.",
     )
     parser.add_argument(
-        "--learning_rate",
-        default=5e-5,
-        type=float,
-        help="The initial learning rate for Adam.",
+        "--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.",
     )
     parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
     parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
     parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
     parser.add_argument(
-        "--num_train_epochs",
-        default=3.0,
-        type=float,
-        help="Total number of training epochs to perform.",
+        "--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.",
     )
     parser.add_argument(
         "--max_steps",
@@ -539,10 +503,7 @@ def main():
 
     parser.add_argument("--logging_steps", type=int, default=500, help="Log every X updates steps.")
     parser.add_argument(
-        "--save_steps",
-        type=int,
-        default=500,
-        help="Save checkpoint every X updates steps.",
+        "--save_steps", type=int, default=500, help="Save checkpoint every X updates steps.",
     )
     parser.add_argument(
         "--eval_all_checkpoints",
@@ -551,14 +512,10 @@ def main():
     )
     parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
     parser.add_argument(
-        "--overwrite_output_dir",
-        action="store_true",
-        help="Overwrite the content of the output directory",
+        "--overwrite_output_dir", action="store_true", help="Overwrite the content of the output directory",
     )
     parser.add_argument(
-        "--overwrite_cache",
-        action="store_true",
-        help="Overwrite the cached training and evaluation sets",
+        "--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets",
     )
     parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
 
@@ -575,10 +532,7 @@ def main():
         "See details at https://nvidia.github.io/apex/amp.html",
     )
     parser.add_argument(
-        "--local_rank",
-        type=int,
-        default=-1,
-        help="For distributed training: local_rank",
+        "--local_rank", type=int, default=-1, help="For distributed training: local_rank",
     )
     parser.add_argument("--server_ip", type=str, default="", help="For distant debugging.")
     parser.add_argument("--server_port", type=str, default="", help="For distant debugging.")
@@ -680,8 +634,7 @@ def main():
     print("Output Layers Parameters:", output_layers_param_num)
     single_output_layer_param_num = sum(param.numel() for param in model.classifiers[0].parameters())
     print(
-        "Added Output Layers Parameters:",
-        output_layers_param_num - single_output_layer_param_num,
+        "Added Output Layers Parameters:", output_layers_param_num - single_output_layer_param_num,
     )
 
     logger.info("Training/evaluation parameters %s", args)

examples/bertology/run_bertology.py

@@ -66,9 +66,9 @@ def print_2d_tensor(tensor):
 def compute_heads_importance(
     args, model, eval_dataloader, compute_entropy=True, compute_importance=True, head_mask=None, actually_pruned=False
 ):
-    """This method shows how to compute:
-    - head attention entropy
-    - head importance scores according to http://arxiv.org/abs/1905.10650
+    """ This method shows how to compute:
+        - head attention entropy
+        - head importance scores according to http://arxiv.org/abs/1905.10650
     """
     # Prepare our tensors
     n_layers, n_heads = model.config.num_hidden_layers, model.config.num_attention_heads
@@ -150,8 +150,8 @@ def compute_heads_importance(
 
 
 def mask_heads(args, model, eval_dataloader):
-    """This method shows how to mask head (set some heads to zero), to test the effect on the network,
-    based on the head importance scores, as described in Michel et al. (http://arxiv.org/abs/1905.10650)
+    """ This method shows how to mask head (set some heads to zero), to test the effect on the network,
+        based on the head importance scores, as described in Michel et al. (http://arxiv.org/abs/1905.10650)
     """
     _, head_importance, preds, labels = compute_heads_importance(args, model, eval_dataloader, compute_entropy=False)
     preds = np.argmax(preds, axis=1) if args.output_mode == "classification" else np.squeeze(preds)
@@ -201,8 +201,8 @@ def mask_heads(args, model, eval_dataloader):
 
 
 def prune_heads(args, model, eval_dataloader, head_mask):
-    """This method shows how to prune head (remove heads weights) based on
-    the head importance scores as described in Michel et al. (http://arxiv.org/abs/1905.10650)
+    """ This method shows how to prune head (remove heads weights) based on
+        the head importance scores as described in Michel et al. (http://arxiv.org/abs/1905.10650)
     """
     # Try pruning and test time speedup
     # Pruning is like masking but we actually remove the masked weights
@@ -395,8 +395,7 @@ def main():
         cache_dir=args.cache_dir,
     )
     tokenizer = AutoTokenizer.from_pretrained(
-        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
-        cache_dir=args.cache_dir,
+        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, cache_dir=args.cache_dir,
     )
     model = AutoModelForSequenceClassification.from_pretrained(
         args.model_name_or_path,

examples/contrib/mm-imdb/utils_mmimdb.py

@@ -138,9 +138,6 @@ def get_image_transforms():
             transforms.Resize(256),
             transforms.CenterCrop(224),
             transforms.ToTensor(),
-            transforms.Normalize(
-                mean=[0.46777044, 0.44531429, 0.40661017],
-                std=[0.12221994, 0.12145835, 0.14380469],
-            ),
+            transforms.Normalize(mean=[0.46777044, 0.44531429, 0.40661017], std=[0.12221994, 0.12145835, 0.14380469],),
         ]
     )

examples/contrib/run_camembert.py

@@ -30,11 +30,7 @@ def fill_mask(masked_input, model, tokenizer, topk=5):
             )
         else:
             topk_filled_outputs.append(
-                (
-                    masked_input.replace(masked_token, predicted_token),
-                    values[index].item(),
-                    predicted_token,
-                )
+                (masked_input.replace(masked_token, predicted_token), values[index].item(), predicted_token,)
             )
     return topk_filled_outputs
 

examples/contrib/run_openai_gpt.py

@@ -71,10 +71,10 @@ def load_rocstories_dataset(dataset_path):
 
 
 def pre_process_datasets(encoded_datasets, input_len, cap_length, start_token, delimiter_token, clf_token):
-    """Pre-process datasets containing lists of tuples(story, 1st continuation, 2nd continuation, label)
+    """ Pre-process datasets containing lists of tuples(story, 1st continuation, 2nd continuation, label)
 
-    To Transformer inputs of shape (n_batch, n_alternative, length) comprising for each batch, continuation:
-    input_ids[batch, alternative, :] = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
+        To Transformer inputs of shape (n_batch, n_alternative, length) comprising for each batch, continuation:
+        input_ids[batch, alternative, :] = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
     """
     tensor_datasets = []
     for dataset in encoded_datasets:
@@ -83,10 +83,7 @@ def pre_process_datasets(encoded_datasets, input_len, cap_length, start_token, d
         mc_token_ids = np.zeros((n_batch, 2), dtype=np.int64)
         lm_labels = np.full((n_batch, 2, input_len), fill_value=-100, dtype=np.int64)
         mc_labels = np.zeros((n_batch,), dtype=np.int64)
-        for (
-            i,
-            (story, cont1, cont2, mc_label),
-        ) in enumerate(dataset):
+        for i, (story, cont1, cont2, mc_label), in enumerate(dataset):
             with_cont1 = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
             with_cont2 = [start_token] + story[:cap_length] + [delimiter_token] + cont2[:cap_length] + [clf_token]
             input_ids[i, 0, : len(with_cont1)] = with_cont1

examples/contrib/run_swag.py

@@ -629,9 +629,7 @@ def main():
         torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
 
     config = AutoConfig.from_pretrained(args.config_name if args.config_name else args.model_name_or_path)
-    tokenizer = AutoTokenizer.from_pretrained(
-        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
-    )
+    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,)
     model = AutoModelForMultipleChoice.from_pretrained(
         args.model_name_or_path, from_tf=bool(".ckpt" in args.model_name_or_path), config=config
     )

examples/deebert/run_glue_deebert.py

@@ -358,11 +358,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False):
             processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
         )
         features = convert_examples_to_features(
-            examples,
-            tokenizer,
-            label_list=label_list,
-            max_length=args.max_seq_length,
-            output_mode=output_mode,
+            examples, tokenizer, label_list=label_list, max_length=args.max_seq_length, output_mode=output_mode,
         )
         if args.local_rank in [-1, 0]:
             logger.info("Saving features into cached file %s", cached_features_file)

examples/deebert/src/modeling_highway_bert.py

@@ -14,7 +14,8 @@ from transformers.modeling_bert import (
 
 
 def entropy(x):
-    """Calculate entropy of a pre-softmax logit Tensor"""
+    """ Calculate entropy of a pre-softmax logit Tensor
+    """
     exp_x = torch.exp(x)
     A = torch.sum(exp_x, dim=1)  # sum of exp(x_i)
     B = torch.sum(x * exp_x, dim=1)  # sum of x_i * exp(x_i)
@@ -103,8 +104,7 @@ class DeeBertEncoder(nn.Module):
 
 
 @add_start_docstrings(
-    "The Bert Model transformer with early exiting (DeeBERT). ",
-    BERT_START_DOCSTRING,
+    "The Bert Model transformer with early exiting (DeeBERT). ", BERT_START_DOCSTRING,
 )
 class DeeBertModel(BertPreTrainedModel):
     def __init__(self, config):
@@ -127,9 +127,9 @@ class DeeBertModel(BertPreTrainedModel):
         self.embeddings.word_embeddings = value
 
     def _prune_heads(self, heads_to_prune):
-        """Prunes heads of the model.
-        heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
-        See base class PreTrainedModel
+        """ Prunes heads of the model.
+            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+            See base class PreTrainedModel
         """
         for layer, heads in heads_to_prune.items():
             self.encoder.layer[layer].attention.prune_heads(heads)
@@ -147,33 +147,33 @@ class DeeBertModel(BertPreTrainedModel):
         encoder_attention_mask=None,
     ):
         r"""
-        Return:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
-            last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
-                Sequence of hidden-states at the output of the last layer of the model.
-            pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
-                Last layer hidden-state of the first token of the sequence (classification token)
-                further processed by a Linear layer and a Tanh activation function. The Linear
-                layer weights are trained from the next sentence prediction (classification)
-                objective during pre-training.
+    Return:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
+        last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during pre-training.
 
-                This output is usually *not* a good summary
-                of the semantic content of the input, you're often better with averaging or pooling
-                the sequence of hidden-states for the whole input sequence.
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+            This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
-            highway_exits (:obj:`tuple(tuple(torch.Tensor))`:
-                Tuple of each early exit's results (total length: number of layers)
-                Each tuple is again, a tuple of length 2 - the first entry is logits and the second entry is hidden states.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        highway_exits (:obj:`tuple(tuple(torch.Tensor))`:
+            Tuple of each early exit's results (total length: number of layers)
+            Each tuple is again, a tuple of length 2 - the first entry is logits and the second entry is hidden states.
         """
         if input_ids is not None and inputs_embeds is not None:
             raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
@@ -302,32 +302,32 @@ class DeeBertForSequenceClassification(BertPreTrainedModel):
         train_highway=False,
     ):
         r"""
-            labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
-                Labels for computing the sequence classification/regression loss.
-                Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
-                If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
-                If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
+            If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
+            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
 
-        Returns:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
-            loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
-                Classification (or regression if config.num_labels==1) loss.
-            logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
-                Classification (or regression if config.num_labels==1) scores (before SoftMax).
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
+            Classification (or regression if config.num_labels==1) loss.
+        logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
-            highway_exits (:obj:`tuple(tuple(torch.Tensor))`:
-                Tuple of each early exit's results (total length: number of layers)
-                Each tuple is again, a tuple of length 2 - the first entry is logits and the second entry is hidden states.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        highway_exits (:obj:`tuple(tuple(torch.Tensor))`:
+            Tuple of each early exit's results (total length: number of layers)
+            Each tuple is again, a tuple of length 2 - the first entry is logits and the second entry is hidden states.
         """
 
         exit_layer = self.num_layers

examples/deebert/src/modeling_highway_roberta.py

@@ -11,8 +11,7 @@ from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayExc
 
 
 @add_start_docstrings(
-    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). ",
-    ROBERTA_START_DOCSTRING,
+    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). ", ROBERTA_START_DOCSTRING,
 )
 class DeeRobertaModel(DeeBertModel):
 
@@ -59,32 +58,32 @@ class DeeRobertaForSequenceClassification(BertPreTrainedModel):
         train_highway=False,
     ):
         r"""
-            labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
-                Labels for computing the sequence classification/regression loss.
-                Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
-                If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
-                If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
+            If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
+            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
 
-        Returns:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.RobertaConfig`) and inputs:
-            loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
-                Classification (or regression if config.num_labels==1) loss.
-            logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
-                Classification (or regression if config.num_labels==1) scores (before SoftMax).
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.RobertaConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
+            Classification (or regression if config.num_labels==1) loss.
+        logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
-            highway_exits (:obj:`tuple(tuple(torch.Tensor))`:
-                Tuple of each early exit's results (total length: number of layers)
-                Each tuple is again, a tuple of length 2 - the first entry is logits and the second entry is hidden states.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        highway_exits (:obj:`tuple(tuple(torch.Tensor))`:
+            Tuple of each early exit's results (total length: number of layers)
+            Each tuple is again, a tuple of length 2 - the first entry is logits and the second entry is hidden states.
         """
 
         exit_layer = self.num_layers

examples/distillation/run_squad_w_distillation.py

@@ -228,20 +228,14 @@ def train(args, train_dataset, model, tokenizer, teacher=None):
                 assert end_logits_tea.size() == end_logits_stu.size()
 
                 loss_fct = nn.KLDivLoss(reduction="batchmean")
-                loss_start = (
-                    loss_fct(
-                        F.log_softmax(start_logits_stu / args.temperature, dim=-1),
-                        F.softmax(start_logits_tea / args.temperature, dim=-1),
-                    )
-                    * (args.temperature ** 2)
-                )
-                loss_end = (
-                    loss_fct(
-                        F.log_softmax(end_logits_stu / args.temperature, dim=-1),
-                        F.softmax(end_logits_tea / args.temperature, dim=-1),
-                    )
-                    * (args.temperature ** 2)
-                )
+                loss_start = loss_fct(
+                    F.log_softmax(start_logits_stu / args.temperature, dim=-1),
+                    F.softmax(start_logits_tea / args.temperature, dim=-1),
+                ) * (args.temperature ** 2)
+                loss_end = loss_fct(
+                    F.log_softmax(end_logits_stu / args.temperature, dim=-1),
+                    F.softmax(end_logits_tea / args.temperature, dim=-1),
+                ) * (args.temperature ** 2)
                 loss_ce = (loss_start + loss_end) / 2.0
 
                 loss = args.alpha_ce * loss_ce + args.alpha_squad * loss

examples/distillation/utils.py

@@ -118,8 +118,7 @@ def init_gpu_params(params):
     if params.multi_gpu:
         logger.info("Initializing PyTorch distributed")
         torch.distributed.init_process_group(
-            init_method="env://",
-            backend="nccl",
+            init_method="env://", backend="nccl",
         )
 
 

examples/language-modeling/run_language_modeling.py

@@ -233,9 +233,7 @@ def main():
     eval_dataset = get_dataset(data_args, tokenizer=tokenizer, evaluate=True) if training_args.do_eval else None
     if config.model_type == "xlnet":
         data_collator = DataCollatorForPermutationLanguageModeling(
-            tokenizer=tokenizer,
-            plm_probability=data_args.plm_probability,
-            max_span_length=data_args.max_span_length,
+            tokenizer=tokenizer, plm_probability=data_args.plm_probability, max_span_length=data_args.max_span_length,
         )
     else:
         data_collator = DataCollatorForLanguageModeling(

examples/lightning_base.py

@@ -226,14 +226,10 @@ class BaseTransformer(pl.LightningModule):
             help="Decoder layer dropout probability (Optional). Goes into model.config",
         )
         parser.add_argument(
-            "--dropout",
-            type=float,
-            help="Dropout probability (Optional). Goes into model.config",
+            "--dropout", type=float, help="Dropout probability (Optional). Goes into model.config",
         )
         parser.add_argument(
-            "--attention_dropout",
-            type=float,
-            help="Attention dropout probability (Optional). Goes into model.config",
+            "--attention_dropout", type=float, help="Attention dropout probability (Optional). Goes into model.config",
         )
         parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
         parser.add_argument(

examples/longform-qa/eli5_app.py

@@ -95,10 +95,7 @@ def make_support(question, source="wiki40b", method="dense", n_results=10):
             )
         else:
             support_doc, hit_lst = query_es_index(
-                question,
-                es_client,
-                index_name="english_wiki40b_snippets_100w",
-                n_results=n_results,
+                question, es_client, index_name="english_wiki40b_snippets_100w", n_results=n_results,
             )
     support_list = [
         (res["article_title"], res["section_title"].strip(), res["score"], res["passage_text"]) for res in hit_lst
@@ -157,8 +154,7 @@ header_full = """
     header_html,
 )
 st.sidebar.markdown(
-    header_full,
-    unsafe_allow_html=True,
+    header_full, unsafe_allow_html=True,
 )
 
 # Long Form QA with ELI5 and Wikipedia
@@ -177,17 +173,9 @@ action_list = [
 ]
 demo_options = st.sidebar.checkbox("Demo options")
 if demo_options:
-    action_st = st.sidebar.selectbox(
-        "",
-        action_list,
-        index=3,
-    )
+    action_st = st.sidebar.selectbox("", action_list, index=3,)
     action = action_list.index(action_st)
-    show_type = st.sidebar.selectbox(
-        "",
-        ["Show full text of passages", "Show passage section titles"],
-        index=0,
-    )
+    show_type = st.sidebar.selectbox("", ["Show full text of passages", "Show passage section titles"], index=0,)
     show_passages = show_type == "Show full text of passages"
 else:
     action = 3
@@ -262,9 +250,7 @@ questions_list = [
     "How does New Zealand have so many large bird predators?",
 ]
 question_s = st.selectbox(
-    "What would you like to ask? ---- select <MY QUESTION> to enter a new query",
-    questions_list,
-    index=1,
+    "What would you like to ask? ---- select <MY QUESTION> to enter a new query", questions_list, index=1,
 )
 if question_s == "<MY QUESTION>":
     question = st.text_input("Enter your question here:", "")

examples/longform-qa/eli5_utils.py

@@ -48,11 +48,7 @@ def make_es_index_snippets(es_client, passages_dset, index_name="english_wiki_ki
             yield passage
 
     # create the ES index
-    for ok, action in streaming_bulk(
-        client=es_client,
-        index=index_name,
-        actions=passage_generator(),
-    ):
+    for ok, action in streaming_bulk(client=es_client, index=index_name, actions=passage_generator(),):
         progress.update(1)
         successes += ok
     print("Indexed %d documents" % (successes,))
@@ -141,11 +137,7 @@ class RetrievalQAEmbedder(torch.nn.Module):
 
             # define function for checkpointing
             def partial_encode(*inputs):
-                encoder_outputs = self.sent_encoder.encoder(
-                    inputs[0],
-                    attention_mask=inputs[1],
-                    head_mask=head_mask,
-                )
+                encoder_outputs = self.sent_encoder.encoder(inputs[0], attention_mask=inputs[1], head_mask=head_mask,)
                 sequence_output = encoder_outputs[0]
                 pooled_output = self.sent_encoder.pooler(sequence_output)
                 return pooled_output
@@ -242,11 +234,7 @@ def train_qa_retriever_epoch(model, dataset, tokenizer, optimizer, scheduler, ar
         if step % args.print_freq == 0 or step == 1:
             print(
                 "{:2d} {:5d} of {:5d} \t L: {:.3f} \t -- {:.3f}".format(
-                    e,
-                    step,
-                    len(dataset) // args.batch_size,
-                    loc_loss / loc_steps,
-                    time() - st_time,
+                    e, step, len(dataset) // args.batch_size, loc_loss / loc_steps, time() - st_time,
                 )
             )
             loc_loss = 0
@@ -285,11 +273,7 @@ def train_qa_retriever_joint_epoch(model, dataset_list, tokenizer, optimizer, sc
         if step % args.print_freq == 0:
             print(
                 "{:2d} {:5d} of {:5d} \t L: {:.3f} \t -- {:.3f}".format(
-                    e,
-                    step,
-                    len(dataset_list[0]) // args.batch_size,
-                    loc_loss / loc_steps,
-                    time() - st_time,
+                    e, step, len(dataset_list[0]) // args.batch_size, loc_loss / loc_steps, time() - st_time,
                 )
             )
             loc_loss = 0
@@ -370,8 +354,7 @@ class ELI5DatasetS2S(Dataset):
             self.document_cache[q_id] = self.document_cache.get(q_id, self.make_doc_function(example["title"]))
         document = self.document_cache[q_id]
         in_st = "question: {} context: {}".format(
-            question.lower().replace(" --t--", "").strip(),
-            document.lower().strip(),
+            question.lower().replace(" --t--", "").strip(), document.lower().strip(),
         )
         out_st = answer
         return (in_st, out_st)
@@ -444,11 +427,7 @@ def train_qa_s2s_epoch(model, dataset, tokenizer, optimizer, scheduler, args, e=
         if step % args.print_freq == 0 or step == 1:
             print(
                 "{:2d} {:5d} of {:5d} \t L: {:.3f} \t -- {:.3f}".format(
-                    e,
-                    step,
-                    len(dataset) // args.batch_size,
-                    loc_loss / loc_steps,
-                    time() - st_time,
+                    e, step, len(dataset) // args.batch_size, loc_loss / loc_steps, time() - st_time,
                 )
             )
             loc_loss = 0
@@ -477,18 +456,10 @@ def eval_qa_s2s_epoch(model, dataset, tokenizer, args):
             if step % args.print_freq == 0:
                 print(
                     "{:5d} of {:5d} \t L: {:.3f} \t -- {:.3f}".format(
-                        step,
-                        len(dataset) // args.batch_size,
-                        loc_loss / loc_steps,
-                        time() - st_time,
+                        step, len(dataset) // args.batch_size, loc_loss / loc_steps, time() - st_time,
                     )
                 )
-    print(
-        "Total \t L: {:.3f} \t -- {:.3f}".format(
-            loc_loss / loc_steps,
-            time() - st_time,
-        )
-    )
+    print("Total \t L: {:.3f} \t -- {:.3f}".format(loc_loss / loc_steps, time() - st_time,))
 
 
 def train_qa_s2s(qa_s2s_model, qa_s2s_tokenizer, s2s_train_dset, s2s_valid_dset, s2s_args):
@@ -535,12 +506,7 @@ def qa_s2s_generate(
     max_input_length=512,
     device="cuda:0",
 ):
-    model_inputs = make_qa_s2s_batch(
-        [(question_doc, "A")],
-        qa_s2s_tokenizer,
-        max_input_length,
-        device=device,
-    )
+    model_inputs = make_qa_s2s_batch([(question_doc, "A")], qa_s2s_tokenizer, max_input_length, device=device,)
     n_beams = num_answers if num_beams is None else max(num_beams, num_answers)
     generated_ids = qa_s2s_model.generate(
         input_ids=model_inputs["input_ids"],

examples/movement-pruning/emmental/modeling_bert_masked.py

@@ -37,7 +37,8 @@ logger = logging.getLogger(__name__)
 
 
 class BertEmbeddings(nn.Module):
-    """Construct the embeddings from word, position and token_type embeddings."""
+    """Construct the embeddings from word, position and token_type embeddings.
+    """
 
     def __init__(self, config):
         super().__init__()
@@ -384,8 +385,8 @@ class BertPooler(nn.Module):
 
 
 class MaskedBertPreTrainedModel(PreTrainedModel):
-    """An abstract class to handle weights initialization and
-    a simple interface for downloading and loading pretrained models.
+    """ An abstract class to handle weights initialization and
+        a simple interface for downloading and loading pretrained models.
     """
 
     config_class = MaskedBertConfig
@@ -491,9 +492,9 @@ class MaskedBertModel(MaskedBertPreTrainedModel):
         self.embeddings.word_embeddings = value
 
     def _prune_heads(self, heads_to_prune):
-        """Prunes heads of the model.
-        heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
-        See base class PreTrainedModel
+        """ Prunes heads of the model.
+            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+            See base class PreTrainedModel
         """
         for layer, heads in heads_to_prune.items():
             self.encoder.layer[layer].attention.prune_heads(heads)
@@ -684,31 +685,31 @@ class MaskedBertForSequenceClassification(MaskedBertPreTrainedModel):
         threshold=None,
     ):
         r"""
-            labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
-                Labels for computing the sequence classification/regression loss.
-                Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
-                If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
-                If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
-            threshold (:obj:`float`):
-                Threshold value (see :class:`~emmental.MaskedLinear`).
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
+            If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
+            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        threshold (:obj:`float`):
+            Threshold value (see :class:`~emmental.MaskedLinear`).
 
-        Returns:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~emmental.MaskedBertConfig`) and inputs:
-            loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
-                Classification (or regression if config.num_labels==1) loss.
-            logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
-                Classification (or regression if config.num_labels==1) scores (before SoftMax).
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~emmental.MaskedBertConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
+            Classification (or regression if config.num_labels==1) loss.
+        logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
         """
 
         outputs = self.bert(
@@ -769,32 +770,32 @@ class MaskedBertForMultipleChoice(MaskedBertPreTrainedModel):
         threshold=None,
     ):
         r"""
-            labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
-                Labels for computing the multiple choice classification loss.
-                Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension
-                of the input tensors. (see `input_ids` above)
-            threshold (:obj:`float`):
-                Threshold value (see :class:`~emmental.MaskedLinear`).
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the multiple choice classification loss.
+            Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension
+            of the input tensors. (see `input_ids` above)
+        threshold (:obj:`float`):
+            Threshold value (see :class:`~emmental.MaskedLinear`).
 
-        Returns:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~emmental.MaskedBertConfig`) and inputs:
-            loss (:obj:`torch.FloatTensor` of shape `(1,)`, `optional`, returned when :obj:`labels` is provided):
-                Classification loss.
-            classification_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, num_choices)`):
-                `num_choices` is the second dimension of the input tensors. (see `input_ids` above).
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~emmental.MaskedBertConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape `(1,)`, `optional`, returned when :obj:`labels` is provided):
+            Classification loss.
+        classification_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, num_choices)`):
+            `num_choices` is the second dimension of the input tensors. (see `input_ids` above).
 
-                Classification scores (before SoftMax).
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+            Classification scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
 
         """
         num_choices = input_ids.shape[1]
@@ -859,29 +860,29 @@ class MaskedBertForTokenClassification(MaskedBertPreTrainedModel):
         threshold=None,
     ):
         r"""
-            labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
-                Labels for computing the token classification loss.
-                Indices should be in ``[0, ..., config.num_labels - 1]``.
-            threshold (:obj:`float`):
-                Threshold value (see :class:`~emmental.MaskedLinear`).
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the token classification loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+        threshold (:obj:`float`):
+            Threshold value (see :class:`~emmental.MaskedLinear`).
 
-        Returns:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~emmental.MaskedBertConfig`) and inputs:
-            loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when ``labels`` is provided) :
-                Classification loss.
-            scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, config.num_labels)`)
-                Classification scores (before SoftMax).
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~emmental.MaskedBertConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when ``labels`` is provided) :
+            Classification loss.
+        scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, config.num_labels)`)
+            Classification scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
         """
 
         outputs = self.bert(
@@ -946,36 +947,36 @@ class MaskedBertForQuestionAnswering(MaskedBertPreTrainedModel):
         threshold=None,
     ):
         r"""
-            start_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
-                Labels for position (index) of the start of the labelled span for computing the token classification loss.
-                Positions are clamped to the length of the sequence (`sequence_length`).
-                Position outside of the sequence are not taken into account for computing the loss.
-            end_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
-                Labels for position (index) of the end of the labelled span for computing the token classification loss.
-                Positions are clamped to the length of the sequence (`sequence_length`).
-                Position outside of the sequence are not taken into account for computing the loss.
-            threshold (:obj:`float`):
-                Threshold value (see :class:`~emmental.MaskedLinear`).
+        start_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for position (index) of the start of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`).
+            Position outside of the sequence are not taken into account for computing the loss.
+        end_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for position (index) of the end of the labelled span for computing the token classification loss.
+            Positions are clamped to the length of the sequence (`sequence_length`).
+            Position outside of the sequence are not taken into account for computing the loss.
+        threshold (:obj:`float`):
+            Threshold value (see :class:`~emmental.MaskedLinear`).
 
-        Returns:
-            :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~emmental.MaskedBertConfig`) and inputs:
-            loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`labels` is provided):
-                Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
-            start_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length,)`):
-                Span-start scores (before SoftMax).
-            end_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length,)`):
-                Span-end scores (before SoftMax).
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~emmental.MaskedBertConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`labels` is provided):
+            Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
+        start_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length,)`):
+            Span-start scores (before SoftMax).
+        end_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length,)`):
+            Span-end scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
-                :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
-                Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
-                heads.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
         """
 
         outputs = self.bert(
@@ -995,10 +996,7 @@ class MaskedBertForQuestionAnswering(MaskedBertPreTrainedModel):
         start_logits = start_logits.squeeze(-1)
         end_logits = end_logits.squeeze(-1)
 
-        outputs = (
-            start_logits,
-            end_logits,
-        ) + outputs[2:]
+        outputs = (start_logits, end_logits,) + outputs[2:]
         if start_positions is not None and end_positions is not None:
             # If we are on multi-GPU, split add a dimension
             if len(start_positions.size()) > 1:

examples/movement-pruning/masked_run_glue.py

@@ -173,10 +173,7 @@ def train(args, train_dataset, model, tokenizer, teacher=None):
     # Distributed training (should be after apex fp16 initialization)
     if args.local_rank != -1:
         model = torch.nn.parallel.DistributedDataParallel(
-            model,
-            device_ids=[args.local_rank],
-            output_device=args.local_rank,
-            find_unused_parameters=True,
+            model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True,
         )
 
     # Train!
@@ -220,10 +217,7 @@ def train(args, train_dataset, model, tokenizer, teacher=None):
     tr_loss, logging_loss = 0.0, 0.0
     model.zero_grad()
     train_iterator = trange(
-        epochs_trained,
-        int(args.num_train_epochs),
-        desc="Epoch",
-        disable=args.local_rank not in [-1, 0],
+        epochs_trained, int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0],
     )
     set_seed(args)  # Added here for reproductibility
     for _ in train_iterator:
@@ -286,14 +280,11 @@ def train(args, train_dataset, model, tokenizer, teacher=None):
                         attention_mask=inputs["attention_mask"],
                     )
 
-                loss_logits = (
-                    F.kl_div(
-                        input=F.log_softmax(logits_stu / args.temperature, dim=-1),
-                        target=F.softmax(logits_tea / args.temperature, dim=-1),
-                        reduction="batchmean",
-                    )
-                    * (args.temperature ** 2)
-                )
+                loss_logits = F.kl_div(
+                    input=F.log_softmax(logits_stu / args.temperature, dim=-1),
+                    target=F.softmax(logits_tea / args.temperature, dim=-1),
+                    reduction="batchmean",
+                ) * (args.temperature ** 2)
 
                 loss = args.alpha_distil * loss_logits + args.alpha_ce * loss
 
@@ -538,11 +529,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False):
             processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
         )
         features = convert_examples_to_features(
-            examples,
-            tokenizer,
-            max_length=args.max_seq_length,
-            label_list=label_list,
-            output_mode=output_mode,
+            examples, tokenizer, max_length=args.max_seq_length, label_list=label_list, output_mode=output_mode,
         )
         if args.local_rank in [-1, 0]:
             logger.info("Saving features into cached file %s", cached_features_file)
@@ -605,10 +592,7 @@ def main():
     )
     # Other parameters
     parser.add_argument(
-        "--config_name",
-        default="",
-        type=str,
-        help="Pretrained config name or path if not the same as model_name",
+        "--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name",
     )
     parser.add_argument(
         "--tokenizer_name",
@@ -632,27 +616,17 @@ def main():
     parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
     parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.")
     parser.add_argument(
-        "--evaluate_during_training",
-        action="store_true",
-        help="Run evaluation during training at each logging step.",
+        "--evaluate_during_training", action="store_true", help="Run evaluation during training at each logging step.",
     )
     parser.add_argument(
-        "--do_lower_case",
-        action="store_true",
-        help="Set this flag if you are using an uncased model.",
+        "--do_lower_case", action="store_true", help="Set this flag if you are using an uncased model.",
     )
 
     parser.add_argument(
-        "--per_gpu_train_batch_size",
-        default=8,
-        type=int,
-        help="Batch size per GPU/CPU for training.",
+        "--per_gpu_train_batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.",
     )
     parser.add_argument(
-        "--per_gpu_eval_batch_size",
-        default=8,
-        type=int,
-        help="Batch size per GPU/CPU for evaluation.",
+        "--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for evaluation.",
     )
     parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
 
@@ -749,10 +723,7 @@ def main():
     parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
     parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
     parser.add_argument(
-        "--num_train_epochs",
-        default=3.0,
-        type=float,
-        help="Total number of training epochs to perform.",
+        "--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.",
     )
     parser.add_argument(
         "--max_steps",
@@ -771,14 +742,10 @@ def main():
     )
     parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
     parser.add_argument(
-        "--overwrite_output_dir",
-        action="store_true",
-        help="Overwrite the content of the output directory",
+        "--overwrite_output_dir", action="store_true", help="Overwrite the content of the output directory",
     )
     parser.add_argument(
-        "--overwrite_cache",
-        action="store_true",
-        help="Overwrite the cached training and evaluation sets",
+        "--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets",
     )
     parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
 

examples/movement-pruning/masked_run_squad.py

@@ -181,10 +181,7 @@ def train(args, train_dataset, model, tokenizer, teacher=None):
     # Distributed training (should be after apex fp16 initialization)
     if args.local_rank != -1:
         model = torch.nn.parallel.DistributedDataParallel(
-            model,
-            device_ids=[args.local_rank],
-            output_device=args.local_rank,
-            find_unused_parameters=True,
+            model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True,
         )
 
     # Train!
@@ -307,22 +304,16 @@ def train(args, train_dataset, model, tokenizer, teacher=None):
                         attention_mask=inputs["attention_mask"],
                     )
 
-                loss_start = (
-                    F.kl_div(
-                        input=F.log_softmax(start_logits_stu / args.temperature, dim=-1),
-                        target=F.softmax(start_logits_tea / args.temperature, dim=-1),
-                        reduction="batchmean",
-                    )
-                    * (args.temperature ** 2)
-                )
-                loss_end = (
-                    F.kl_div(
-                        input=F.log_softmax(end_logits_stu / args.temperature, dim=-1),
-                        target=F.softmax(end_logits_tea / args.temperature, dim=-1),
-                        reduction="batchmean",
-                    )
-                    * (args.temperature ** 2)
-                )
+                loss_start = F.kl_div(
+                    input=F.log_softmax(start_logits_stu / args.temperature, dim=-1),
+                    target=F.softmax(start_logits_tea / args.temperature, dim=-1),
+                    reduction="batchmean",
+                ) * (args.temperature ** 2)
+                loss_end = F.kl_div(
+                    input=F.log_softmax(end_logits_stu / args.temperature, dim=-1),
+                    target=F.softmax(end_logits_tea / args.temperature, dim=-1),
+                    reduction="batchmean",
+                ) * (args.temperature ** 2)
                 loss_logits = (loss_start + loss_end) / 2.0
 
                 loss = args.alpha_distil * loss_logits + args.alpha_ce * loss
@@ -868,10 +859,7 @@ def main():
     parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
     parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
     parser.add_argument(
-        "--num_train_epochs",
-        default=3.0,
-        type=float,
-        help="Total number of training epochs to perform.",
+        "--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.",
     )
     parser.add_argument(
         "--max_steps",

examples/multiple-choice/utils_multiple_choice.py

@@ -26,8 +26,8 @@ from enum import Enum
 from typing import List, Optional
 
 import tqdm
-
 from filelock import FileLock
+
 from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available
 
 
@@ -100,12 +100,7 @@ if is_torch_available():
 
             cached_features_file = os.path.join(
                 data_dir,
-                "cached_{}_{}_{}_{}".format(
-                    mode.value,
-                    tokenizer.__class__.__name__,
-                    str(max_seq_length),
-                    task,
-                ),
+                "cached_{}_{}_{}_{}".format(mode.value, tokenizer.__class__.__name__, str(max_seq_length), task,),
             )
 
             # Make sure only the first process in distributed training processes the dataset,
@@ -126,12 +121,7 @@ if is_torch_available():
                     else:
                         examples = processor.get_train_examples(data_dir)
                     logger.info("Training examples: %s", len(examples))
-                    self.features = convert_examples_to_features(
-                        examples,
-                        label_list,
-                        max_seq_length,
-                        tokenizer,
-                    )
+                    self.features = convert_examples_to_features(examples, label_list, max_seq_length, tokenizer,)
                     logger.info("Saving features into cached file %s", cached_features_file)
                     torch.save(self.features, cached_features_file)
 
@@ -174,12 +164,7 @@ if is_tf_available():
                 examples = processor.get_train_examples(data_dir)
             logger.info("Training examples: %s", len(examples))
 
-            self.features = convert_examples_to_features(
-                examples,
-                label_list,
-                max_seq_length,
-                tokenizer,
-            )
+            self.features = convert_examples_to_features(examples, label_list, max_seq_length, tokenizer,)
 
             def gen():
                 for (ex_index, ex) in tqdm.tqdm(enumerate(self.features), desc="convert examples to features"):
@@ -506,10 +491,7 @@ class ArcProcessor(DataProcessor):
 
 
 def convert_examples_to_features(
-    examples: List[InputExample],
-    label_list: List[str],
-    max_length: int,
-    tokenizer: PreTrainedTokenizer,
+    examples: List[InputExample], label_list: List[str], max_length: int, tokenizer: PreTrainedTokenizer,
 ) -> List[InputFeatures]:
     """
     Loads a data file into a list of `InputFeatures`

examples/question-answering/run_squad_trainer.py

@@ -137,12 +137,7 @@ def main():
     )
 
     # Initialize our Trainer
-    trainer = Trainer(
-        model=model,
-        args=training_args,
-        train_dataset=train_dataset,
-        eval_dataset=eval_dataset,
-    )
+    trainer = Trainer(model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset,)
 
     # Training
     if training_args.do_train:

examples/question-answering/run_tf_squad.py

@@ -231,12 +231,7 @@ def main():
     eval_dataset = eval_dataset.apply(tf.data.experimental.assert_cardinality(len(eval_examples)))
 
     # Initialize our Trainer
-    trainer = TFTrainer(
-        model=model,
-        args=training_args,
-        train_dataset=train_dataset,
-        eval_dataset=eval_dataset,
-    )
+    trainer = TFTrainer(model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset,)
 
     # Training
     if training_args.do_train:

examples/seq2seq/README.md

@@ -71,7 +71,7 @@ Summarization Tips:
 (It rarely makes sense to start from `bart-large` unless you are a researching finetuning methods).
 
 **Update 2018-07-18**
-Datasets: `LegacySeq2SeqDataset` should be used for all tokenizers without a `prepare_seq2seq_batch` method. For those who do (like Marian, MBart), `Seq2SeqDataset` should be used.**
+Datasets: `Seq2SeqDataset` should be used for all tokenizers without a `prepare_seq2seq_batch` method. For those who do (like Marian, MBart), `TranslationDataset` should be used.**
 A new dataset is needed to support multilingual tasks.
 
 
@@ -122,8 +122,8 @@ Best performing command:
 export ENRO_DIR='wmt_en_ro' # Download instructions above
 # export WANDB_PROJECT="MT" # optional
 export MAX_LEN=128
-export BS=4
-./train_mbart_cc25_enro.sh --output_dir enro_finetune_baseline --label_smoothing 0.1 --fp16_opt_level=O1 --logger_name wandb --sortish_sampler
+export BS=8
+./train_mbart_cc25_enro.sh --output_dir enro_finetune_baseline_dropper --label_smoothing 0 --fp16_opt_level=O1 --logger_name wandb --sortish_sampler
 ```
 This should take < 6h/epoch on a 16GB v100 and achieve test BLEU above 26
 To get results in line with fairseq, you need to do some postprocessing. (see `romanian_postprocessing.md`)
@@ -141,7 +141,7 @@ export BS=4
 As you train, `output_dir` will be filled with files, that look kind of like this (comments are mine).
 Some of them are metrics, some of them are checkpoints, some of them are metadata. Here is a quick tour:
 
-```bash
+```
 output_dir
 ├── best_tfmr  # this is a huggingface checkpoint generated by save_pretrained. It is the same model as the PL .ckpt file below
 │   ├── config.json

examples/seq2seq/bertabs/configuration_bertabs.py

@@ -28,7 +28,7 @@ BERTABS_FINETUNED_CONFIG_MAP = {
 
 
 class BertAbsConfig(PretrainedConfig):
-    r"""Class to store the configuration of the BertAbs model.
+    r""" Class to store the configuration of the BertAbs model.
 
     Arguments:
         vocab_size: int

examples/seq2seq/bertabs/convert_bertabs_original_pytorch_checkpoint.py

@@ -62,7 +62,7 @@ BertAbsConfig = namedtuple(
 
 
 def convert_bertabs_checkpoints(path_to_checkpoints, dump_path):
-    """Copy/paste and tweak the pre-trained weights provided by the creators
+    """ Copy/paste and tweak the pre-trained weights provided by the creators
     of BertAbs for the internal architecture.
     """
 
@@ -164,22 +164,13 @@ def convert_bertabs_checkpoints(path_to_checkpoints, dump_path):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--bertabs_checkpoint_path",
-        default=None,
-        type=str,
-        required=True,
-        help="Path the official PyTorch dump.",
+        "--bertabs_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump.",
     )
     parser.add_argument(
-        "--pytorch_dump_folder_path",
-        default=None,
-        type=str,
-        required=True,
-        help="Path to the output PyTorch model.",
+        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model.",
     )
     args = parser.parse_args()
 
     convert_bertabs_checkpoints(
-        args.bertabs_checkpoint_path,
-        args.pytorch_dump_folder_path,
+        args.bertabs_checkpoint_path, args.pytorch_dump_folder_path,
     )

examples/seq2seq/bertabs/modeling_bertabs.py

@@ -105,17 +105,10 @@ class BertAbs(BertAbsPreTrainedModel):
                 p.data.zero_()
 
     def forward(
-        self,
-        encoder_input_ids,
-        decoder_input_ids,
-        token_type_ids,
-        encoder_attention_mask,
-        decoder_attention_mask,
+        self, encoder_input_ids, decoder_input_ids, token_type_ids, encoder_attention_mask, decoder_attention_mask,
     ):
         encoder_output = self.bert(
-            input_ids=encoder_input_ids,
-            token_type_ids=token_type_ids,
-            attention_mask=encoder_attention_mask,
+            input_ids=encoder_input_ids, token_type_ids=token_type_ids, attention_mask=encoder_attention_mask,
         )
         encoder_hidden_states = encoder_output[0]
         dec_state = self.decoder.init_decoder_state(encoder_input_ids, encoder_hidden_states)
@@ -124,7 +117,8 @@ class BertAbs(BertAbsPreTrainedModel):
 
 
 class Bert(nn.Module):
-    """This class is not really necessary and should probably disappear."""
+    """ This class is not really necessary and should probably disappear.
+    """
 
     def __init__(self):
         super().__init__()
@@ -313,14 +307,7 @@ class TransformerDecoderLayer(nn.Module):
         self.register_buffer("mask", mask)
 
     def forward(
-        self,
-        inputs,
-        memory_bank,
-        src_pad_mask,
-        tgt_pad_mask,
-        previous_input=None,
-        layer_cache=None,
-        step=None,
+        self, inputs, memory_bank, src_pad_mask, tgt_pad_mask, previous_input=None, layer_cache=None, step=None,
     ):
         """
         Args:
@@ -344,25 +331,13 @@ class TransformerDecoderLayer(nn.Module):
             all_input = torch.cat((previous_input, input_norm), dim=1)
             dec_mask = None
 
-        query = self.self_attn(
-            all_input,
-            all_input,
-            input_norm,
-            mask=dec_mask,
-            layer_cache=layer_cache,
-            type="self",
-        )
+        query = self.self_attn(all_input, all_input, input_norm, mask=dec_mask, layer_cache=layer_cache, type="self",)
 
         query = self.drop(query) + inputs
 
         query_norm = self.layer_norm_2(query)
         mid = self.context_attn(
-            memory_bank,
-            memory_bank,
-            query_norm,
-            mask=src_pad_mask,
-            layer_cache=layer_cache,
-            type="context",
+            memory_bank, memory_bank, query_norm, mask=src_pad_mask, layer_cache=layer_cache, type="context",
         )
         output = self.feed_forward(self.drop(mid) + query)
 
@@ -447,14 +422,7 @@ class MultiHeadedAttention(nn.Module):
             self.final_linear = nn.Linear(model_dim, model_dim)
 
     def forward(
-        self,
-        key,
-        value,
-        query,
-        mask=None,
-        layer_cache=None,
-        type=None,
-        predefined_graph_1=None,
+        self, key, value, query, mask=None, layer_cache=None, type=None, predefined_graph_1=None,
     ):
         """
         Compute the context vector and the attention vectors.
@@ -660,7 +628,7 @@ def gelu(x):
 
 
 class PositionwiseFeedForward(nn.Module):
-    """A two-layer Feed-Forward-Network with residual layer norm.
+    """ A two-layer Feed-Forward-Network with residual layer norm.
 
     Args:
         d_model (int): the size of input for the first-layer of the FFN.
@@ -802,7 +770,8 @@ class Translator(object):
         self.max_length = args.max_length
 
     def translate(self, batch, step, attn_debug=False):
-        """Generates summaries from one batch of data."""
+        """ Generates summaries from one batch of data.
+        """
         self.model.eval()
         with torch.no_grad():
             batch_data = self.translate_batch(batch)
@@ -829,7 +798,8 @@ class Translator(object):
     # Where the beam search lives
     # I have no idea why it is being called from the method above
     def _fast_translate_batch(self, batch, max_length, min_length=0):
-        """Beam Search using the encoder inputs contained in `batch`."""
+        """ Beam Search using the encoder inputs contained in `batch`.
+        """
 
         # The batch object is funny
         # Instead of just looking at the size of the arguments we encapsulate
@@ -1011,7 +981,7 @@ def tile(x, count, dim=0):
 
 
 class BertSumOptimizer(object):
-    """Specific optimizer for BertSum.
+    """ Specific optimizer for BertSum.
 
     As described in [1], the authors fine-tune BertSum for abstractive
     summarization using two Adam Optimizers with different warm-up steps and
@@ -1029,16 +999,10 @@ class BertSumOptimizer(object):
 
         self.optimizers = {
             "encoder": torch.optim.Adam(
-                model.encoder.parameters(),
-                lr=lr["encoder"],
-                betas=(beta_1, beta_2),
-                eps=eps,
+                model.encoder.parameters(), lr=lr["encoder"], betas=(beta_1, beta_2), eps=eps,
             ),
             "decoder": torch.optim.Adam(
-                model.decoder.parameters(),
-                lr=lr["decoder"],
-                betas=(beta_1, beta_2),
-                eps=eps,
+                model.decoder.parameters(), lr=lr["decoder"], betas=(beta_1, beta_2), eps=eps,
             ),
         }
 

examples/seq2seq/bertabs/run_summarization.py

@@ -44,9 +44,8 @@ def evaluate(args):
         reference_summaries = []
         generated_summaries = []
 
-        import nltk
-
         import rouge
+        import nltk
 
         nltk.download("punkt")
         rouge_evaluator = rouge.Rouge(
@@ -99,7 +98,7 @@ def evaluate(args):
 
 
 def save_summaries(summaries, path, original_document_name):
-    """Write the summaries in fies that are prefixed by the original
+    """ Write the summaries in fies that are prefixed by the original
     files' name with the `_summary` appended.
 
     Attributes:
@@ -125,7 +124,7 @@ def save_summaries(summaries, path, original_document_name):
 
 
 def format_summary(translation):
-    """Transforms the output of the `from_batch` function
+    """ Transforms the output of the `from_batch` function
     into nicely formatted summaries.
     """
     raw_summary, _, _ = translation
@@ -190,12 +189,7 @@ def build_data_iterator(args, tokenizer):
     def collate_fn(data):
         return collate(data, tokenizer, block_size=512, device=args.device)
 
-    iterator = DataLoader(
-        dataset,
-        sampler=sampler,
-        batch_size=args.batch_size,
-        collate_fn=collate_fn,
-    )
+    iterator = DataLoader(dataset, sampler=sampler, batch_size=args.batch_size, collate_fn=collate_fn,)
 
     return iterator
 
@@ -206,7 +200,7 @@ def load_and_cache_examples(args, tokenizer):
 
 
 def collate(data, tokenizer, block_size, device):
-    """Collate formats the data passed to the data loader.
+    """ Collate formats the data passed to the data loader.
 
     In particular we tokenize the data batch after batch to avoid keeping them
     all in memory. We output the data as a namedtuple to fit the original BertAbs's
@@ -236,7 +230,7 @@ def collate(data, tokenizer, block_size, device):
 
 
 def decode_summary(summary_tokens, tokenizer):
-    """Decode the summary and return it in a format
+    """ Decode the summary and return it in a format
     suitable for evaluation.
     """
     summary_tokens = summary_tokens.to("cpu").numpy()
@@ -247,7 +241,8 @@ def decode_summary(summary_tokens, tokenizer):
 
 
 def main():
-    """The main function defines the interface with the users."""
+    """ The main function defines the interface with the users.
+    """
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--documents_dir",
@@ -272,41 +267,23 @@ def main():
     )
     # EVALUATION options
     parser.add_argument(
-        "--no_cuda",
-        default=False,
-        type=bool,
-        help="Whether to force the execution on CPU.",
+        "--no_cuda", default=False, type=bool, help="Whether to force the execution on CPU.",
     )
     parser.add_argument(
-        "--batch_size",
-        default=4,
-        type=int,
-        help="Batch size per GPU/CPU for training.",
+        "--batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.",
     )
     # BEAM SEARCH arguments
     parser.add_argument(
-        "--min_length",
-        default=50,
-        type=int,
-        help="Minimum number of tokens for the summaries.",
+        "--min_length", default=50, type=int, help="Minimum number of tokens for the summaries.",
     )
     parser.add_argument(
-        "--max_length",
-        default=200,
-        type=int,
-        help="Maixmum number of tokens for the summaries.",
+        "--max_length", default=200, type=int, help="Maixmum number of tokens for the summaries.",
     )
     parser.add_argument(
-        "--beam_size",
-        default=5,
-        type=int,
-        help="The number of beams to start with for each example.",
+        "--beam_size", default=5, type=int, help="The number of beams to start with for each example.",
     )
     parser.add_argument(
-        "--alpha",
-        default=0.95,
-        type=float,
-        help="The value of alpha for the length penalty in the beam search.",
+        "--alpha", default=0.95, type=float, help="The value of alpha for the length penalty in the beam search.",
     )
     parser.add_argument(
         "--block_trigram",

examples/seq2seq/bertabs/test_utils_summarization.py

@@ -43,7 +43,8 @@ class SummarizationDataProcessingTest(unittest.TestCase):
         self.assertEqual(truncate_or_pad(sequence, self.block_size, 0), expected_output)
 
     def test_process_story_no_highlights(self):
-        """Processing a story with no highlights returns an empty list for the summary."""
+        """ Processing a story with no highlights returns an empty list for the summary.
+        """
         raw_story = """It was the year of Our Lord one thousand seven hundred and
         seventy-five.\n\nSpiritual revelations were conceded to England at that
         favoured period, as at this."""
@@ -51,7 +52,8 @@ class SummarizationDataProcessingTest(unittest.TestCase):
         self.assertEqual(summary_lines, [])
 
     def test_process_empty_story(self):
-        """An empty story returns an empty collection of lines."""
+        """ An empty story returns an empty collection of lines.
+        """
         raw_story = ""
         story_lines, summary_lines = process_story(raw_story)
         self.assertEqual(story_lines, [])

examples/seq2seq/bertabs/utils_summarization.py

@@ -11,7 +11,7 @@ from torch.utils.data import Dataset
 
 
 class CNNDMDataset(Dataset):
-    """Abstracts the dataset used to train seq2seq models.
+    """ Abstracts the dataset used to train seq2seq models.
 
     The class will process the documents that are located in the specified
     folder. The preprocessing will work on any document that is reasonably
@@ -31,7 +31,7 @@ class CNNDMDataset(Dataset):
     """
 
     def __init__(self, path="", prefix="train"):
-        """We initialize the class by listing all the documents to summarize.
+        """ We initialize the class by listing all the documents to summarize.
         Files are not read in memory due to the size of some datasets (like CNN/DailyMail).
         """
         assert os.path.isdir(path)
@@ -60,7 +60,7 @@ class CNNDMDataset(Dataset):
 
 
 def process_story(raw_story):
-    """Extract the story and summary from a story file.
+    """ Extract the story and summary from a story file.
 
     Arguments:
         raw_story (str): content of the story file as an utf-8 encoded string.
@@ -108,7 +108,7 @@ def _add_missing_period(line):
 
 
 def truncate_or_pad(sequence, block_size, pad_token_id):
-    """Adapt the source and target sequences' lengths to the block size.
+    """ Adapt the source and target sequences' lengths to the block size.
     If the sequence is shorter we append padding token to the right of the sequence.
     """
     if len(sequence) > block_size:
@@ -119,8 +119,8 @@ def truncate_or_pad(sequence, block_size, pad_token_id):
 
 
 def build_mask(sequence, pad_token_id):
-    """Builds the mask. The attention mechanism will only attend to positions
-    with value 1."""
+    """ Builds the mask. The attention mechanism will only attend to positions
+    with value 1. """
     mask = torch.ones_like(sequence)
     idx_pad_tokens = sequence == pad_token_id
     mask[idx_pad_tokens] = 0
@@ -128,7 +128,7 @@ def build_mask(sequence, pad_token_id):
 
 
 def encode_for_summarization(story_lines, summary_lines, tokenizer):
-    """Encode the story and summary lines, and join them
+    """ Encode the story and summary lines, and join them
     as specified in [1] by using `[SEP] [CLS]` tokens to separate
     sentences.
     """
@@ -141,7 +141,7 @@ def encode_for_summarization(story_lines, summary_lines, tokenizer):
 
 
 def compute_token_type_ids(batch, separator_token_id):
-    """Segment embeddings as described in [1]
+    """ Segment embeddings as described in [1]
 
     The values {0,1} were found in the repository [2].
 

examples/seq2seq/callbacks.py

@@ -97,9 +97,4 @@ def get_checkpoint_callback(output_dir, metric):
 
 
 def get_early_stopping_callback(metric, patience):
-    return EarlyStopping(
-        monitor=f"val_{metric}",
-        mode="max",
-        patience=patience,
-        verbose=True,
-    )
+    return EarlyStopping(monitor=f"val_{metric}", mode="max", patience=patience, verbose=True,)

examples/seq2seq/distillation.py

@@ -1,7 +1,6 @@
 import argparse
 import gc
 import os
-import warnings
 from pathlib import Path
 from typing import List
 
@@ -12,34 +11,31 @@ from torch.nn import functional as F
 
 from lightning_base import generic_train
 from transformers import BartConfig, BartForConditionalGeneration, MBartTokenizer, T5Config, T5ForConditionalGeneration
-from transformers.modeling_bart import shift_tokens_right
 
 
 try:
     from .finetune import SummarizationModule, TranslationModule
-    from .finetune import main as ft_main
-    from .initialization_utils import copy_layers, init_student
+    from .initialization_utils import init_student, copy_layers
     from .utils import (
-        any_requires_grad,
-        assert_all_frozen,
-        calculate_bleu,
-        freeze_params,
-        label_smoothed_nll_loss,
-        pickle_load,
         use_task_specific_params,
+        pickle_load,
+        freeze_params,
+        assert_all_frozen,
+        any_requires_grad,
+        calculate_bleu_score,
     )
+    from .finetune import main as ft_main
 except ImportError:
     from finetune import SummarizationModule, TranslationModule
     from finetune import main as ft_main
-    from initialization_utils import copy_layers, init_student
+    from initialization_utils import init_student, copy_layers
     from utils import (
-        any_requires_grad,
-        assert_all_frozen,
-        calculate_bleu,
-        freeze_params,
-        label_smoothed_nll_loss,
-        pickle_load,
         use_task_specific_params,
+        pickle_load,
+        freeze_params,
+        assert_all_frozen,
+        any_requires_grad,
+        calculate_bleu_score,
     )
 
 
@@ -164,42 +160,22 @@ class BartSummarizationDistiller(SummarizationModule):
     def _step(self, batch):
         # assert is_frozen(self.teacher)
         pad_token_id = self.tokenizer.pad_token_id
-        input_ids, src_mask = batch["input_ids"], batch["attention_mask"]
-        if "labels" in batch:
-            lm_labels = batch["labels"]
-            decoder_input_ids = shift_tokens_right(lm_labels, pad_token_id)
-        else:
-            raise ValueError()
-        # decoder_input_ids = y[:, :-1].contiguous()
-        # labels = y[:, 1:].clone()
-        # labels[y[:, 1:] == pad_token_id] = -100
+        input_ids, src_mask, y = batch["input_ids"], batch["attention_mask"], batch["decoder_input_ids"]
+        decoder_input_ids = y[:, :-1].contiguous()
+        labels = y[:, 1:].clone()
+        labels[y[:, 1:] == pad_token_id] = -100
         # noinspection PyCallingNonCallable
-        outputs = self(
+        sloss, slogits, dec_hidden, enc_outputs, enc_hidden_state = self(
             input_ids,
             attention_mask=src_mask,
             decoder_input_ids=decoder_input_ids,
-            # labels=labels,
+            labels=labels,
             output_hidden_states=True,
             output_attentions=False,
-            # return_dict=True,
-            use_cache=False,
         )
-        lm_logits, dec_hidden, enc_outputs, enc_hidden_state = outputs
-
-        if self.hparams.label_smoothing == 0:
-            # Same behavior as modeling_bart.py, besides pad_token_id
-            loss_fct = torch.nn.CrossEntropyLoss(ignore_index=pad_token_id)
-
-            assert lm_logits.shape[-1] == self.model.config.vocab_size
-            student_lm_loss = loss_fct(lm_logits.view(-1, lm_logits.shape[-1]), lm_labels.view(-1))
-        else:
-            lprobs = torch.nn.functional.log_softmax(lm_logits, dim=-1)
-            student_lm_loss, _ = label_smoothed_nll_loss(
-                lprobs, lm_labels, self.hparams.label_smoothing, ignore_index=pad_token_id
-            )
 
         def zero_tensor():
-            return torch.tensor(0.0).type_as(student_lm_loss)
+            return torch.tensor(0.0).type_as(sloss)
 
         loss_encoder, hid_loss_enc, hid_loss_dec = zero_tensor(), zero_tensor(), zero_tensor()
         if self.different_encoder:
@@ -223,21 +199,21 @@ class BartSummarizationDistiller(SummarizationModule):
                 attention_mask=src_mask,
                 encoder_outputs=teacher_enc_outputs,
                 decoder_input_ids=decoder_input_ids,
-                lm_labels=lm_labels,
+                lm_labels=labels,
                 output_hidden_states=True,
             )
         dec_mask = decoder_input_ids.ne(pad_token_id)
-        loss_ce, s_logits_slct, t_logits_slct = self.calc_ce_loss(dec_mask, lm_logits, tlogits)
+        loss_ce, s_logits_slct, t_logits_slct = self.calc_ce_loss(dec_mask, slogits, tlogits)
         if self.alpha_hid > 0:
             hid_loss_dec = self.calc_hidden_loss(dec_mask, dec_hidden, tdec_hidden, self.hparams.d_layer_to_copy)
 
         blended_loss = (
             self.alpha_ce * loss_ce
-            + self.alpha_mlm * student_lm_loss
+            + self.alpha_mlm * sloss
             + self.hparams.alpha_encoder_loss * loss_encoder
             + self.hparams.alpha_hid * (hid_loss_enc + hid_loss_dec)
         )
-        return blended_loss, loss_ce, student_lm_loss, loss_encoder, hid_loss_enc, hid_loss_dec
+        return blended_loss, loss_ce, sloss, loss_encoder, hid_loss_enc, hid_loss_dec
 
     def calc_hidden_loss(self, attention_mask, hidden_states, hidden_states_T, matches):
         assert not isinstance(
@@ -257,7 +233,7 @@ class BartSummarizationDistiller(SummarizationModule):
 
 
 def add_distill_args(parser):
-    parser.add_argument("--teacher", type=str)
+    parser.add_argument("--teacher", default="facebook/bart-large-cnn", type=str)
     parser.add_argument("--alpha_ce", default=0.8, type=float)
     parser.add_argument("--alpha_mlm", default=0.2, type=float)
     parser.add_argument("--alpha_encoder_loss", default=0.0, type=float)
@@ -270,12 +246,13 @@ def add_distill_args(parser):
 
 class BartTranslationDistiller(BartSummarizationDistiller):
     mode = "translation"
-    loss_names = ["loss", "ce_loss", "mlm_loss", "enc_mse_loss", "hid_loss_enc", "hid_loss_dec"]
+    loss_names = ["loss"]
     metric_names = ["bleu"]
     val_metric = "bleu"
 
     def __init__(self, hparams, **kwargs):
         super().__init__(hparams, **kwargs)
+        assert isinstance(self.tokenizer, MBartTokenizer)
         assert hparams.src_lang is not None
         assert hparams.tgt_lang is not None
         self.dataset_kwargs["src_lang"] = hparams.src_lang
@@ -284,7 +261,7 @@ class BartTranslationDistiller(BartSummarizationDistiller):
             self.decoder_start_token_id = self.tokenizer.lang_code_to_id[hparams.tgt_lang]
 
     def calc_generative_metrics(self, preds, target) -> dict:
-        return calculate_bleu(preds, target)
+        return calculate_bleu_score(preds, target)
 
     @staticmethod
     def add_model_specific_args(parser, root_dir):
@@ -371,10 +348,7 @@ class T5SummarizationDistiller(BartSummarizationDistiller):
         if self.different_encoder:
             with torch.no_grad():
                 teacher_enc_outputs, teacher_enc_hid = self.teacher.encoder(
-                    source_ids,
-                    attention_mask=source_mask,
-                    output_hidden_states=True,
-                    use_cache=False,
+                    source_ids, attention_mask=source_mask, output_hidden_states=True, use_cache=False,
                 )
             if self.hparams.alpha_encoder_loss > 0:
                 loss_encoder = self.calc_mse_loss(enc_outputs, teacher_enc_outputs, source_mask)
@@ -392,7 +366,7 @@ class T5SummarizationDistiller(BartSummarizationDistiller):
                 attention_mask=source_mask,
                 encoder_outputs=teacher_enc_outputs,
                 decoder_input_ids=decoder_input_ids,
-                labels=labels,
+                lm_labels=labels,
                 output_hidden_states=True,
                 use_cache=False,
             )
@@ -448,40 +422,33 @@ def evaluate_checkpoint(ckpt_path: Path, dest_dir=None):
     trainer.test(model)
 
 
-LAYERS_TO_COPY = {
-    # maps  num layers in student -> which teacher layers to copy.
-    # 12:bart, 16: pegasus, 6: marian/Helsinki-NLP
-    12: {
-        1: [0],
-        2: [0, 6],
-        3: [0, 6, 11],
-        4: [0, 4, 8, 11],
-        6: [0, 2, 4, 7, 9, 11],
-        9: [0, 1, 2, 4, 5, 7, 9, 10, 11],
-        12: list(range(12)),
-    },
-    16: {  # maps  num layers in student -> which teacher layers to copy
-        1: [0],
-        2: [0, 8],
-        3: [0, 8, 15],
-        4: [0, 5, 10, 15],
-        6: [0, 3, 6, 9, 12, 15],
-        8: [0, 2, 4, 6, 8, 10, 12, 15],
-        9: [0, 1, 3, 5, 7, 9, 11, 13, 15],
-        16: list(range(16)),
-    },
-    6: {1: [0], 2: [0, 5], 3: [0, 2, 5], 4: [0, 1, 3, 5], 6: list(range(6))},
-}
-
-
-def get_layers_to_copy(n_student, n_teacher):
-    try:
-        return LAYERS_TO_COPY[n_teacher][n_student]
-    except KeyError:
-        warnings.warn(
-            f"no hardcoded layers to copy for teacher {n_teacher} -> student {n_student}, defaulting to first {n_student}"
-        )
-        return list(range(n_student))
+def get_layers_to_copy(n_to_get, tot):
+    all_layers = list(range(tot))
+    if tot == 12:  # Alternating for special cases
+        layers_to_copy = {  # maps  num layers in student -> which teacher layers to copy
+            1: [0],
+            2: [0, 6],
+            3: [0, 6, 11],
+            4: [0, 4, 8, 11],
+            6: [0, 2, 4, 7, 9, 11],
+            9: [0, 1, 2, 4, 5, 7, 9, 10, 11],
+            12: all_layers,
+        }
+        return layers_to_copy[n_to_get]
+    elif tot == 16:
+        layers_to_copy = {  # maps  num layers in student -> which teacher layers to copy
+            1: [0],
+            2: [0, 8],
+            3: [0, 8, 15],
+            4: [0, 5, 10, 15],
+            6: [0, 3, 6, 9, 12, 15],
+            8: [0, 2, 4, 6, 8, 10, 12, 15],
+            9: [0, 1, 3, 5, 7, 9, 11, 13, 15],
+            16: all_layers,
+        }
+        return layers_to_copy[n_to_get]
+    else:
+        return all_layers[:n_to_get]  # TODO: better version on theseus-bart branch
 
 
 def distill_main(args):

examples/seq2seq/finetune.py

@@ -13,61 +13,65 @@ import torch
 from torch.utils.data import DataLoader
 
 from lightning_base import BaseTransformer, add_generic_args, generic_train
-from transformers import MBartTokenizer, T5ForConditionalGeneration
+from transformers import MarianTokenizer, MBartTokenizer, T5ForConditionalGeneration
 from transformers.modeling_bart import shift_tokens_right
 
 
 try:
-    from .callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
     from .utils import (
-        ROUGE_KEYS,
-        LegacySeq2SeqDataset,
-        Seq2SeqDataset,
         assert_all_frozen,
-        calculate_bleu,
-        calculate_rouge,
-        flatten_list,
-        freeze_params,
-        get_git_info,
-        label_smoothed_nll_loss,
+        use_task_specific_params,
         lmap,
+        flatten_list,
         pickle_save,
         save_git_info,
         save_json,
-        use_task_specific_params,
+        freeze_params,
+        calculate_rouge,
+        get_git_info,
+        ROUGE_KEYS,
+        calculate_bleu_score,
+        Seq2SeqDataset,
+        TranslationDataset,
+        label_smoothed_nll_loss,
     )
+
+    from .callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
+    from .loss_dropper import LossDropper
 except ImportError:
-    from callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
     from utils import (
-        ROUGE_KEYS,
-        LegacySeq2SeqDataset,
         Seq2SeqDataset,
+        TranslationDataset,
         assert_all_frozen,
-        calculate_bleu,
-        calculate_rouge,
-        flatten_list,
-        freeze_params,
-        get_git_info,
-        label_smoothed_nll_loss,
+        use_task_specific_params,
         lmap,
+        flatten_list,
         pickle_save,
         save_git_info,
         save_json,
-        use_task_specific_params,
+        freeze_params,
+        calculate_rouge,
+        get_git_info,
+        ROUGE_KEYS,
+        calculate_bleu_score,
+        label_smoothed_nll_loss,
     )
+    from callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
+    from loss_dropper import LossDropper
 
 logger = logging.getLogger(__name__)
 
 
 class SummarizationModule(BaseTransformer):
     mode = "summarization"
-    loss_names = ["loss"]
+    loss_names = ["loss", "dropper_mask_mean"]
     metric_names = ROUGE_KEYS
     val_metric = "rouge2"
 
     def __init__(self, hparams, **kwargs):
         super().__init__(hparams, num_labels=None, mode=self.mode, **kwargs)
         use_task_specific_params(self.model, "summarization")
+        self.dropper = LossDropper(dropc=.05)
         save_git_info(self.hparams.output_dir)
         self.metrics_save_path = Path(self.output_dir) / "metrics.json"
         self.hparams_save_path = Path(self.output_dir) / "hparams.pkl"
@@ -103,13 +107,14 @@ class SummarizationModule(BaseTransformer):
 
         self.hparams.git_sha = get_git_info()["repo_sha"]
         self.num_workers = hparams.num_workers
-        self.decoder_start_token_id = None  # default to config
+        self.decoder_start_token_id = None
         if self.model.config.decoder_start_token_id is None and isinstance(self.tokenizer, MBartTokenizer):
             self.decoder_start_token_id = self.tokenizer.lang_code_to_id[hparams.tgt_lang]
             self.model.config.decoder_start_token_id = self.decoder_start_token_id
-        self.dataset_class = (
-            Seq2SeqDataset if hasattr(self.tokenizer, "prepare_seq2seq_batch") else LegacySeq2SeqDataset
-        )
+        if isinstance(self.tokenizer, MBartTokenizer) or isinstance(self.tokenizer, MarianTokenizer):
+            self.dataset_class = TranslationDataset
+        else:
+            self.dataset_class = Seq2SeqDataset
 
     def freeze_embeds(self):
         """Freeze token embeddings and positional embeddings for bart, just token embeddings for t5."""
@@ -134,34 +139,45 @@ class SummarizationModule(BaseTransformer):
 
     def _step(self, batch: dict) -> Tuple:
         pad_token_id = self.tokenizer.pad_token_id
-        source_ids, source_mask = batch["input_ids"], batch["attention_mask"]  # , batch["decoder_input_ids"]
+        source_ids, source_mask, target_ids = batch["input_ids"], batch["attention_mask"], batch["decoder_input_ids"]
 
         if "labels" in batch:
             lm_labels = batch["labels"]
             decoder_input_ids = shift_tokens_right(lm_labels, pad_token_id)
         elif isinstance(self.model, T5ForConditionalGeneration):
-            lm_labels = batch["labels"]
-            decoder_input_ids = self.model._shift_right(lm_labels)
+            decoder_input_ids = self.model._shift_right(target_ids)
+            lm_labels = target_ids
         else:
-            target_ids = batch["decoder_input_ids"]
-            # This is a slightly worse way of shifting tokens right -- it deletes token 0 from target_id
-            decoder_input_ids = target_ids[:, :-1].contiguous()
-            lm_labels = target_ids[:, 1:].clone()
+            decoder_input_ids = target_ids[:, :-1].contiguous()  # Why this line?
+            lm_labels = target_ids[:, 1:].clone()  # why clone?
 
         outputs = self(source_ids, attention_mask=source_mask, decoder_input_ids=decoder_input_ids, use_cache=False)
+        bs = source_ids.shape[0]
 
         if self.hparams.label_smoothing == 0:
-            # Same behavior as modeling_bart.py, besides ignoring pad_token_id
-            loss_fct = torch.nn.CrossEntropyLoss(ignore_index=pad_token_id)
+
+            # Same behavior as modeling_bart.py
+            loss_fct = torch.nn.CrossEntropyLoss(reduction='none', ignore_index=pad_token_id)
             lm_logits = outputs[0]
             assert lm_logits.shape[-1] == self.model.config.vocab_size
+
+            #loss_fct = torch.nn.NLLLoss(reduction='none', ignore_index=pad_token_id)
+            #logit_shape =
+            #weights = torch.ones(logit_shape
             loss = loss_fct(lm_logits.view(-1, lm_logits.shape[-1]), lm_labels.view(-1))
+            loss = loss.view(-1, bs)
+            loss = loss.mean(dim=0)
+            mask = self.dropper(loss)
+            loss *= mask
+            loss = loss.mean()
+            return (loss, 1-mask.mean())
+            #loss = loss.view(-1, bs)
         else:
             lprobs = torch.nn.functional.log_softmax(outputs[0], dim=-1)
             loss, nll_loss = label_smoothed_nll_loss(
                 lprobs, lm_labels, self.hparams.label_smoothing, ignore_index=pad_token_id
             )
-        return (loss,)
+        return (loss,torch.tensor(1.))
 
     @property
     def pad(self) -> int:
@@ -172,7 +188,7 @@ class SummarizationModule(BaseTransformer):
 
         logs = {name: loss for name, loss in zip(self.loss_names, loss_tensors)}
         # tokens per batch
-        logs["tpb"] = batch["input_ids"].ne(self.pad).sum() + batch["labels"].ne(self.pad).sum()
+        logs["tpb"] = batch["input_ids"].ne(self.pad).sum() + batch["decoder_input_ids"].ne(self.pad).sum()
         return {"loss": loss_tensors[0], "log": logs}
 
     def validation_step(self, batch, batch_idx) -> Dict:
@@ -209,7 +225,7 @@ class SummarizationModule(BaseTransformer):
         )
         gen_time = (time.time() - t0) / batch["input_ids"].shape[0]
         preds: List[str] = self.ids_to_clean_text(generated_ids)
-        target: List[str] = self.ids_to_clean_text(batch["labels"])
+        target: List[str] = self.ids_to_clean_text(batch["decoder_input_ids"])
         loss_tensors = self._step(batch)
         base_metrics = {name: loss for name, loss in zip(self.loss_names, loss_tensors)}
         rouge: Dict = self.calc_generative_metrics(preds, target)
@@ -306,6 +322,7 @@ class SummarizationModule(BaseTransformer):
             "--task", type=str, default="summarization", required=False, help="# examples. -1 means use all."
         )
         parser.add_argument("--label_smoothing", type=float, default=0.0, required=False)
+        parser.add_argument("--loss_dropper", type=float, default=0.0, required=False)
         parser.add_argument("--src_lang", type=str, default="", required=False)
         parser.add_argument("--tgt_lang", type=str, default="", required=False)
         parser.add_argument(
@@ -320,7 +337,6 @@ class SummarizationModule(BaseTransformer):
 
 class TranslationModule(SummarizationModule):
     mode = "translation"
-    loss_names = ["loss"]
     metric_names = ["bleu"]
     val_metric = "bleu"
 
@@ -330,7 +346,7 @@ class TranslationModule(SummarizationModule):
         self.dataset_kwargs["tgt_lang"] = hparams.tgt_lang
 
     def calc_generative_metrics(self, preds, target) -> dict:
-        return calculate_bleu(preds, target)
+        return calculate_bleu_score(preds, target)
 
 
 def main(args, model=None) -> SummarizationModule:

examples/seq2seq/loss_dropper.py

@@ -0,0 +1,53 @@
+import numpy as np
+import torch.nn as nn
+
+
+class LossDropper(nn.Module):
+    def __init__(
+            self,
+            dropc=0.4,
+            min_count=10000,
+            recompute=10000,
+            verbose=True
+    ):
+        super().__init__()
+        self.keepc = 1. - dropc
+        self.count = 0
+        self.min_count = min_count
+
+        self.recompute = recompute
+        self.last_computed = 0
+        self.percentile_val = 100000000.
+        self.cur_idx = 0
+
+        self.verbose = verbose
+
+        self.vals = np.zeros(self.recompute, dtype=np.float32)
+
+    def forward(self, loss):
+        if loss is None:
+            return loss
+
+        self.last_computed += loss.numel()
+        self.count += loss.numel()
+        if self.count < len(self.vals):
+            self.vals[self.count - loss.numel():self.count] = loss.detach().cpu().numpy().flatten()
+            self.cur_idx += loss.numel()
+            return (loss < np.inf).type(loss.dtype)
+        else:
+            for idx, item in enumerate(loss):
+                self.vals[self.cur_idx] = item
+                self.cur_idx += 1
+                if self.cur_idx >= len(self.vals):
+                    self.cur_idx = 0
+        if self.count < self.min_count:
+            return (loss < np.inf).type(loss.dtype)
+
+        if self.last_computed > self.recompute:
+            self.percentile_val = np.percentile(self.vals, self.keepc * 100)
+            if self.verbose:
+                print('Using cutoff', self.percentile_val)
+            self.last_computed = 0
+
+        mask = (loss < self.percentile_val).type(loss.dtype)
+        return mask

examples/seq2seq/run_eval.py

@@ -9,9 +9,9 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 
 
 try:
-    from .utils import calculate_bleu, calculate_rouge, trim_batch, use_task_specific_params
+    from .utils import calculate_rouge, use_task_specific_params, calculate_bleu_score, trim_batch
 except ImportError:
-    from utils import calculate_bleu, calculate_rouge, trim_batch, use_task_specific_params
+    from utils import calculate_rouge, use_task_specific_params, calculate_bleu_score, trim_batch
 
 DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
@@ -103,7 +103,7 @@ def run_generate():
     if args.reference_path is None:
         return
     # Compute scores
-    score_fn = calculate_bleu if "translation" in args.task else calculate_rouge
+    score_fn = calculate_bleu_score if "translation" in args.task else calculate_rouge
     output_lns = [x.rstrip() for x in open(args.save_path).readlines()]
     reference_lns = [x.rstrip() for x in open(args.reference_path).readlines()][: len(output_lns)]
     scores: dict = score_fn(output_lns, reference_lns)

examples/seq2seq/test_seq2seq_examples.py

@@ -15,14 +15,13 @@ from torch.utils.data import DataLoader
 
 import lightning_base
 from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
-from transformers.modeling_bart import shift_tokens_right
 from transformers.testing_utils import CaptureStderr, CaptureStdout, require_multigpu
 
 from .distillation import distill_main, evaluate_checkpoint
 from .finetune import SummarizationModule, main
 from .pack_dataset import pack_data_dir
 from .run_eval import generate_summaries_or_translations, run_generate
-from .utils import LegacySeq2SeqDataset, Seq2SeqDataset, label_smoothed_nll_loss, lmap, load_json
+from .utils import Seq2SeqDataset, TranslationDataset, label_smoothed_nll_loss, lmap, load_json
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -118,12 +117,7 @@ class TestSummarizationDistiller(unittest.TestCase):
 
     @require_multigpu
     def test_multigpu(self):
-        updates = dict(
-            no_teacher=True,
-            freeze_encoder=True,
-            gpus=2,
-            sortish_sampler=False,
-        )
+        updates = dict(no_teacher=True, freeze_encoder=True, gpus=2, sortish_sampler=False,)
         self._test_distiller_cli(updates)
 
     def test_distill_no_teacher(self):
@@ -267,8 +261,7 @@ def test_run_eval_bart(model):
 
 
 @pytest.mark.parametrize(
-    ["model"],
-    [pytest.param(T5_TINY), pytest.param(BART_TINY), pytest.param(MBART_TINY), pytest.param(MARIAN_TINY)],
+    ["model"], [pytest.param(T5_TINY), pytest.param(BART_TINY), pytest.param(MBART_TINY), pytest.param(MARIAN_TINY)],
 )
 def test_finetune(model):
     args_d: dict = CHEAP_ARGS.copy()
@@ -336,8 +329,7 @@ def test_finetune_extra_model_args():
     output_dir = tempfile.mkdtemp(prefix="output_1_")
     args_d1 = args_d.copy()
     args_d1.update(
-        model_name_or_path=model,
-        output_dir=output_dir,
+        model_name_or_path=model, output_dir=output_dir,
     )
     extra_model_params = ("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
     for p in extra_model_params:
@@ -352,8 +344,7 @@ def test_finetune_extra_model_args():
     output_dir = tempfile.mkdtemp(prefix="output_2_")
     args_d2 = args_d.copy()
     args_d2.update(
-        model_name_or_path=model,
-        output_dir=output_dir,
+        model_name_or_path=model, output_dir=output_dir,
     )
     unsupported_param = "encoder_layerdrop"
     args_d2[unsupported_param] = 0.5
@@ -440,20 +431,18 @@ def test_pack_dataset():
     assert orig_paths == new_paths
 
 
-@pytest.mark.parametrize(
-    ["tok_name"], [pytest.param(MBART_TINY), pytest.param(MARIAN_TINY), pytest.param(T5_TINY), pytest.param(BART_TINY)]
-)
-def test_seq2seq_dataset_truncation(tok_name):
+@pytest.mark.parametrize(["tok_name"], [pytest.param(MBART_TINY), pytest.param(MARIAN_TINY)])
+def test_mbart_dataset_truncation(tok_name):
     tokenizer = AutoTokenizer.from_pretrained(tok_name)
     tmp_dir = make_test_data_dir()
     max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
     max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
     max_src_len = 4
     max_tgt_len = 8
-    assert max_len_target > max_src_len  # Will be truncated
-    assert max_len_source > max_src_len  # Will be truncated
-    src_lang, tgt_lang = "ro_RO", "de_DE"  # ignored for all but mbart, but never causes error.
-    train_dataset = Seq2SeqDataset(
+    assert max_len_target > max_src_len  # Truncated
+    assert max_len_source > max_src_len
+    src_lang, tgt_lang = "ro_RO", "de_DE"  # NOT WHAT IT WAS TRAINED ON
+    train_dataset = TranslationDataset(
         tokenizer,
         data_dir=tmp_dir,
         type_path="train",
@@ -469,11 +458,10 @@ def test_seq2seq_dataset_truncation(tok_name):
         # show that articles were trimmed.
         assert batch["input_ids"].shape[1] == max_src_len
         # show that targets are the same len
-        assert batch["labels"].shape[1] == max_tgt_len
-        if tok_name != MBART_TINY:
+        assert batch["decoder_input_ids"].shape[1] == max_tgt_len
+        if tok_name == MARIAN_TINY:
             continue
         # check language codes in correct place
-        batch["decoder_input_ids"] = shift_tokens_right(batch["labels"], tokenizer.pad_token_id)
         assert batch["decoder_input_ids"][0, 0].item() == tokenizer.lang_code_to_id[tgt_lang]
         assert batch["decoder_input_ids"][0, -1].item() == tokenizer.eos_token_id
         assert batch["input_ids"][0, -2].item() == tokenizer.eos_token_id
@@ -483,18 +471,14 @@ def test_seq2seq_dataset_truncation(tok_name):
 
 
 @pytest.mark.parametrize(["tok"], [pytest.param(T5_TINY), pytest.param(BART_TINY), param(MARIAN_TINY)])
-def test_legacy_dataset_truncation(tok):
+def test_summarization_dataset_truncation(tok):
     tokenizer = AutoTokenizer.from_pretrained(tok)
     tmp_dir = make_test_data_dir()
     max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
     max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
     trunc_target = 4
-    train_dataset = LegacySeq2SeqDataset(
-        tokenizer,
-        data_dir=tmp_dir,
-        type_path="train",
-        max_source_length=20,
-        max_target_length=trunc_target,
+    train_dataset = Seq2SeqDataset(
+        tokenizer, data_dir=tmp_dir, type_path="train", max_source_length=20, max_target_length=trunc_target,
     )
     dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
     for batch in dataloader:

examples/seq2seq/train_distilbart_cnn.sh

@@ -6,7 +6,7 @@ export GAS=1
 
 python finetune.py \
     --learning_rate=3e-5 \
-    --fp16 \
+    --fp16 --fp16_opt_level=O1 \
     --gpus 1 \
     --do_train \
     --do_predict \

examples/seq2seq/train_distilbart_xsum.sh

@@ -6,7 +6,7 @@ python distillation.py \
   --learning_rate=3e-4 \
   --do_train \
   --do_predict \
-  --fp16 \
+  --fp16 --fp16_opt_level=O1 \
   --val_check_interval 0.1 --n_val 1000 \
   --teacher facebook/bart-large-xsum --data_dir $XSUM_DIR \
   --max_target_length=60 --val_max_target_length=60 --test_max_target_length=100 \

examples/seq2seq/train_mbart_cc25_enro.sh

@@ -14,5 +14,4 @@ python finetune.py \
     --task translation \
     --warmup_steps 500 \
     --freeze_embeds \
-    --model_name_or_path=facebook/mbart-large-cc25 \
     "$@"

examples/seq2seq/utils.py

@@ -57,15 +57,13 @@ def lmap(f: Callable, x: Iterable) -> List:
     return list(map(f, x))
 
 
-def calculate_bleu(output_lns, refs_lns, **kwargs) -> dict:
+def calculate_bleu_score(output_lns, refs_lns, **kwargs) -> dict:
     """Uses sacrebleu's corpus_bleu implementation."""
-    return {"bleu": round(corpus_bleu(output_lns, [refs_lns], **kwargs).score, 4)}
+    return {"bleu": corpus_bleu(output_lns, [refs_lns], **kwargs).score}
 
 
 def trim_batch(
-    input_ids,
-    pad_token_id,
-    attention_mask=None,
+    input_ids, pad_token_id, attention_mask=None,
 ):
     """Remove columns that are populated exclusively by pad_token_id"""
     keep_column_mask = input_ids.ne(pad_token_id).any(dim=0)
@@ -75,7 +73,7 @@ def trim_batch(
         return (input_ids[:, keep_column_mask], attention_mask[:, keep_column_mask])
 
 
-class LegacySeq2SeqDataset(Dataset):
+class Seq2SeqDataset(Dataset):
     def __init__(
         self,
         tokenizer,
@@ -146,7 +144,7 @@ class LegacySeq2SeqDataset(Dataset):
         return SortishSampler(self.src_lens, batch_size)
 
 
-class Seq2SeqDataset(LegacySeq2SeqDataset):
+class TranslationDataset(Seq2SeqDataset):
     """A dataset that calls prepare_seq2seq_batch."""
 
     def __init__(self, *args, **kwargs):
@@ -176,7 +174,6 @@ class Seq2SeqDataset(LegacySeq2SeqDataset):
             tgt_lang=self.tgt_lang,
             max_length=self.max_source_length,
             max_target_length=self.max_target_length,
-            return_tensors="pt",
         )
         return batch_encoding.data
 
@@ -274,7 +271,7 @@ def calculate_rouge(output_lns: List[str], reference_lns: List[str], use_stemmer
         aggregator.add_scores(scores)
 
     result = aggregator.aggregate()
-    return {k: round(v.mid.fmeasure * 100, 4) for k, v in result.items()}
+    return {k: v.mid.fmeasure * 100 for k, v in result.items()}
 
 
 def freeze_params(model: nn.Module):

examples/test_examples.py

@@ -22,8 +22,7 @@ from unittest.mock import patch
 
 import torch
 
-from transformers.file_utils import is_apex_available
-from transformers.testing_utils import TestCasePlus, torch_device
+from transformers.testing_utils import TestCasePlus
 
 
 SRC_DIRS = [
@@ -36,8 +35,8 @@ sys.path.extend(SRC_DIRS)
 if SRC_DIRS is not None:
     import run_generation
     import run_glue
-    import run_language_modeling
     import run_pl_glue
+    import run_language_modeling
     import run_squad
 
 
@@ -53,11 +52,6 @@ def get_setup_file():
     return args.f
 
 
-def is_cuda_and_apex_avaliable():
-    is_using_cuda = torch.cuda.is_available() and torch_device == "cuda"
-    return is_using_cuda and is_apex_available()
-
-
 class ExamplesTests(TestCasePlus):
     def test_run_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
@@ -80,13 +74,7 @@ class ExamplesTests(TestCasePlus):
             --warmup_steps=2
             --seed=42
             --max_seq_length=128
-            """
-        output_dir = "./tests/fixtures/tests_samples/temp_dir_{}".format(hash(testargs))
-        testargs += "--output_dir " + output_dir
-        testargs = testargs.split()
-
-        if is_cuda_and_apex_avaliable():
-            testargs.append("--fp16")
+            """.split()
 
         with patch.object(sys, "argv", testargs):
             result = run_glue.main()
@@ -147,13 +135,8 @@ class ExamplesTests(TestCasePlus):
             --do_train
             --do_eval
             --num_train_epochs=1
-            """
-        output_dir = "./tests/fixtures/tests_samples/temp_dir_{}".format(hash(testargs))
-        testargs += "--output_dir " + output_dir
-        testargs = testargs.split()
-
-        if torch_device != "cuda":
-            testargs.append("--no_cuda")
+            --no_cuda
+            """.split()
 
         with patch.object(sys, "argv", testargs):
             result = run_language_modeling.main()
@@ -192,14 +175,7 @@ class ExamplesTests(TestCasePlus):
         logger.addHandler(stream_handler)
 
         testargs = ["run_generation.py", "--prompt=Hello", "--length=10", "--seed=42"]
-
-        if is_cuda_and_apex_avaliable():
-            testargs.append("--fp16")
-
-        model_type, model_name = (
-            "--model_type=gpt2",
-            "--model_name_or_path=sshleifer/tiny-gpt2",
-        )
+        model_type, model_name = ("--model_type=gpt2", "--model_name_or_path=sshleifer/tiny-gpt2")
         with patch.object(sys, "argv", testargs + [model_type, model_name]):
             result = run_generation.main()
             self.assertGreaterEqual(len(result[0]), 10)

examples/text-classification/run_pl_glue.py

@@ -153,11 +153,7 @@ class GLUETransformer(BaseTransformer):
         )
 
         parser.add_argument(
-            "--task",
-            default="",
-            type=str,
-            required=True,
-            help="The GLUE task to run",
+            "--task", default="", type=str, required=True, help="The GLUE task to run",
         )
         parser.add_argument(
             "--gpus",
@@ -181,10 +177,7 @@ def main():
 
     # If output_dir not provided, a folder will be generated in pwd
     if args.output_dir is None:
-        args.output_dir = os.path.join(
-            "./results",
-            f"{args.task}_{time.strftime('%Y%m%d_%H%M%S')}",
-        )
+        args.output_dir = os.path.join("./results", f"{args.task}_{time.strftime('%Y%m%d_%H%M%S')}",)
         os.makedirs(args.output_dir)
 
     model = GLUETransformer(args)

examples/text-classification/run_xnli.py

@@ -328,11 +328,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False):
             processor.get_test_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
         )
         features = convert_examples_to_features(
-            examples,
-            tokenizer,
-            max_length=args.max_seq_length,
-            label_list=label_list,
-            output_mode=output_mode,
+            examples, tokenizer, max_length=args.max_seq_length, label_list=label_list, output_mode=output_mode,
         )
         if args.local_rank in [-1, 0]:
             logger.info("Saving features into cached file %s", cached_features_file)

examples/text-generation/pplm/run_pplm.py

@@ -698,9 +698,7 @@ def run_pplm_example(
                 for word_id in pert_gen_tok_text.tolist()[0]:
                     if word_id in bow_word_ids:
                         pert_gen_text += "{}{}{}".format(
-                            colorama.Fore.RED,
-                            tokenizer.decode([word_id]),
-                            colorama.Style.RESET_ALL,
+                            colorama.Fore.RED, tokenizer.decode([word_id]), colorama.Style.RESET_ALL,
                         )
                     else:
                         pert_gen_text += tokenizer.decode([word_id])
@@ -731,10 +729,7 @@ if __name__ == "__main__":
     parser.add_argument("--cond_text", type=str, default="The lake", help="Prefix texts to condition on")
     parser.add_argument("--uncond", action="store_true", help="Generate from end-of-text as prefix")
     parser.add_argument(
-        "--num_samples",
-        type=int,
-        default=1,
-        help="Number of samples to generate from the modified latents",
+        "--num_samples", type=int, default=1, help="Number of samples to generate from the modified latents",
     )
     parser.add_argument(
         "--bag_of_words",
@@ -756,22 +751,13 @@ if __name__ == "__main__":
         help="Discriminator to use",
     )
     parser.add_argument(
-        "--discrim_weights",
-        type=str,
-        default=None,
-        help="Weights for the generic discriminator",
+        "--discrim_weights", type=str, default=None, help="Weights for the generic discriminator",
     )
     parser.add_argument(
-        "--discrim_meta",
-        type=str,
-        default=None,
-        help="Meta information for the generic discriminator",
+        "--discrim_meta", type=str, default=None, help="Meta information for the generic discriminator",
     )
     parser.add_argument(
-        "--class_label",
-        type=int,
-        default=-1,
-        help="Class label used for the discriminator",
+        "--class_label", type=int, default=-1, help="Class label used for the discriminator",
     )
     parser.add_argument("--length", type=int, default=100)
     parser.add_argument("--stepsize", type=float, default=0.02)
@@ -787,10 +773,7 @@ if __name__ == "__main__":
         help="Length of past which is being optimized; 0 corresponds to infinite window length",
     )
     parser.add_argument(
-        "--horizon_length",
-        type=int,
-        default=1,
-        help="Length of future to optimize over",
+        "--horizon_length", type=int, default=1, help="Length of future to optimize over",
     )
     parser.add_argument("--decay", action="store_true", help="whether to decay or not")
     parser.add_argument("--gamma", type=float, default=1.5)
@@ -800,10 +783,7 @@ if __name__ == "__main__":
     parser.add_argument("--no_cuda", action="store_true", help="no cuda")
     parser.add_argument("--colorama", action="store_true", help="colors keywords")
     parser.add_argument(
-        "--repetition_penalty",
-        type=float,
-        default=1.0,
-        help="Penalize repetition. More than 1.0 -> less repetition",
+        "--repetition_penalty", type=float, default=1.0, help="Penalize repetition. More than 1.0 -> less repetition",
     )
 
     args = parser.parse_args()

examples/text-generation/pplm/run_pplm_discrim_train.py

@@ -242,12 +242,7 @@ def train_discriminator(
 
         text = torchtext_data.Field()
         label = torchtext_data.Field(sequential=False)
-        train_data, val_data, test_data = datasets.SST.splits(
-            text,
-            label,
-            fine_grained=True,
-            train_subtrees=True,
-        )
+        train_data, val_data, test_data = datasets.SST.splits(text, label, fine_grained=True, train_subtrees=True,)
 
         x = []
         y = []

examples/text-generation/run_generation.py

@@ -41,9 +41,7 @@ from transformers import (
 
 
 logging.basicConfig(
-    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
-    datefmt="%m/%d/%Y %H:%M:%S",
-    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO,
 )
 logger = logging.getLogger(__name__)
 
@@ -188,23 +186,11 @@ def main():
     parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
     parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
     parser.add_argument("--num_return_sequences", type=int, default=1, help="The number of samples to generate.")
-    parser.add_argument(
-        "--fp16",
-        action="store_true",
-        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
-    )
     args = parser.parse_args()
 
     args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
     args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
 
-    logger.warning(
-        "device: %s, n_gpu: %s, 16-bits training: %s",
-        args.device,
-        args.n_gpu,
-        args.fp16,
-    )
-
     set_seed(args)
 
     # Initialize the model and tokenizer
@@ -218,9 +204,6 @@ def main():
     model = model_class.from_pretrained(args.model_name_or_path)
     model.to(args.device)
 
-    if args.fp16:
-        model.half()
-
     args.length = adjust_length_to_model(args.length, max_sequence_length=model.config.max_position_embeddings)
     logger.info(args)
 

examples/token-classification/run.sh

@@ -1,11 +1,8 @@
-## The relevant files are currently on a shared Google
-## drive at https://drive.google.com/drive/folders/1kC0I2UGl2ltrluI9NqDjaQJGw5iliw_J
-## Monitor for changes and eventually migrate to nlp dataset
-curl -L 'https://drive.google.com/uc?export=download&id=1Jjhbal535VVz2ap4v4r_rN1UEHTdLK5P' \
+curl -L 'https://sites.google.com/site/germeval2014ner/data/NER-de-train.tsv?attredirects=0&d=1' \
 | grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > train.txt.tmp
-curl -L 'https://drive.google.com/uc?export=download&id=1ZfRcQThdtAR5PPRjIDtrVP7BtXSCUBbm' \
+curl -L 'https://sites.google.com/site/germeval2014ner/data/NER-de-dev.tsv?attredirects=0&d=1' \
 | grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > dev.txt.tmp
-curl -L 'https://drive.google.com/uc?export=download&id=1u9mb7kNJHWQCWyweMDRMuTFoOHOfeBTH' \
+curl -L 'https://sites.google.com/site/germeval2014ner/data/NER-de-test.tsv?attredirects=0&d=1' \
 | grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > test.txt.tmp
 
 export MAX_LENGTH=128

examples/token-classification/run_pl.sh

@@ -3,14 +3,11 @@
 # for seqeval metrics import
 pip install -r ../requirements.txt
 
-## The relevant files are currently on a shared Google
-## drive at https://drive.google.com/drive/folders/1kC0I2UGl2ltrluI9NqDjaQJGw5iliw_J
-## Monitor for changes and eventually migrate to nlp dataset
-curl -L 'https://drive.google.com/uc?export=download&id=1Jjhbal535VVz2ap4v4r_rN1UEHTdLK5P' \
+curl -L 'https://sites.google.com/site/germeval2014ner/data/NER-de-train.tsv?attredirects=0&d=1' \
 | grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > train.txt.tmp
-curl -L 'https://drive.google.com/uc?export=download&id=1ZfRcQThdtAR5PPRjIDtrVP7BtXSCUBbm' \
+curl -L 'https://sites.google.com/site/germeval2014ner/data/NER-de-dev.tsv?attredirects=0&d=1' \
 | grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > dev.txt.tmp
-curl -L 'https://drive.google.com/uc?export=download&id=1u9mb7kNJHWQCWyweMDRMuTFoOHOfeBTH' \
+curl -L 'https://sites.google.com/site/germeval2014ner/data/NER-de-test.tsv?attredirects=0&d=1' \
 | grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > test.txt.tmp
 
 export MAX_LENGTH=128
@@ -32,6 +29,7 @@ mkdir -p $OUTPUT_DIR
 export PYTHONPATH="../":"${PYTHONPATH}"
 
 python3 run_pl_ner.py --data_dir ./ \
+--model_type bert \
 --labels ./labels.txt \
 --model_name_or_path $BERT_MODEL \
 --output_dir $OUTPUT_DIR \
@@ -39,6 +37,5 @@ python3 run_pl_ner.py --data_dir ./ \
 --num_train_epochs $NUM_EPOCHS \
 --train_batch_size $BATCH_SIZE \
 --seed $SEED \
---gpus 1 \
 --do_train \
 --do_predict

examples/token-classification/run_pl_ner.py

@@ -86,7 +86,7 @@ class NERTransformer(BaseTransformer):
                 logger.info("Saving features into cached file %s", cached_features_file)
                 torch.save(features, cached_features_file)
 
-    def get_dataloader(self, mode: int, batch_size: int, shuffle: bool = False) -> DataLoader:
+    def get_dataloader(self, mode: int, batch_size: int) -> DataLoader:
         "Load datasets. Called after prepare data."
         cached_features_file = self._feature_file(mode)
         logger.info("Loading features from cached file %s", cached_features_file)

examples/token-classification/utils_ner.py

@@ -23,6 +23,7 @@ from enum import Enum
 from typing import List, Optional, Union
 
 from filelock import FileLock
+
 from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available
 
 
@@ -90,11 +91,11 @@ class TokenClassificationTask:
         sequence_a_segment_id=0,
         mask_padding_with_zero=True,
     ) -> List[InputFeatures]:
-        """Loads a data file into a list of `InputFeatures`
-        `cls_token_at_end` define the location of the CLS token:
-            - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
-            - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
-        `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)
+        """ Loads a data file into a list of `InputFeatures`
+            `cls_token_at_end` define the location of the CLS token:
+                - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
+                - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
+            `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)
         """
         # TODO clean up all this to leverage built-in features of tokenizers
 
@@ -230,8 +231,7 @@ if is_torch_available():
         ):
             # Load data features from cache or dataset file
             cached_features_file = os.path.join(
-                data_dir,
-                "cached_{}_{}_{}".format(mode.value, tokenizer.__class__.__name__, str(max_seq_length)),
+                data_dir, "cached_{}_{}_{}".format(mode.value, tokenizer.__class__.__name__, str(max_seq_length)),
             )
 
             # Make sure only the first process in distributed training processes the dataset,

model_cards/patrickvonplaten/bert2bert-cnn_dailymail-fp16/README.md

@@ -47,8 +47,8 @@ output_ids = model.generate(input_ids)
 
 print(tokenizer.decode(output_ids[0], skip_special_tokens=True))
 # should produce
-# sae was founded in 1856, five years before the civil war. the fraternity has had to work hard to change recently. the university of oklahoma president says the university's affiliation with the fraternity is permanently done. the sae has had a string of members in recent mon
-ths.
+# SAE's national chapter suspended the students from campus activities. The fraternity is under fire for a video showing the students singing a racist chant. SAE has had fewer than 400 members of the
+# fraternity. The group had fewer alcohol consumption, along with about 15, 000 undergraduates populating 219 chapters.
 ```
 
 ## Training script:
@@ -69,7 +69,7 @@ from transformers import BertTokenizer, EncoderDecoderModel, Trainer, TrainingAr
 logging.basicConfig(level=logging.INFO)
 
 model = EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-uncased", "bert-base-uncased")
-tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
 
 # CLS token will work as BOS token
 tokenizer.bos_token = tokenizer.cls_token
@@ -226,4 +226,4 @@ The obtained results should be:
 
 | -   |      Rouge2 - mid -precision      |  Rouge2 - mid - recall | Rouge2 - mid - fmeasure |
 |----------|:-------------:|:------:|:------:|
-| **CNN/Daily Mail** |  16.12 | 17.07 | **16.1** |
+| **CNN/Daily Mail** |  14.12 | 14.37 | **13.8** |

model_cards/patrickvonplaten/roberta2roberta-cnn_dailymail-fp16/README.md

@@ -1,238 +0,0 @@
-# Roberta2Roberta Summarization with 🤗 EncoderDecoder Framework
-
-This model is a Roberta2Roberta model fine-tuned on summarization.
-
-Roberta2Roberta is a `EncoderDecoderModel`, meaning that both the encoder and the decoder are `roberta-base` 
-RoBERTa models. Leveraging the [EncoderDecoderFramework](https://huggingface.co/transformers/model_doc/encoderdecoder.html#encoder-decoder-models), the 
-two pretrained models can simply be loaded into the framework via:
-
-```python
-roberta2roberta = EncoderDecoderModel.from_encoder_decoder_pretrained("roberta-base", "roberta-base")
-```
-
-The decoder of an `EncoderDecoder` model needs cross-attention layers and usually makes use of causal 
-masking for auto-regressiv generation.
-Thus, ``roberta2roberta`` is consequently fined-tuned on the `CNN/Daily Mail`dataset and the resulting model
-`roberta2roberta-cnn_dailymail-fp16` is uploaded here.
-
-## Example
-
-The model is by no means a state-of-the-art model, but nevertheless 
-produces reasonable summarization results. It was mainly fine-tuned 
-as a proof-of-concept for the 🤗 EncoderDecoder Framework.
-
-The model can be used as follows:
-
-```python
-from transformers import BertTokenizer, EncoderDecoderModel
-
-model = EncoderDecoderModel.from_pretrained("patrickvonplaten/roberta2roberta-cnn_dailymail-fp16")
-tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
-
-article = """(CNN)Sigma Alpha Epsilon is under fire for a video showing party-bound fraternity members singing a racist chant. SAE's national chapter suspended the students, but University of Oklahoma President David B
-oren took it a step further, saying the university's affiliation with the fraternity is permanently done. The news is shocking, but it's not the first time SAE has faced controversy. SAE was founded March 9, 185
-6, at the University of Alabama, five years before the American Civil War, according to the fraternity website. When the war began, the group had fewer than 400 members, of which "369 went to war for the Confede
-rate States and seven for the Union Army," the website says. The fraternity now boasts more than 200,000 living alumni, along with about 15,000 undergraduates populating 219 chapters and 20 "colonies" seeking fu
-ll membership at universities. SAE has had to work hard to change recently after a string of member deaths, many blamed on the hazing of new recruits, SAE national President Bradley Cohen wrote in a message on t
-he fraternity's website. The fraternity's website lists more than 130 chapters cited or suspended for "health and safety incidents" since 2010. At least 30 of the incidents involved hazing, and dozens more invol
-ved alcohol. However, the list is missing numerous incidents from recent months. Among them, according to various media outlets: Yale University banned the SAEs from campus activities last month after members al
-legedly tried to interfere with a sexual misconduct investigation connected to an initiation rite. Stanford University in December suspended SAE housing privileges after finding sorority members attending a frat
-ernity function were subjected to graphic sexual content. And Johns Hopkins University in November suspended the fraternity for underage drinking. "The media has labeled us as the 'nation's deadliest fraternity,
-' " Cohen said. In 2011, for example, a student died while being coerced into excessive alcohol consumption, according to a lawsuit. SAE's previous insurer dumped the fraternity. "As a result, we are paying Lloy
-d's of London the highest insurance rates in the Greek-letter world," Cohen said. Universities have turned down SAE's attempts to open new chapters, and the fraternity had to close 12 in 18 months over hazing in
-cidents."""
-
-input_ids = tokenizer(article, return_tensors="pt").input_ids
-output_ids = model.generate(input_ids)
-
-print(tokenizer.decode(output_ids[0], skip_special_tokens=True))
-# should produce
-# Sigma Alpha Epsilon is under fire for a video showing party-bound fraternity members singing racist chants. The fraternity's national chapter has had to close 12 in 18 months over hazing. 
-# Sigma has had more than 130 chapters in 18 states. University of Oklahoma president says fraternity has been "deteriorated".
-```
-
-## Training script:
-
-**IMPORTANT**: In order for this code to work, make sure you checkout to the branch 
-[more_general_trainer_metric](https://github.com/huggingface/transformers/tree/more_general_trainer_metric), which slightly adapts 
-the `Trainer` for `EncoderDecoderModels` according to this PR: https://github.com/huggingface/transformers/pull/5840. 
-
-The following code shows the complete training script that was used to fine-tune `roberta2roberta-cnn_dailymail-fp16
-` for reproducability. The training last ~9h on a standard GPU.
-
-```python
-#!/usr/bin/env python3
-import nlp
-import logging
-from transformers import RobertaTokenizer, EncoderDecoderModel, Trainer, TrainingArguments
-
-logging.basicConfig(level=logging.INFO)
-
-model = EncoderDecoderModel.from_encoder_decoder_pretrained("roberta-base", "roberta-base")
-tokenizer =  RobertaTokenizer.from_pretrained("roberta-base")
-
-# load train and validation data
-train_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="train")
-val_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="validation[:5%]")
-
-# load rouge for validation
-rouge = nlp.load_metric("rouge", experiment_id=0)
-
-# set decoding params
-model.config.decoder_start_token_id = tokenizer.bos_token_id
-model.config.eos_token_id = tokenizer.eos_token_id
-model.config.max_length = 142
-model.config.min_length = 56
-model.config.no_repeat_ngram_size = 3
-model.early_stopping = True
-model.length_penalty = 2.0
-model.num_beams = 4
-
-encoder_length = 512
-decoder_length = 128
-batch_size = 16
-
-
-# map data correctly
-def map_to_encoder_decoder_inputs(batch):
-    # Tokenizer will automatically set [BOS] <text> [EOS]
-    # cut off at Longformer at 2048
-    inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=encoder_length)
-    # force summarization <= 256
-    outputs = tokenizer(batch["highlights"], padding="max_length", truncation=True, max_length=decoder_length)
-
-    batch["input_ids"] = inputs.input_ids
-    batch["attention_mask"] = inputs.attention_mask
-    batch["decoder_input_ids"] = outputs.input_ids
-    batch["labels"] = outputs.input_ids.copy()
-    # mask loss for padding
-    batch["labels"] = [
-        [-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch["labels"]
-    ]
-    batch["decoder_attention_mask"] = outputs.attention_mask
-
-    assert all([len(x) == encoder_length for x in inputs.input_ids])
-    assert all([len(x) == decoder_length for x in outputs.input_ids])
-
-    return batch
-
-
-def compute_metrics(pred):
-    labels_ids = pred.label_ids
-    pred_ids = pred.predictions
-
-    # all unnecessary tokens are removed
-    pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True)
-    labels_ids[labels_ids == -100] = tokenizer.eos_token_id
-    label_str = tokenizer.batch_decode(labels_ids, skip_special_tokens=True)
-
-    rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid
-
-    return {
-        "rouge2_precision": round(rouge_output.precision, 4),
-        "rouge2_recall": round(rouge_output.recall, 4),
-        "rouge2_fmeasure": round(rouge_output.fmeasure, 4),
-    }
-
-
-# make train dataset ready
-train_dataset = train_dataset.map(
-    map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"],
-)
-train_dataset.set_format(
-    type="torch", columns=["input_ids", "attention_mask", "decoder_attention_mask", "decoder_input_ids", "labels"],
-)
-
-# same for validation dataset
-val_dataset = val_dataset.map(
-    map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"],
-)
-val_dataset.set_format(
-    type="torch", columns=["input_ids", "decoder_attention_mask", "attention_mask", "decoder_input_ids", "labels"],
-)
-
-# set training arguments - these params are not really tuned, feel free to change
-training_args = TrainingArguments(
-    output_dir="./",
-    per_device_train_batch_size=batch_size,
-    per_device_eval_batch_size=batch_size,
-    predict_from_generate=True,
-    evaluate_during_training=True,
-    do_train=True,
-    do_eval=True,
-    logging_steps=1000,
-    save_steps=1000,
-    eval_steps=1000,
-    overwrite_output_dir=True,
-    warmup_steps=2000,
-    save_total_limit=3,
-    fp16=True,
-)
-
-# instantiate trainer
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    compute_metrics=compute_metrics,
-    train_dataset=train_dataset,
-    eval_dataset=val_dataset,
-)
-
-# start training
-trainer.train()
-```
-
-## Evaluation
-
-The following script evaluates the model on the test set of 
-CNN/Daily Mail.
-
-```python
-#!/usr/bin/env python3
-import nlp
-from transformers import RobertaTokenizer, EncoderDecoderModel
-
-tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
-model = EncoderDecoderModel.from_pretrained("patrickvonplaten/roberta2roberta-cnn_dailymail-fp16")
-model.to("cuda")
-
-test_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="test")
-batch_size = 128
-
-
-# map data correctly
-def generate_summary(batch):
-    # Tokenizer will automatically set [BOS] <text> [EOS]
-    # cut off at BERT max length 512
-    inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=512, return_tensors="pt")
-    input_ids = inputs.input_ids.to("cuda")
-    attention_mask = inputs.attention_mask.to("cuda")
-
-    outputs = model.generate(input_ids, attention_mask=attention_mask)
-
-    # all special tokens including will be removed
-    output_str = tokenizer.batch_decode(outputs, skip_special_tokens=True)
-
-    batch["pred"] = output_str
-
-    return batch
-
-
-results = test_dataset.map(generate_summary, batched=True, batch_size=batch_size, remove_columns=["article"])
-
-# load rouge for validation
-rouge = nlp.load_metric("rouge")
-
-pred_str = results["pred"]
-label_str = results["highlights"]
-
-rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid
-
-print(rouge_output)
-```
-
-The obtained results should be:
-
-| -   |      Rouge2 - mid -precision      |  Rouge2 - mid - recall | Rouge2 - mid - fmeasure |
-|----------|:-------------:|:------:|:------:|
-| **CNN/Daily Mail** |  15.79 | 19.05 | **16.79** |

model_cards/patrickvonplaten/roberta2roberta-share-cnn_dailymail-fp16/README.md

@@ -1,238 +0,0 @@
-# Shared Roberta2Roberta Summarization with 🤗 EncoderDecoder Framework
-
-This model is a shared Roberta2Roberta model, meaning that the encoder and decoder weights are tied, fine-tuned on summarization.
-
-Roberta2Roberta is a `EncoderDecoderModel`, meaning that both the encoder and the decoder are `roberta-base` 
-RoBERTa models. In this setup the encoder and decoder weights are tied. Leveraging the [EncoderDecoderFramework](https://huggingface.co/transformers/model_doc/encoderdecoder.html#encoder-decoder-models), the 
-two pretrained models can simply be loaded into the framework via:
-
-```python
-roberta2roberta = EncoderDecoderModel.from_encoder_decoder_pretrained("roberta-base", "roberta-base", tie_encoder_decoder=True)
-```
-
-The decoder of an `EncoderDecoder` model needs cross-attention layers and usually makes use of causal 
-masking for auto-regressiv generation.
-Thus, ``roberta2roberta`` is consequently fined-tuned on the `CNN/Daily Mail`dataset and the resulting model
-`roberta2roberta-share-cnn_dailymail-fp16` is uploaded here.
-
-## Example
-
-The model is by no means a state-of-the-art model, but nevertheless 
-produces reasonable summarization results. It was mainly fine-tuned 
-as a proof-of-concept for the 🤗 EncoderDecoder Framework.
-
-The model can be used as follows:
-
-```python
-from transformers import RobertaTokenizer, EncoderDecoderModel
-
-model = EncoderDecoderModel.from_pretrained("patrickvonplaten/roberta2roberta-share-cnn_dailymail-fp16")
-tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
-
-article = """(CNN)Sigma Alpha Epsilon is under fire for a video showing party-bound fraternity members singing a racist chant. SAE's national chapter suspended the students, but University of Oklahoma President David B
-oren took it a step further, saying the university's affiliation with the fraternity is permanently done. The news is shocking, but it's not the first time SAE has faced controversy. SAE was founded March 9, 185
-6, at the University of Alabama, five years before the American Civil War, according to the fraternity website. When the war began, the group had fewer than 400 members, of which "369 went to war for the Confede
-rate States and seven for the Union Army," the website says. The fraternity now boasts more than 200,000 living alumni, along with about 15,000 undergraduates populating 219 chapters and 20 "colonies" seeking fu
-ll membership at universities. SAE has had to work hard to change recently after a string of member deaths, many blamed on the hazing of new recruits, SAE national President Bradley Cohen wrote in a message on t
-he fraternity's website. The fraternity's website lists more than 130 chapters cited or suspended for "health and safety incidents" since 2010. At least 30 of the incidents involved hazing, and dozens more invol
-ved alcohol. However, the list is missing numerous incidents from recent months. Among them, according to various media outlets: Yale University banned the SAEs from campus activities last month after members al
-legedly tried to interfere with a sexual misconduct investigation connected to an initiation rite. Stanford University in December suspended SAE housing privileges after finding sorority members attending a frat
-ernity function were subjected to graphic sexual content. And Johns Hopkins University in November suspended the fraternity for underage drinking. "The media has labeled us as the 'nation's deadliest fraternity,
-' " Cohen said. In 2011, for example, a student died while being coerced into excessive alcohol consumption, according to a lawsuit. SAE's previous insurer dumped the fraternity. "As a result, we are paying Lloy
-d's of London the highest insurance rates in the Greek-letter world," Cohen said. Universities have turned down SAE's attempts to open new chapters, and the fraternity had to close 12 in 18 months over hazing in
-cidents."""
-
-input_ids = tokenizer(article, return_tensors="pt").input_ids
-output_ids = model.generate(input_ids)
-
-print(tokenizer.decode(output_ids[0], skip_special_tokens=True))
-# should produce
-# SAE's national chapter suspended after video shows party-bound fraternity members singing racist chant. University of Oklahoma president says university's affiliation with fraternity is permanently done.
-# SAE has had to close 12 chapters since 2010 after members were killed in hazing. The fraternity has had more than 130 chapters in 18 months.
-```
-
-## Training script:
-
-**IMPORTANT**: In order for this code to work, make sure you checkout to the branch 
-[more_general_trainer_metric](https://github.com/huggingface/transformers/tree/more_general_trainer_metric), which slightly adapts 
-the `Trainer` for `EncoderDecoderModels` according to this PR: https://github.com/huggingface/transformers/pull/5840. 
-
-The following code shows the complete training script that was used to fine-tune `roberta2roberta-cnn_dailymail-fp16
-` for reproducability. The training last ~9h on a standard GPU.
-
-```python
-#!/usr/bin/env python3
-import nlp
-import logging
-from transformers import RobertaTokenizer, EncoderDecoderModel, Trainer, TrainingArguments
-
-logging.basicConfig(level=logging.INFO)
-
-model = EncoderDecoderModel.from_encoder_decoder_pretrained("roberta-base", "roberta-base", tie_encoder_decoder=True)
-tokenizer =  RobertaTokenizer.from_pretrained("roberta-base")
-
-# load train and validation data
-train_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="train")
-val_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="validation[:5%]")
-
-# load rouge for validation
-rouge = nlp.load_metric("rouge", experiment_id=0)
-
-# set decoding params
-model.config.decoder_start_token_id = tokenizer.bos_token_id
-model.config.eos_token_id = tokenizer.eos_token_id
-model.config.max_length = 142
-model.config.min_length = 56
-model.config.no_repeat_ngram_size = 3
-model.early_stopping = True
-model.length_penalty = 2.0
-model.num_beams = 4
-
-encoder_length = 512
-decoder_length = 128
-batch_size = 16
-
-
-# map data correctly
-def map_to_encoder_decoder_inputs(batch):
-    # Tokenizer will automatically set [BOS] <text> [EOS]
-    # cut off at Longformer at 2048
-    inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=encoder_length)
-    # force summarization <= 256
-    outputs = tokenizer(batch["highlights"], padding="max_length", truncation=True, max_length=decoder_length)
-
-    batch["input_ids"] = inputs.input_ids
-    batch["attention_mask"] = inputs.attention_mask
-    batch["decoder_input_ids"] = outputs.input_ids
-    batch["labels"] = outputs.input_ids.copy()
-    # mask loss for padding
-    batch["labels"] = [
-        [-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch["labels"]
-    ]
-    batch["decoder_attention_mask"] = outputs.attention_mask
-
-    assert all([len(x) == encoder_length for x in inputs.input_ids])
-    assert all([len(x) == decoder_length for x in outputs.input_ids])
-
-    return batch
-
-
-def compute_metrics(pred):
-    labels_ids = pred.label_ids
-    pred_ids = pred.predictions
-
-    # all unnecessary tokens are removed
-    pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True)
-    labels_ids[labels_ids == -100] = tokenizer.eos_token_id
-    label_str = tokenizer.batch_decode(labels_ids, skip_special_tokens=True)
-
-    rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid
-
-    return {
-        "rouge2_precision": round(rouge_output.precision, 4),
-        "rouge2_recall": round(rouge_output.recall, 4),
-        "rouge2_fmeasure": round(rouge_output.fmeasure, 4),
-    }
-
-
-# make train dataset ready
-train_dataset = train_dataset.map(
-    map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"],
-)
-train_dataset.set_format(
-    type="torch", columns=["input_ids", "attention_mask", "decoder_attention_mask", "decoder_input_ids", "labels"],
-)
-
-# same for validation dataset
-val_dataset = val_dataset.map(
-    map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"],
-)
-val_dataset.set_format(
-    type="torch", columns=["input_ids", "decoder_attention_mask", "attention_mask", "decoder_input_ids", "labels"],
-)
-
-# set training arguments - these params are not really tuned, feel free to change
-training_args = TrainingArguments(
-    output_dir="./",
-    per_device_train_batch_size=batch_size,
-    per_device_eval_batch_size=batch_size,
-    predict_from_generate=True,
-    evaluate_during_training=True,
-    do_train=True,
-    do_eval=True,
-    logging_steps=1000,
-    save_steps=1000,
-    eval_steps=1000,
-    overwrite_output_dir=True,
-    warmup_steps=2000,
-    save_total_limit=3,
-    fp16=True,
-)
-
-# instantiate trainer
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    compute_metrics=compute_metrics,
-    train_dataset=train_dataset,
-    eval_dataset=val_dataset,
-)
-
-# start training
-trainer.train()
-```
-
-## Evaluation
-
-The following script evaluates the model on the test set of 
-CNN/Daily Mail.
-
-```python
-#!/usr/bin/env python3
-import nlp
-from transformers import RobertaTokenizer, EncoderDecoderModel
-
-tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
-model = EncoderDecoderModel.from_pretrained("patrickvonplaten/roberta2roberta-share-cnn_dailymail-fp16")
-model.to("cuda")
-
-test_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="test")
-batch_size = 128
-
-
-# map data correctly
-def generate_summary(batch):
-    # Tokenizer will automatically set [BOS] <text> [EOS]
-    # cut off at BERT max length 512
-    inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=512, return_tensors="pt")
-    input_ids = inputs.input_ids.to("cuda")
-    attention_mask = inputs.attention_mask.to("cuda")
-
-    outputs = model.generate(input_ids, attention_mask=attention_mask)
-
-    # all special tokens including will be removed
-    output_str = tokenizer.batch_decode(outputs, skip_special_tokens=True)
-
-    batch["pred"] = output_str
-
-    return batch
-
-
-results = test_dataset.map(generate_summary, batched=True, batch_size=batch_size, remove_columns=["article"])
-
-# load rouge for validation
-rouge = nlp.load_metric("rouge")
-
-pred_str = results["pred"]
-label_str = results["highlights"]
-
-rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid
-
-print(rouge_output)
-```
-
-The obtained results should be:
-
-| -   |      Rouge2 - mid -precision      |  Rouge2 - mid - recall | Rouge2 - mid - fmeasure |
-|----------|:-------------:|:------:|:------:|
-| **CNN/Daily Mail** |  15.6 | 18.79 | **16.59** |

model_cards/sagorsarker/codeswitch-hineng-lid-lince/README.md

@@ -1,38 +0,0 @@
----
-language:
-- hi
-- en
----
-
-# codeswitch-hineng-lid-lince
-This is a pretrained model for **language identification** of `hindi-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home)
-
-
-## Identify Language
-
-* Method-1
-
-```py
-
-from transformers import AutoTokenizer, AutoModelForTokenClassification
-
-tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-hineng-lid-lince")
-
-model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-hineng-lid-lince")
-lid_model = pipeline('ner', model=model, tokenizer=tokenizer)
-
-lid_model("put any hindi english code-mixed sentence")
-
-```
-
-* Method-2
-
-```py
-# !pip install codeswitch
-from codeswitch.codeswitch import LanguageIdentification
-lid = LanguageIdentification('hin-eng') 
-text = "" # your code-mixed sentence 
-result = lid.identify(text)
-print(result)
-```
-

model_cards/sagorsarker/codeswitch-hineng-ner-lince/README.md

@@ -1,47 +0,0 @@
----
-language:
-- hi
-- en
----
-
-# codeswitch-hineng-ner-lince
-This is a pretrained model for **Name Entity Recognition** of `Hindi-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home)
-
-This model is trained for this below repository. 
-
-[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch)
-
-To install codeswitch:
-
-```
-pip install codeswitch
-```
-
-## Identify Language
-
-* Method-1
-
-```py
-
-from transformers import AutoTokenizer, AutoModelForTokenClassification
-
-tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-hineng-ner-lince")
-
-model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-hineng-ner-lince")
-
-ner_model = pipeline('ner', model=model, tokenizer=tokenizer)
-
-ner_model("put any hindi english code-mixed sentence")
-
-```
-
-* Method-2
-
-```py
-from codeswitch.codeswitch import NER
-ner = NER('hin-eng')
-text = "" # your mixed sentence 
-result = ner.tag(text)
-print(result)
-
-```

model_cards/sagorsarker/codeswitch-hineng-pos-lince/README.md

@@ -1,45 +0,0 @@
----
-language:
-- hi
-- en
----
-
-# codeswitch-hineng-pos-lince
-This is a pretrained model for **Part of Speech Tagging** of `hindi-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home)
-
-This model is trained for this below repository. 
-
-[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch)
-
-To install codeswitch:
-
-```
-pip install codeswitch
-```
-
-## Identify Language
-
-* Method-1
-
-```py
-
-from transformers import AutoTokenizer, AutoModelForTokenClassification
-
-tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-hineng-pos-lince")
-
-model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-hineng-pos-lince")
-pos_model = pipeline('ner', model=model, tokenizer=tokenizer)
-
-pos_model("put any hindi english code-mixed sentence")
-
-```
-
-* Method-2
-
-```py
-from codeswitch.codeswitch import POS
-pos = POS('hin-eng')
-text = "" # your mixed sentence 
-result = pos.tag(text)
-print(result)
-```

model_cards/sagorsarker/codeswitch-nepeng-lid-lince/README.md

@@ -1,47 +0,0 @@
----
-language:
-- ne
-- en
----
-
-# codeswitch-nepeng-lid-lince
-This is a pretrained model for **language identification** of `nepali-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home).
-
-This model is trained for this below repository. 
-
-[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch)
-
-To install codeswitch:
-
-```
-pip install codeswitch
-```
-
-## Identify Language
-
-* Method-1
-
-```py
-
-from transformers import AutoTokenizer, AutoModelForTokenClassification
-
-tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-nepeng-lid-lince")
-
-model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-nepeng-lid-lince")
-lid_model = pipeline('ner', model=model, tokenizer=tokenizer)
-
-lid_model("put any nepali english code-mixed sentence")
-
-```
-
-* Method-2
-
-```py
-from codeswitch.codeswitch import LanguageIdentification
-lid = LanguageIdentification('nep-eng') 
-text = "" # your code-mixed sentence 
-result = lid.identify(text)
-print(result)
-
-```
-

model_cards/sagorsarker/codeswitch-spaeng-lid-lince/README.md

@@ -1,37 +0,0 @@
----
-language:
-- es
-- en
----
-
-# codeswitch-spaeng-lid-lince
-This is a pretrained model for **language identification** of `spanish-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home)
-
-
-## Identify Language
-
-* Method-1
-
-```py
-
-from transformers import AutoTokenizer, AutoModelForTokenClassification
-
-tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-spaeng-lid-lince")
-
-model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-spaeng-lid-lince")
-lid_model = pipeline('ner', model=model, tokenizer=tokenizer)
-
-lid_model("put any spanish english code-mixed sentence")
-
-```
-
-* Method-2
-
-```py
-# !pip install codeswitch
-from codeswitch.codeswitch import LanguageIdentification
-lid = LanguageIdentification('spa-eng') 
-text = "" # your code-mixed sentence 
-result = lid.identify(text)
-print(result)
-```

model_cards/sagorsarker/codeswitch-spaeng-ner-lince/README.md

@@ -1,46 +0,0 @@
----
-language:
-- es
-- en
----
-
-# codeswitch-spaeng-ner-lince
-This is a pretrained model for **Name Entity Recognition** of `spanish-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home)
-
-This model is trained for this below repository. 
-
-[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch)
-
-To install codeswitch:
-
-```
-pip install codeswitch
-```
-
-## Identify Language
-
-* Method-1
-
-```py
-
-from transformers import AutoTokenizer, AutoModelForTokenClassification
-
-tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-spaeng-ner-lince")
-
-model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-spaeng-ner-lince")
-
-ner_model = pipeline('ner', model=model, tokenizer=tokenizer)
-
-ner_model("put any spanish english code-mixed sentence")
-
-```
-
-* Method-2
-
-```py
-from codeswitch.codeswitch import NER
-ner = NER('spa-eng')
-text = "" # your mixed sentence 
-result = ner.tag(text)
-print(result)
-```

model_cards/sagorsarker/codeswitch-spaeng-pos-lince/README.md

@@ -1,45 +0,0 @@
----
-language:
-- es
-- en
----
-
-# codeswitch-spaeng-pos-lince
-This is a pretrained model for **Part of Speech Tagging** of `spanish-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home)
-
-This model is trained for this below repository. 
-
-[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch)
-
-To install codeswitch:
-
-```
-pip install codeswitch
-```
-
-## Identify Language
-
-* Method-1
-
-```py
-
-from transformers import AutoTokenizer, AutoModelForTokenClassification
-
-tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-spaeng-pos-lince")
-
-model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-spaeng-pos-lince")
-pos_model = pipeline('ner', model=model, tokenizer=tokenizer)
-
-pos_model("put any spanish english code-mixed sentence")
-
-```
-
-* Method-2
-
-```py
-from codeswitch.codeswitch import POS
-pos = POS('spa-eng')
-text = "" # your mixed sentence 
-result = pos.tag(text)
-print(result)
-```

model_cards/zanelim/singbert-large-sg/README.md

@@ -1,215 +0,0 @@
----
-language: en
-tags:
-- singapore
-- sg
-- singlish
-- malaysia
-- ms
-- manglish
-- bert-large-uncased
-license: mit
-datasets:
-- reddit singapore, malaysia
-- hardwarezone
-widget:
-- text: "die [MASK] must try"
-- text: "kopi c siew [MASK]"
----
-
-# Model name
-
-SingBert - Bert for Singlish (SG) and Manglish (MY).
-
-## Model description
-
-Similar to [SingBert](https://huggingface.co/zanelim/singbert) but initialized from [BERT large uncased (whole word masking)](https://github.com/google-research/bert#pre-trained-models), with pre-training finetuned on
-[singlish](https://en.wikipedia.org/wiki/Singlish) and [manglish](https://en.wikipedia.org/wiki/Manglish) data.
-
-## Intended uses & limitations
-
-#### How to use
-
-```python
->>> from transformers import pipeline
->>> nlp = pipeline('fill-mask', model='zanelim/singbert-large-sg')
->>> nlp("kopi c siew [MASK]")
-
-[{'sequence': '[CLS] kopi c siew dai [SEP]',
-  'score': 0.9003700017929077,
-  'token': 18765,
-  'token_str': 'dai'},
- {'sequence': '[CLS] kopi c siew mai [SEP]',
-  'score': 0.0779474675655365,
-  'token': 14736,
-  'token_str': 'mai'},
- {'sequence': '[CLS] kopi c siew. [SEP]',
-  'score': 0.0032227332703769207,
-  'token': 1012,
-  'token_str': '.'},
- {'sequence': '[CLS] kopi c siew bao [SEP]',
-  'score': 0.0017727474914863706,
-  'token': 25945,
-  'token_str': 'bao'},
- {'sequence': '[CLS] kopi c siew peng [SEP]',
-  'score': 0.0012526646023616195,
-  'token': 26473,
-  'token_str': 'peng'}]
-
->>> nlp("one teh c siew dai, and one kopi [MASK]")
-
-[{'sequence': '[CLS] one teh c siew dai, and one kopi. [SEP]',
-  'score': 0.5249741077423096,
-  'token': 1012,
-  'token_str': '.'},
- {'sequence': '[CLS] one teh c siew dai, and one kopi o [SEP]',
-  'score': 0.27349168062210083,
-  'token': 1051,
-  'token_str': 'o'},
- {'sequence': '[CLS] one teh c siew dai, and one kopi peng [SEP]',
-  'score': 0.057190295308828354,
-  'token': 26473,
-  'token_str': 'peng'},
- {'sequence': '[CLS] one teh c siew dai, and one kopi c [SEP]',
-  'score': 0.04022320732474327,
-  'token': 1039,
-  'token_str': 'c'},
- {'sequence': '[CLS] one teh c siew dai, and one kopi? [SEP]',
-  'score': 0.01191170234233141,
-  'token': 1029,
-  'token_str': '?'}]
-
->>> nlp("die [MASK] must try")
-
-[{'sequence': '[CLS] die die must try [SEP]',
-  'score': 0.9921030402183533,
-  'token': 3280,
-  'token_str': 'die'},
- {'sequence': '[CLS] die also must try [SEP]',
-  'score': 0.004993876442313194,
-  'token': 2036,
-  'token_str': 'also'},
- {'sequence': '[CLS] die liao must try [SEP]',
-  'score': 0.000317625846946612,
-  'token': 727,
-  'token_str': 'liao'},
- {'sequence': '[CLS] die still must try [SEP]',
-  'score': 0.0002260878391098231,
-  'token': 2145,
-  'token_str': 'still'},
- {'sequence': '[CLS] die i must try [SEP]',
-  'score': 0.00016935862367972732,
-  'token': 1045,
-  'token_str': 'i'}]
-
->>> nlp("dont play [MASK] leh")
-
-[{'sequence': '[CLS] dont play play leh [SEP]',
-  'score': 0.9079819321632385,
-  'token': 2377,
-  'token_str': 'play'},
- {'sequence': '[CLS] dont play punk leh [SEP]',
-  'score': 0.006846973206847906,
-  'token': 7196,
-  'token_str': 'punk'},
- {'sequence': '[CLS] dont play games leh [SEP]',
-  'score': 0.004041737411171198,
-  'token': 2399,
-  'token_str': 'games'},
- {'sequence': '[CLS] dont play politics leh [SEP]',
-  'score': 0.003728888463228941,
-  'token': 4331,
-  'token_str': 'politics'},
- {'sequence': '[CLS] dont play cheat leh [SEP]',
-  'score': 0.0032805048394948244,
-  'token': 21910,
-  'token_str': 'cheat'}]
-
->>> nlp("confirm plus [MASK]")
-
-{'sequence': '[CLS] confirm plus chop [SEP]',
-  'score': 0.9749826192855835,
-  'token': 24494,
-  'token_str': 'chop'},
- {'sequence': '[CLS] confirm plus chopped [SEP]',
-  'score': 0.017554156482219696,
-  'token': 24881,
-  'token_str': 'chopped'},
- {'sequence': '[CLS] confirm plus minus [SEP]',
-  'score': 0.002725469646975398,
-  'token': 15718,
-  'token_str': 'minus'},
- {'sequence': '[CLS] confirm plus guarantee [SEP]',
-  'score': 0.000900257145985961,
-  'token': 11302,
-  'token_str': 'guarantee'},
- {'sequence': '[CLS] confirm plus one [SEP]',
-  'score': 0.0004384620988275856,
-  'token': 2028,
-  'token_str': 'one'}]
-
->>> nlp("catch no [MASK]")
-
-[{'sequence': '[CLS] catch no ball [SEP]',
-  'score': 0.9381157159805298,
-  'token': 3608,
-  'token_str': 'ball'},
- {'sequence': '[CLS] catch no balls [SEP]',
-  'score': 0.060842301696538925,
-  'token': 7395,
-  'token_str': 'balls'},
- {'sequence': '[CLS] catch no fish [SEP]',
-  'score': 0.00030917322146706283,
-  'token': 3869,
-  'token_str': 'fish'},
- {'sequence': '[CLS] catch no breath [SEP]',
-  'score': 7.552534952992573e-05,
-  'token': 3052,
-  'token_str': 'breath'},
- {'sequence': '[CLS] catch no tail [SEP]',
-  'score': 4.208395694149658e-05,
-  'token': 5725,
-  'token_str': 'tail'}]
-
-```
-
-Here is how to use this model to get the features of a given text in PyTorch:
-```python
-from transformers import BertTokenizer, BertModel
-tokenizer = BertTokenizer.from_pretrained('zanelim/singbert-large-sg')
-model = BertModel.from_pretrained("zanelim/singbert-large-sg")
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='pt')
-output = model(**encoded_input)
-```
-
-and in TensorFlow:
-```python
-from transformers import BertTokenizer, TFBertModel
-tokenizer = BertTokenizer.from_pretrained("zanelim/singbert-large-sg")
-model = TFBertModel.from_pretrained("zanelim/singbert-large-sg")
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='tf')
-output = model(encoded_input)
-```
-
-#### Limitations and bias
-This model was finetuned on colloquial Singlish and Manglish corpus, hence it is best applied on downstream tasks involving the main
-constituent languages- english, mandarin, malay. Also, as the training data is mainly from forums, beware of existing inherent bias.
-
-## Training data
-Colloquial singlish and manglish (both are a mixture of English, Mandarin, Tamil, Malay, and other local dialects like Hokkien, Cantonese or Teochew)
-corpus. The corpus is collected from subreddits- `r/singapore` and `r/malaysia`, and forums such as `hardwarezone`.
-
-## Training procedure
-
-Initialized with [bert large uncased (whole word masking)](https://github.com/google-research/bert#pre-trained-models) vocab and checkpoints (pre-trained weights).
-Top 1000 custom vocab tokens (non-overlapped with original bert vocab) were further extracted from training data and filled into unused tokens in original bert vocab.
-
-Pre-training was further finetuned on training data with the following hyperparameters
-* train_batch_size: 512
-* max_seq_length: 128
-* num_train_steps: 300000
-* num_warmup_steps: 5000
-* learning_rate: 2e-5
-* hardware: TPU v3-8

model_cards/zanelim/singbert/README.md

@@ -1,215 +0,0 @@
----
-language: en
-tags:
-- singapore
-- sg
-- singlish
-- malaysia
-- ms
-- manglish
-- bert-base-uncased
-license: mit
-datasets:
-- reddit singapore, malaysia
-- hardwarezone
-widget:
-- text: "die [MASK] must try"
-- text: "kopi c siew [MASK]"
----
-
-# Model name
-
-SingBert - Bert for Singlish (SG) and Manglish (MY).
-
-## Model description
-
-[BERT base uncased](https://github.com/google-research/bert#pre-trained-models), with pre-training finetuned on
-[singlish](https://en.wikipedia.org/wiki/Singlish) and [manglish](https://en.wikipedia.org/wiki/Manglish) data.
-
-## Intended uses & limitations
-
-#### How to use
-
-```python
->>> from transformers import pipeline
->>> nlp = pipeline('fill-mask', model='zanelim/singbert')
->>> nlp("kopi c siew [MASK]")
-
-[{'sequence': '[CLS] kopi c siew dai [SEP]',
-  'score': 0.5092713236808777,
-  'token': 18765,
-  'token_str': 'dai'},
- {'sequence': '[CLS] kopi c siew mai [SEP]',
-  'score': 0.3515934646129608,
-  'token': 14736,
-  'token_str': 'mai'},
- {'sequence': '[CLS] kopi c siew bao [SEP]',
-  'score': 0.05576375499367714,
-  'token': 25945,
-  'token_str': 'bao'},
- {'sequence': '[CLS] kopi c siew. [SEP]',
-  'score': 0.006019321270287037,
-  'token': 1012,
-  'token_str': '.'},
- {'sequence': '[CLS] kopi c siew sai [SEP]',
-  'score': 0.0038361591286957264,
-  'token': 18952,
-  'token_str': 'sai'}]
-
->>> nlp("one teh c siew dai, and one kopi [MASK].")
-
-[{'sequence': '[CLS] one teh c siew dai, and one kopi c [SEP]',
-  'score': 0.6176503300666809,
-  'token': 1039,
-  'token_str': 'c'},
- {'sequence': '[CLS] one teh c siew dai, and one kopi o [SEP]',
-  'score': 0.21094971895217896,
-  'token': 1051,
-  'token_str': 'o'},
- {'sequence': '[CLS] one teh c siew dai, and one kopi. [SEP]',
-  'score': 0.13027705252170563,
-  'token': 1012,
-  'token_str': '.'},
- {'sequence': '[CLS] one teh c siew dai, and one kopi! [SEP]',
-  'score': 0.004680239595472813,
-  'token': 999,
-  'token_str': '!'},
- {'sequence': '[CLS] one teh c siew dai, and one kopi w [SEP]',
-  'score': 0.002034128177911043,
-  'token': 1059,
-  'token_str': 'w'}]
-
->>> nlp("dont play [MASK] leh")
-
-[{'sequence': '[CLS] dont play play leh [SEP]',
-  'score': 0.9281464219093323,
-  'token': 2377,
-  'token_str': 'play'},
- {'sequence': '[CLS] dont play politics leh [SEP]',
-  'score': 0.010990909300744534,
-  'token': 4331,
-  'token_str': 'politics'},
- {'sequence': '[CLS] dont play punk leh [SEP]',
-  'score': 0.005583590362221003,
-  'token': 7196,
-  'token_str': 'punk'},
- {'sequence': '[CLS] dont play dirty leh [SEP]',
-  'score': 0.0025784350000321865,
-  'token': 6530,
-  'token_str': 'dirty'},
- {'sequence': '[CLS] dont play cheat leh [SEP]',
-  'score': 0.0025066907983273268,
-  'token': 21910,
-  'token_str': 'cheat'}]
-
->>> nlp("catch no [MASK]")
-
-[{'sequence': '[CLS] catch no ball [SEP]',
-  'score': 0.7922210693359375,
-  'token': 3608,
-  'token_str': 'ball'},
- {'sequence': '[CLS] catch no balls [SEP]',
-  'score': 0.20503675937652588,
-  'token': 7395,
-  'token_str': 'balls'},
- {'sequence': '[CLS] catch no tail [SEP]',
-  'score': 0.0006608376861549914,
-  'token': 5725,
-  'token_str': 'tail'},
- {'sequence': '[CLS] catch no talent [SEP]',
-  'score': 0.0002158183924620971,
-  'token': 5848,
-  'token_str': 'talent'},
- {'sequence': '[CLS] catch no prisoners [SEP]',
-  'score': 5.3481446229852736e-05,
-  'token': 5895,
-  'token_str': 'prisoners'}]
-
->>> nlp("confirm plus [MASK]")
-
-[{'sequence': '[CLS] confirm plus chop [SEP]',
-  'score': 0.992355227470398,
-  'token': 24494,
-  'token_str': 'chop'},
- {'sequence': '[CLS] confirm plus one [SEP]',
-  'score': 0.0037301010452210903,
-  'token': 2028,
-  'token_str': 'one'},
- {'sequence': '[CLS] confirm plus minus [SEP]',
-  'score': 0.0014284878270700574,
-  'token': 15718,
-  'token_str': 'minus'},
- {'sequence': '[CLS] confirm plus 1 [SEP]',
-  'score': 0.0011354683665558696,
-  'token': 1015,
-  'token_str': '1'},
- {'sequence': '[CLS] confirm plus chopped [SEP]',
-  'score': 0.0003804611915256828,
-  'token': 24881,
-  'token_str': 'chopped'}]
-
->>> nlp("die [MASK] must try")
-
-[{'sequence': '[CLS] die die must try [SEP]',
-  'score': 0.9552758932113647,
-  'token': 3280,
-  'token_str': 'die'},
- {'sequence': '[CLS] die also must try [SEP]',
-  'score': 0.03644804656505585,
-  'token': 2036,
-  'token_str': 'also'},
- {'sequence': '[CLS] die liao must try [SEP]',
-  'score': 0.003282855963334441,
-  'token': 727,
-  'token_str': 'liao'},
- {'sequence': '[CLS] die already must try [SEP]',
-  'score': 0.0004937972989864647,
-  'token': 2525,
-  'token_str': 'already'},
- {'sequence': '[CLS] die hard must try [SEP]',
-  'score': 0.0003659659414552152,
-  'token': 2524,
-  'token_str': 'hard'}]
-
-```
-
-Here is how to use this model to get the features of a given text in PyTorch:
-```python
-from transformers import BertTokenizer, BertModel
-tokenizer = BertTokenizer.from_pretrained('zanelim/singbert')
-model = BertModel.from_pretrained("zanelim/singbert")
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='pt')
-output = model(**encoded_input)
-```
-
-and in TensorFlow:
-```python
-from transformers import BertTokenizer, TFBertModel
-tokenizer = BertTokenizer.from_pretrained("zanelim/singbert")
-model = TFBertModel.from_pretrained("zanelim/singbert")
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='tf')
-output = model(encoded_input)
-```
-
-#### Limitations and bias
-This model was finetuned on colloquial Singlish and Manglish corpus, hence it is best applied on downstream tasks involving the main
-constituent languages- english, mandarin, malay. Also, as the training data is mainly from forums, beware of existing inherent bias.
-
-## Training data
-Colloquial singlish and manglish (both are a mixture of English, Mandarin, Tamil, Malay, and other local dialects like Hokkien, Cantonese or Teochew)
-corpus. The corpus is collected from subreddits- `r/singapore` and `r/malaysia`, and forums such as `hardwarezone`.
-
-## Training procedure
-
-Initialized with [bert base uncased](https://github.com/google-research/bert#pre-trained-models) vocab and checkpoints (pre-trained weights).
-Top 1000 custom vocab tokens (non-overlapped with original bert vocab) were further extracted from training data and filled into unused tokens in original bert vocab.
-
-Pre-training was further finetuned on training data with the following hyperparameters
-* train_batch_size: 512
-* max_seq_length: 128
-* num_train_steps: 300000
-* num_warmup_steps: 5000
-* learning_rate: 2e-5
-* hardware: TPU v3-8

setup.cfg

@@ -1,5 +1,4 @@
 [isort]
-default_section = FIRSTPARTY
 ensure_newline_before_comments = True
 force_grid_wrap = 0
 include_trailing_comma = True

setup.py

@@ -74,17 +74,16 @@ extras["tf"] = [
     # "onnxconverter-common",
     # "keras2onnx"
     "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
-    "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx",
+    "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx"
 ]
 extras["tf-cpu"] = [
     "tensorflow-cpu",
     # "onnxconverter-common",
     # "keras2onnx"
     "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
-    "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx",
+    "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx"
 ]
 extras["torch"] = ["torch"]
-extras["onnxruntime"] = ["onnxruntime>=1.4.0", "onnxruntime-tools>=1.4.2"]
 
 extras["serving"] = ["pydantic", "uvicorn", "fastapi", "starlette"]
 extras["all"] = extras["serving"] + ["tensorflow", "torch"]
@@ -92,7 +91,12 @@ extras["all"] = extras["serving"] + ["tensorflow", "torch"]
 extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "psutil"]
 # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
 extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
-extras["quality"] = ["black", "isort >= 5", "flake8"]
+extras["quality"] = [
+    "black",
+    # "isort",
+    "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
+    "flake8",
+]
 extras["dev"] = extras["testing"] + extras["quality"] + extras["ja"] + ["scikit-learn", "tensorflow", "torch"]
 
 setup(

src/transformers/__init__.py

@@ -17,6 +17,8 @@ else:
     absl.logging.set_stderrthreshold("info")
     absl.logging._warn_preinit_stderr = False
 
+import logging
+
 # Configurations
 from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
 from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, AutoConfig
@@ -90,13 +92,7 @@ from .file_utils import (
 from .hf_argparser import HfArgumentParser
 
 # Integrations
-from .integrations import (
-    is_comet_available,
-    is_optuna_available,
-    is_ray_available,
-    is_tensorboard_available,
-    is_wandb_available,
-)
+from .integrations import is_comet_available, is_tensorboard_available, is_wandb_available
 
 # Model Cards
 from .modelcard import ModelCard
@@ -182,10 +178,9 @@ from .tokenization_xlnet import SPIECE_UNDERLINE, XLNetTokenizer
 from .trainer_utils import EvalPrediction, set_seed
 from .training_args import TrainingArguments
 from .training_args_tf import TFTrainingArguments
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
 if is_sklearn_available():
@@ -194,246 +189,240 @@ if is_sklearn_available():
 
 # Modeling
 if is_torch_available():
-    # Benchmarks
-    from .benchmark.benchmark import PyTorchBenchmark
-    from .benchmark.benchmark_args import PyTorchBenchmarkArguments
-    from .data.data_collator import (
-        DataCollator,
-        DataCollatorForLanguageModeling,
-        DataCollatorForPermutationLanguageModeling,
-        DataCollatorWithPadding,
-        default_data_collator,
-    )
-    from .data.datasets import (
-        GlueDataset,
-        GlueDataTrainingArguments,
-        LineByLineTextDataset,
-        SquadDataset,
-        SquadDataTrainingArguments,
-        TextDataset,
-    )
     from .generation_utils import top_k_top_p_filtering
-    from .modeling_albert import (
-        ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        AlbertForMaskedLM,
-        AlbertForMultipleChoice,
-        AlbertForPreTraining,
-        AlbertForQuestionAnswering,
-        AlbertForSequenceClassification,
-        AlbertForTokenClassification,
-        AlbertModel,
-        AlbertPreTrainedModel,
-        load_tf_weights_in_albert,
-    )
+    from .modeling_utils import PreTrainedModel, prune_layer, Conv1D, apply_chunking_to_forward
     from .modeling_auto import (
-        MODEL_FOR_CAUSAL_LM_MAPPING,
-        MODEL_FOR_MASKED_LM_MAPPING,
-        MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
-        MODEL_FOR_PRETRAINING_MAPPING,
-        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
-        MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
-        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
-        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
-        MODEL_MAPPING,
-        MODEL_WITH_LM_HEAD_MAPPING,
         AutoModel,
+        AutoModelForPreTraining,
+        AutoModelForSequenceClassification,
+        AutoModelForQuestionAnswering,
+        AutoModelWithLMHead,
         AutoModelForCausalLM,
         AutoModelForMaskedLM,
-        AutoModelForMultipleChoice,
-        AutoModelForPreTraining,
-        AutoModelForQuestionAnswering,
         AutoModelForSeq2SeqLM,
-        AutoModelForSequenceClassification,
         AutoModelForTokenClassification,
-        AutoModelWithLMHead,
+        AutoModelForMultipleChoice,
+        MODEL_MAPPING,
+        MODEL_FOR_PRETRAINING_MAPPING,
+        MODEL_WITH_LM_HEAD_MAPPING,
+        MODEL_FOR_CAUSAL_LM_MAPPING,
+        MODEL_FOR_MASKED_LM_MAPPING,
+        MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
+        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
+        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
+        MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
     )
+
+    from .modeling_mobilebert import (
+        MobileBertPreTrainedModel,
+        MobileBertModel,
+        MobileBertForPreTraining,
+        MobileBertForSequenceClassification,
+        MobileBertForQuestionAnswering,
+        MobileBertForMaskedLM,
+        MobileBertForNextSentencePrediction,
+        MobileBertForMultipleChoice,
+        MobileBertForTokenClassification,
+        load_tf_weights_in_mobilebert,
+        MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        MobileBertLayer,
+    )
+
+    from .modeling_bert import (
+        BertPreTrainedModel,
+        BertModel,
+        BertForPreTraining,
+        BertForMaskedLM,
+        BertLMHeadModel,
+        BertForNextSentencePrediction,
+        BertForSequenceClassification,
+        BertForMultipleChoice,
+        BertForTokenClassification,
+        BertForQuestionAnswering,
+        load_tf_weights_in_bert,
+        BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        BertLayer,
+    )
+    from .modeling_openai import (
+        OpenAIGPTPreTrainedModel,
+        OpenAIGPTModel,
+        OpenAIGPTLMHeadModel,
+        OpenAIGPTDoubleHeadsModel,
+        load_tf_weights_in_openai_gpt,
+        OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+    from .modeling_transfo_xl import (
+        TransfoXLPreTrainedModel,
+        TransfoXLModel,
+        TransfoXLLMHeadModel,
+        AdaptiveEmbedding,
+        load_tf_weights_in_transfo_xl,
+        TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+    from .modeling_gpt2 import (
+        GPT2PreTrainedModel,
+        GPT2Model,
+        GPT2LMHeadModel,
+        GPT2DoubleHeadsModel,
+        load_tf_weights_in_gpt2,
+        GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+    from .modeling_ctrl import CTRLPreTrainedModel, CTRLModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_LIST
+    from .modeling_xlnet import (
+        XLNetPreTrainedModel,
+        XLNetModel,
+        XLNetLMHeadModel,
+        XLNetForSequenceClassification,
+        XLNetForTokenClassification,
+        XLNetForMultipleChoice,
+        XLNetForQuestionAnsweringSimple,
+        XLNetForQuestionAnswering,
+        load_tf_weights_in_xlnet,
+        XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+    from .modeling_xlm import (
+        XLMPreTrainedModel,
+        XLMModel,
+        XLMWithLMHeadModel,
+        XLMForSequenceClassification,
+        XLMForTokenClassification,
+        XLMForQuestionAnswering,
+        XLMForQuestionAnsweringSimple,
+        XLMForMultipleChoice,
+        XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+    from .modeling_pegasus import PegasusForConditionalGeneration
     from .modeling_bart import (
-        BART_PRETRAINED_MODEL_ARCHIVE_LIST,
-        BartForConditionalGeneration,
-        BartForQuestionAnswering,
+        PretrainedBartModel,
         BartForSequenceClassification,
         BartModel,
-        PretrainedBartModel,
+        BartForConditionalGeneration,
+        BartForQuestionAnswering,
     )
-    from .modeling_bert import (
-        BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        BertForMaskedLM,
-        BertForMultipleChoice,
-        BertForNextSentencePrediction,
-        BertForPreTraining,
-        BertForQuestionAnswering,
-        BertForSequenceClassification,
-        BertForTokenClassification,
-        BertLayer,
-        BertLMHeadModel,
-        BertModel,
-        BertPreTrainedModel,
-        load_tf_weights_in_bert,
+    from .modeling_mbart import MBartForConditionalGeneration
+    from .modeling_marian import MarianMTModel
+    from .tokenization_marian import MarianTokenizer
+    from .modeling_roberta import (
+        RobertaForMaskedLM,
+        RobertaForCausalLM,
+        RobertaModel,
+        RobertaForSequenceClassification,
+        RobertaForMultipleChoice,
+        RobertaForTokenClassification,
+        RobertaForQuestionAnswering,
+        ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+    from .modeling_distilbert import (
+        DistilBertPreTrainedModel,
+        DistilBertForMaskedLM,
+        DistilBertModel,
+        DistilBertForMultipleChoice,
+        DistilBertForSequenceClassification,
+        DistilBertForQuestionAnswering,
+        DistilBertForTokenClassification,
+        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
     )
     from .modeling_camembert import (
-        CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        CamembertForCausalLM,
         CamembertForMaskedLM,
-        CamembertForMultipleChoice,
-        CamembertForQuestionAnswering,
-        CamembertForSequenceClassification,
-        CamembertForTokenClassification,
         CamembertModel,
+        CamembertForSequenceClassification,
+        CamembertForMultipleChoice,
+        CamembertForTokenClassification,
+        CamembertForQuestionAnswering,
+        CamembertForCausalLM,
+        CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
     )
-    from .modeling_ctrl import CTRL_PRETRAINED_MODEL_ARCHIVE_LIST, CTRLLMHeadModel, CTRLModel, CTRLPreTrainedModel
-    from .modeling_distilbert import (
-        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        DistilBertForMaskedLM,
-        DistilBertForMultipleChoice,
-        DistilBertForQuestionAnswering,
-        DistilBertForSequenceClassification,
-        DistilBertForTokenClassification,
-        DistilBertModel,
-        DistilBertPreTrainedModel,
+    from .modeling_encoder_decoder import EncoderDecoderModel
+    from .modeling_t5 import (
+        T5PreTrainedModel,
+        T5Model,
+        T5ForConditionalGeneration,
+        load_tf_weights_in_t5,
+        T5_PRETRAINED_MODEL_ARCHIVE_LIST,
     )
+    from .modeling_albert import (
+        AlbertPreTrainedModel,
+        AlbertModel,
+        AlbertForPreTraining,
+        AlbertForMaskedLM,
+        AlbertForMultipleChoice,
+        AlbertForSequenceClassification,
+        AlbertForQuestionAnswering,
+        AlbertForTokenClassification,
+        load_tf_weights_in_albert,
+        ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+    from .modeling_xlm_roberta import (
+        XLMRobertaForMaskedLM,
+        XLMRobertaModel,
+        XLMRobertaForMultipleChoice,
+        XLMRobertaForSequenceClassification,
+        XLMRobertaForTokenClassification,
+        XLMRobertaForQuestionAnswering,
+        XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+    from .modeling_mmbt import ModalEmbeddings, MMBTModel, MMBTForClassification
+
+    from .modeling_flaubert import (
+        FlaubertModel,
+        FlaubertWithLMHeadModel,
+        FlaubertForSequenceClassification,
+        FlaubertForTokenClassification,
+        FlaubertForQuestionAnswering,
+        FlaubertForQuestionAnsweringSimple,
+        FlaubertForTokenClassification,
+        FlaubertForMultipleChoice,
+        FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+
+    from .modeling_electra import (
+        ElectraForPreTraining,
+        ElectraForMaskedLM,
+        ElectraForTokenClassification,
+        ElectraPreTrainedModel,
+        ElectraForMultipleChoice,
+        ElectraForSequenceClassification,
+        ElectraForQuestionAnswering,
+        ElectraModel,
+        load_tf_weights_in_electra,
+        ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+
+    from .modeling_reformer import (
+        ReformerAttention,
+        ReformerLayer,
+        ReformerModel,
+        ReformerForMaskedLM,
+        ReformerModelWithLMHead,
+        ReformerForSequenceClassification,
+        ReformerForQuestionAnswering,
+        REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+
+    from .modeling_longformer import (
+        LongformerModel,
+        LongformerForMaskedLM,
+        LongformerForSequenceClassification,
+        LongformerForMultipleChoice,
+        LongformerForTokenClassification,
+        LongformerForQuestionAnswering,
+        LongformerSelfAttention,
+        LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
+    )
+
     from .modeling_dpr import (
-        DPRContextEncoder,
         DPRPretrainedContextEncoder,
         DPRPretrainedQuestionEncoder,
         DPRPretrainedReader,
+        DPRContextEncoder,
         DPRQuestionEncoder,
         DPRReader,
     )
-    from .modeling_electra import (
-        ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
-        ElectraForMaskedLM,
-        ElectraForMultipleChoice,
-        ElectraForPreTraining,
-        ElectraForQuestionAnswering,
-        ElectraForSequenceClassification,
-        ElectraForTokenClassification,
-        ElectraModel,
-        ElectraPreTrainedModel,
-        load_tf_weights_in_electra,
-    )
-    from .modeling_encoder_decoder import EncoderDecoderModel
-    from .modeling_flaubert import (
-        FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        FlaubertForMultipleChoice,
-        FlaubertForQuestionAnswering,
-        FlaubertForQuestionAnsweringSimple,
-        FlaubertForSequenceClassification,
-        FlaubertForTokenClassification,
-        FlaubertModel,
-        FlaubertWithLMHeadModel,
-    )
-    from .modeling_gpt2 import (
-        GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
-        GPT2DoubleHeadsModel,
-        GPT2LMHeadModel,
-        GPT2Model,
-        GPT2PreTrainedModel,
-        load_tf_weights_in_gpt2,
-    )
-    from .modeling_longformer import (
-        LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
-        LongformerForMaskedLM,
-        LongformerForMultipleChoice,
-        LongformerForQuestionAnswering,
-        LongformerForSequenceClassification,
-        LongformerForTokenClassification,
-        LongformerModel,
-        LongformerSelfAttention,
-    )
-    from .modeling_marian import MarianMTModel
-    from .modeling_mbart import MBartForConditionalGeneration
-    from .modeling_mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings
-    from .modeling_mobilebert import (
-        MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        MobileBertForMaskedLM,
-        MobileBertForMultipleChoice,
-        MobileBertForNextSentencePrediction,
-        MobileBertForPreTraining,
-        MobileBertForQuestionAnswering,
-        MobileBertForSequenceClassification,
-        MobileBertForTokenClassification,
-        MobileBertLayer,
-        MobileBertModel,
-        MobileBertPreTrainedModel,
-        load_tf_weights_in_mobilebert,
-    )
-    from .modeling_openai import (
-        OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        OpenAIGPTDoubleHeadsModel,
-        OpenAIGPTLMHeadModel,
-        OpenAIGPTModel,
-        OpenAIGPTPreTrainedModel,
-        load_tf_weights_in_openai_gpt,
-    )
-    from .modeling_pegasus import PegasusForConditionalGeneration
-    from .modeling_reformer import (
-        REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
-        ReformerAttention,
-        ReformerForMaskedLM,
-        ReformerForQuestionAnswering,
-        ReformerForSequenceClassification,
-        ReformerLayer,
-        ReformerModel,
-        ReformerModelWithLMHead,
-    )
-    from .modeling_retribert import RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST, RetriBertModel, RetriBertPreTrainedModel
-    from .modeling_roberta import (
-        ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
-        RobertaForCausalLM,
-        RobertaForMaskedLM,
-        RobertaForMultipleChoice,
-        RobertaForQuestionAnswering,
-        RobertaForSequenceClassification,
-        RobertaForTokenClassification,
-        RobertaModel,
-    )
-    from .modeling_t5 import (
-        T5_PRETRAINED_MODEL_ARCHIVE_LIST,
-        T5ForConditionalGeneration,
-        T5Model,
-        T5PreTrainedModel,
-        load_tf_weights_in_t5,
-    )
-    from .modeling_transfo_xl import (
-        TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
-        AdaptiveEmbedding,
-        TransfoXLLMHeadModel,
-        TransfoXLModel,
-        TransfoXLPreTrainedModel,
-        load_tf_weights_in_transfo_xl,
-    )
-    from .modeling_utils import Conv1D, PreTrainedModel, apply_chunking_to_forward, prune_layer
-    from .modeling_xlm import (
-        XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
-        XLMForMultipleChoice,
-        XLMForQuestionAnswering,
-        XLMForQuestionAnsweringSimple,
-        XLMForSequenceClassification,
-        XLMForTokenClassification,
-        XLMModel,
-        XLMPreTrainedModel,
-        XLMWithLMHeadModel,
-    )
-    from .modeling_xlm_roberta import (
-        XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
-        XLMRobertaForMaskedLM,
-        XLMRobertaForMultipleChoice,
-        XLMRobertaForQuestionAnswering,
-        XLMRobertaForSequenceClassification,
-        XLMRobertaForTokenClassification,
-        XLMRobertaModel,
-    )
-    from .modeling_xlnet import (
-        XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
-        XLNetForMultipleChoice,
-        XLNetForQuestionAnswering,
-        XLNetForQuestionAnsweringSimple,
-        XLNetForSequenceClassification,
-        XLNetForTokenClassification,
-        XLNetLMHeadModel,
-        XLNetModel,
-        XLNetPreTrainedModel,
-        load_tf_weights_in_xlnet,
+    from .modeling_retribert import (
+        RetriBertPreTrainedModel,
+        RetriBertModel,
+        RETRIBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
     )
 
     # Optimization
@@ -446,18 +435,61 @@ if is_torch_available():
         get_linear_schedule_with_warmup,
         get_polynomial_decay_schedule_with_warmup,
     )
-    from .tokenization_marian import MarianTokenizer
 
     # Trainer
-    from .trainer import EvalPrediction, Trainer, set_seed, torch_distributed_zero_first
+    from .trainer import Trainer, set_seed, torch_distributed_zero_first, EvalPrediction
+    from .data.data_collator import (
+        default_data_collator,
+        DataCollator,
+        DataCollatorForLanguageModeling,
+        DataCollatorForPermutationLanguageModeling,
+        DataCollatorWithPadding,
+    )
+    from .data.datasets import (
+        GlueDataset,
+        TextDataset,
+        LineByLineTextDataset,
+        GlueDataTrainingArguments,
+        SquadDataset,
+        SquadDataTrainingArguments,
+    )
+
+    # Benchmarks
+    from .benchmark.benchmark import PyTorchBenchmark
+    from .benchmark.benchmark_args import PyTorchBenchmarkArguments
 
 # TensorFlow
 if is_tf_available():
-    from .benchmark.benchmark_args_tf import TensorFlowBenchmarkArguments
-
-    # Benchmarks
-    from .benchmark.benchmark_tf import TensorFlowBenchmark
     from .generation_tf_utils import tf_top_k_top_p_filtering
+    from .modeling_tf_utils import (
+        shape_list,
+        TFPreTrainedModel,
+        TFSequenceSummary,
+        TFSharedEmbeddings,
+    )
+    from .modeling_tf_auto import (
+        TF_MODEL_MAPPING,
+        TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
+        TF_MODEL_FOR_PRETRAINING_MAPPING,
+        TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+        TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
+        TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
+        TF_MODEL_WITH_LM_HEAD_MAPPING,
+        TF_MODEL_FOR_CAUSAL_LM_MAPPING,
+        TF_MODEL_FOR_MASKED_LM_MAPPING,
+        TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
+        TFAutoModel,
+        TFAutoModelForMultipleChoice,
+        TFAutoModelForPreTraining,
+        TFAutoModelForQuestionAnswering,
+        TFAutoModelForSequenceClassification,
+        TFAutoModelForTokenClassification,
+        TFAutoModelWithLMHead,
+        TFAutoModelForCausalLM,
+        TFAutoModelForMaskedLM,
+        TFAutoModelForSeq2SeqLM,
+    )
+
     from .modeling_tf_albert import (
         TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFAlbertForMaskedLM,
@@ -470,31 +502,11 @@ if is_tf_available():
         TFAlbertModel,
         TFAlbertPreTrainedModel,
     )
-    from .modeling_tf_auto import (
-        TF_MODEL_FOR_CAUSAL_LM_MAPPING,
-        TF_MODEL_FOR_MASKED_LM_MAPPING,
-        TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
-        TF_MODEL_FOR_PRETRAINING_MAPPING,
-        TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
-        TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
-        TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
-        TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
-        TF_MODEL_MAPPING,
-        TF_MODEL_WITH_LM_HEAD_MAPPING,
-        TFAutoModel,
-        TFAutoModelForCausalLM,
-        TFAutoModelForMaskedLM,
-        TFAutoModelForMultipleChoice,
-        TFAutoModelForPreTraining,
-        TFAutoModelForQuestionAnswering,
-        TFAutoModelForSeq2SeqLM,
-        TFAutoModelForSequenceClassification,
-        TFAutoModelForTokenClassification,
-        TFAutoModelWithLMHead,
-    )
+
     from .modeling_tf_bert import (
         TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFBertEmbeddings,
+        TFBertLMHeadModel,
         TFBertForMaskedLM,
         TFBertForMultipleChoice,
         TFBertForNextSentencePrediction,
@@ -502,26 +514,28 @@ if is_tf_available():
         TFBertForQuestionAnswering,
         TFBertForSequenceClassification,
         TFBertForTokenClassification,
-        TFBertLMHeadModel,
         TFBertMainLayer,
         TFBertModel,
         TFBertPreTrainedModel,
     )
+
     from .modeling_tf_camembert import (
         TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFCamembertForMaskedLM,
+        TFCamembertModel,
         TFCamembertForMultipleChoice,
         TFCamembertForQuestionAnswering,
         TFCamembertForSequenceClassification,
         TFCamembertForTokenClassification,
-        TFCamembertModel,
     )
+
     from .modeling_tf_ctrl import (
         TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFCTRLLMHeadModel,
         TFCTRLModel,
         TFCTRLPreTrainedModel,
     )
+
     from .modeling_tf_distilbert import (
         TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFDistilBertForMaskedLM,
@@ -533,6 +547,7 @@ if is_tf_available():
         TFDistilBertModel,
         TFDistilBertPreTrainedModel,
     )
+
     from .modeling_tf_electra import (
         TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFElectraForMaskedLM,
@@ -544,15 +559,17 @@ if is_tf_available():
         TFElectraModel,
         TFElectraPreTrainedModel,
     )
+
     from .modeling_tf_flaubert import (
         TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFFlaubertForMultipleChoice,
         TFFlaubertForQuestionAnsweringSimple,
         TFFlaubertForSequenceClassification,
         TFFlaubertForTokenClassification,
-        TFFlaubertModel,
         TFFlaubertWithLMHeadModel,
+        TFFlaubertModel,
     )
+
     from .modeling_tf_gpt2 import (
         TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFGPT2DoubleHeadsModel,
@@ -561,26 +578,29 @@ if is_tf_available():
         TFGPT2Model,
         TFGPT2PreTrainedModel,
     )
+
     from .modeling_tf_longformer import (
         TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
+        TFLongformerModel,
         TFLongformerForMaskedLM,
         TFLongformerForQuestionAnswering,
-        TFLongformerModel,
         TFLongformerSelfAttention,
     )
+
     from .modeling_tf_mobilebert import (
         TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        TFMobileBertForMaskedLM,
-        TFMobileBertForMultipleChoice,
-        TFMobileBertForNextSentencePrediction,
-        TFMobileBertForPreTraining,
-        TFMobileBertForQuestionAnswering,
-        TFMobileBertForSequenceClassification,
-        TFMobileBertForTokenClassification,
-        TFMobileBertMainLayer,
         TFMobileBertModel,
         TFMobileBertPreTrainedModel,
+        TFMobileBertForPreTraining,
+        TFMobileBertForSequenceClassification,
+        TFMobileBertForQuestionAnswering,
+        TFMobileBertForMaskedLM,
+        TFMobileBertForNextSentencePrediction,
+        TFMobileBertForMultipleChoice,
+        TFMobileBertForTokenClassification,
+        TFMobileBertMainLayer,
     )
+
     from .modeling_tf_openai import (
         TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFOpenAIGPTDoubleHeadsModel,
@@ -589,6 +609,7 @@ if is_tf_available():
         TFOpenAIGPTModel,
         TFOpenAIGPTPreTrainedModel,
     )
+
     from .modeling_tf_roberta import (
         TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFRobertaForMaskedLM,
@@ -600,12 +621,14 @@ if is_tf_available():
         TFRobertaModel,
         TFRobertaPreTrainedModel,
     )
+
     from .modeling_tf_t5 import (
         TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFT5ForConditionalGeneration,
         TFT5Model,
         TFT5PreTrainedModel,
     )
+
     from .modeling_tf_transfo_xl import (
         TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFAdaptiveEmbedding,
@@ -614,18 +637,19 @@ if is_tf_available():
         TFTransfoXLModel,
         TFTransfoXLPreTrainedModel,
     )
-    from .modeling_tf_utils import TFPreTrainedModel, TFSequenceSummary, TFSharedEmbeddings, shape_list
+
     from .modeling_tf_xlm import (
         TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFXLMForMultipleChoice,
         TFXLMForQuestionAnsweringSimple,
         TFXLMForSequenceClassification,
         TFXLMForTokenClassification,
+        TFXLMWithLMHeadModel,
         TFXLMMainLayer,
         TFXLMModel,
         TFXLMPreTrainedModel,
-        TFXLMWithLMHeadModel,
     )
+
     from .modeling_tf_xlm_roberta import (
         TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFXLMRobertaForMaskedLM,
@@ -635,6 +659,7 @@ if is_tf_available():
         TFXLMRobertaForTokenClassification,
         TFXLMRobertaModel,
     )
+
     from .modeling_tf_xlnet import (
         TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFXLNetForMultipleChoice,
@@ -648,11 +673,20 @@ if is_tf_available():
     )
 
     # Optimization
-    from .optimization_tf import AdamWeightDecay, GradientAccumulator, WarmUp, create_optimizer
+    from .optimization_tf import (
+        AdamWeightDecay,
+        create_optimizer,
+        GradientAccumulator,
+        WarmUp,
+    )
 
     # Trainer
     from .trainer_tf import TFTrainer
 
+    # Benchmarks
+    from .benchmark.benchmark_tf import TensorFlowBenchmark
+    from .benchmark.benchmark_args_tf import TensorFlowBenchmarkArguments
+
 
 if not is_tf_available() and not is_torch_available():
     logger.warning(

src/transformers/activations.py

@@ -1,12 +1,11 @@
+import logging
 import math
 
 import torch
 import torch.nn.functional as F
 
-from .utils import logging
 
-
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def swish(x):
@@ -14,18 +13,18 @@ def swish(x):
 
 
 def _gelu_python(x):
-    """Original Implementation of the gelu activation function in Google Bert repo when initially created.
-    For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
-    0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
-    This is now written in C in torch.nn.functional
-    Also see https://arxiv.org/abs/1606.08415
+    """ Original Implementation of the gelu activation function in Google Bert repo when initially created.
+        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
+        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+        This is now written in C in torch.nn.functional
+        Also see https://arxiv.org/abs/1606.08415
     """
     return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
 
 
 def gelu_new(x):
-    """Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
-    Also see https://arxiv.org/abs/1606.08415
+    """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
+        Also see https://arxiv.org/abs/1606.08415
     """
     return 0.5 * x * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0))))
 

src/transformers/benchmark/benchmark.py

@@ -18,13 +18,18 @@
 """
 
 
+import logging
 import timeit
 from typing import Callable, Optional
 
-from ..configuration_utils import PretrainedConfig
-from ..file_utils import is_py3nvml_available, is_torch_available
-from ..modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
-from ..utils import logging
+from transformers import (
+    MODEL_MAPPING,
+    MODEL_WITH_LM_HEAD_MAPPING,
+    PretrainedConfig,
+    is_py3nvml_available,
+    is_torch_available,
+)
+
 from .benchmark_utils import (
     Benchmark,
     Memory,
@@ -37,7 +42,6 @@ from .benchmark_utils import (
 
 if is_torch_available():
     import torch
-
     from .benchmark_args import PyTorchBenchmarkArguments
 
 
@@ -45,7 +49,7 @@ if is_py3nvml_available():
     import py3nvml.py3nvml as nvml
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 class PyTorchBenchmark(Benchmark):
@@ -199,17 +203,11 @@ class PyTorchBenchmark(Benchmark):
                 # run additional 10 times to stabilize compilation for tpu and torchscript
                 logger.info("Do inference on TPU or torchscript. Running model 5 times to stabilize compilation")
                 timeit.repeat(
-                    func,
-                    repeat=1,
-                    number=5,
+                    func, repeat=1, number=5,
                 )
 
             # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
-            runtimes = timeit.repeat(
-                func,
-                repeat=self.args.repeat,
-                number=10,
-            )
+            runtimes = timeit.repeat(func, repeat=self.args.repeat, number=10,)
 
             if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
                 import torch_xla.debug.metrics as met

src/transformers/benchmark/benchmark_args.py

@@ -14,11 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 from dataclasses import dataclass, field
 from typing import Tuple
 
 from ..file_utils import cached_property, is_torch_available, is_torch_tpu_available, torch_required
-from ..utils import logging
 from .benchmark_args_utils import BenchmarkArguments
 
 
@@ -29,7 +29,7 @@ if is_torch_tpu_available():
     import torch_xla.core.xla_model as xm
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 @dataclass

src/transformers/benchmark/benchmark_args_tf.py

@@ -14,11 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 from dataclasses import dataclass, field
 from typing import Tuple
 
 from ..file_utils import cached_property, is_tf_available, tf_required
-from ..utils import logging
 from .benchmark_args_utils import BenchmarkArguments
 
 
@@ -26,18 +26,16 @@ if is_tf_available():
     import tensorflow as tf
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 @dataclass
 class TensorFlowBenchmarkArguments(BenchmarkArguments):
     tpu_name: str = field(
-        default=None,
-        metadata={"help": "Name of TPU"},
+        default=None, metadata={"help": "Name of TPU"},
     )
     device_idx: int = field(
-        default=0,
-        metadata={"help": "CPU / GPU device index. Defaults to 0."},
+        default=0, metadata={"help": "CPU / GPU device index. Defaults to 0."},
     )
     eager_mode: bool = field(default=False, metadata={"help": "Benchmark models in eager model."})
     use_xla: bool = field(

src/transformers/benchmark/benchmark_args_utils.py

@@ -16,14 +16,13 @@
 
 import dataclasses
 import json
+import logging
 from dataclasses import dataclass, field
 from time import time
 from typing import List
 
-from ..utils import logging
 
-
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def list_field(default=None, metadata=None):

src/transformers/benchmark/benchmark_tf.py

@@ -18,15 +18,20 @@
 """
 
 
+import logging
 import random
 import timeit
 from functools import wraps
 from typing import Callable, Optional
 
-from ..configuration_utils import PretrainedConfig
-from ..file_utils import is_py3nvml_available, is_tf_available
-from ..modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
-from ..utils import logging
+from transformers import (
+    TF_MODEL_MAPPING,
+    TF_MODEL_WITH_LM_HEAD_MAPPING,
+    PretrainedConfig,
+    is_py3nvml_available,
+    is_tf_available,
+)
+
 from .benchmark_utils import (
     Benchmark,
     Memory,
@@ -39,14 +44,13 @@ from .benchmark_utils import (
 
 if is_tf_available():
     import tensorflow as tf
-    from tensorflow.python.framework.errors_impl import ResourceExhaustedError
-
     from .benchmark_args_tf import TensorFlowBenchmarkArguments
+    from tensorflow.python.framework.errors_impl import ResourceExhaustedError
 
 if is_py3nvml_available():
     import py3nvml.py3nvml as nvml
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def run_with_tf_optimizations(do_eager_mode: bool, use_xla: bool):
@@ -219,11 +223,7 @@ class TensorFlowBenchmark(Benchmark):
                     timeit.repeat(func, repeat=1, number=5)
 
                 # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
-                runtimes = timeit.repeat(
-                    func,
-                    repeat=self.args.repeat,
-                    number=10,
-                )
+                runtimes = timeit.repeat(func, repeat=self.args.repeat, number=10,)
 
                 return min(runtimes) / 10.0
             except ResourceExhaustedError as e:

src/transformers/benchmark/benchmark_utils.py

@@ -7,6 +7,7 @@ Copyright by the AllenNLP authors.
 import copy
 import csv
 import linecache
+import logging
 import os
 import platform
 import sys
@@ -21,7 +22,6 @@ from transformers import AutoConfig, PretrainedConfig
 from transformers import __version__ as version
 
 from ..file_utils import is_psutil_available, is_py3nvml_available, is_tf_available, is_torch_available
-from ..utils import logging
 from .benchmark_args_utils import BenchmarkArguments
 
 
@@ -43,7 +43,7 @@ else:
     from signal import SIGKILL
 
 
-logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
 _is_memory_tracing_enabled = False
@@ -63,15 +63,15 @@ BenchmarkOutput = namedtuple(
 
 def separate_process_wrapper_fn(func: Callable[[], None], do_multi_processing: bool) -> Callable[[], None]:
     """
-    This function wraps another function into its own separated process.
-    In order to ensure accurate memory measurements it is important that the function
-    is executed in a separate process
+        This function wraps another function into its own separated process.
+        In order to ensure accurate memory measurements it is important that the function
+        is executed in a separate process
 
-    Args:
-        - `func`: (`callable`): function() -> ...
-            generic function which will be executed in its own separate process
-        - `do_multi_processing`: (`bool`)
-            Whether to run function on separate process or not
+        Args:
+            - `func`: (`callable`): function() -> ...
+                generic function which will be executed in its own separate process
+            - `do_multi_processing`: (`bool`)
+                Whether to run function on separate process or not
     """
 
     def multi_process_func(*args, **kwargs):
@@ -94,7 +94,7 @@ def separate_process_wrapper_fn(func: Callable[[], None], do_multi_processing: b
         return result
 
     if do_multi_processing:
-        logger.info(f"Function {func} is executed in its own process...")
+        logging.info("fFunction {func} is executed in its own process...")
         return multi_process_func
     else:
         return func
@@ -106,13 +106,13 @@ def is_memory_tracing_enabled():
 
 
 class Frame(NamedTuple):
-    """`Frame` is a NamedTuple used to gather the current frame state.
-    `Frame` has the following fields:
-    - 'filename' (string): Name of the file currently executed
-    - 'module' (string): Name of the module currently executed
-    - 'line_number' (int): Number of the line currently executed
-    - 'event' (string): Event that triggered the tracing (default will be "line")
-    - 'line_text' (string): Text of the line in the python script
+    """ `Frame` is a NamedTuple used to gather the current frame state.
+            `Frame` has the following fields:
+            - 'filename' (string): Name of the file currently executed
+            - 'module' (string): Name of the module currently executed
+            - 'line_number' (int): Number of the line currently executed
+            - 'event' (string): Event that triggered the tracing (default will be "line")
+            - 'line_text' (string): Text of the line in the python script
     """
 
     filename: str
@@ -123,10 +123,10 @@ class Frame(NamedTuple):
 
 
 class UsedMemoryState(NamedTuple):
-    """`UsedMemoryState` are named tuples with the following fields:
-    - 'frame': a `Frame` namedtuple (see below) storing information on the current tracing frame (current file, location in current file)
-    - 'cpu_memory': CPU RSS memory state *before* executing the line
-    - 'gpu_memory': GPU used memory *before* executing the line (sum for all GPUs or for only `gpus_to_trace` if provided)
+    """ `UsedMemoryState` are named tuples with the following fields:
+        - 'frame': a `Frame` namedtuple (see below) storing information on the current tracing frame (current file, location in current file)
+        - 'cpu_memory': CPU RSS memory state *before* executing the line
+        - 'gpu_memory': GPU used memory *before* executing the line (sum for all GPUs or for only `gpus_to_trace` if provided)
     """
 
     frame: Frame
@@ -135,9 +135,9 @@ class UsedMemoryState(NamedTuple):
 
 
 class Memory(NamedTuple):
-    """`Memory` NamedTuple have a single field `bytes` and
-    you can get a human readable str of the number of mega bytes by calling `__repr__`
-        - `byte` (integer): number of bytes,
+    """ `Memory` NamedTuple have a single field `bytes` and
+        you can get a human readable str of the number of mega bytes by calling `__repr__`
+            - `byte` (integer): number of bytes,
     """
 
     bytes: int
@@ -147,11 +147,11 @@ class Memory(NamedTuple):
 
 
 class MemoryState(NamedTuple):
-    """`MemoryState` are namedtuples listing frame + CPU/GPU memory with the following fields:
-    - `frame` (`Frame`): the current frame (see above)
-    - `cpu`: CPU memory consumed at during the current frame as a `Memory` named tuple
-    - `gpu`: GPU memory consumed at during the current frame as a `Memory` named tuple
-    - `cpu_gpu`: CPU + GPU memory consumed at during the current frame as a `Memory` named tuple
+    """ `MemoryState` are namedtuples listing frame + CPU/GPU memory with the following fields:
+        - `frame` (`Frame`): the current frame (see above)
+        - `cpu`: CPU memory consumed at during the current frame as a `Memory` named tuple
+        - `gpu`: GPU memory consumed at during the current frame as a `Memory` named tuple
+        - `cpu_gpu`: CPU + GPU memory consumed at during the current frame as a `Memory` named tuple
     """
 
     frame: Frame
@@ -161,14 +161,14 @@ class MemoryState(NamedTuple):
 
 
 class MemorySummary(NamedTuple):
-    """`MemorySummary` namedtuple otherwise with the fields:
-    - `sequential`: a list of `MemoryState` namedtuple (see below) computed from the provided `memory_trace`
-        by substracting the memory after executing each line from the memory before executing said line.
-    - `cumulative`: a list of `MemoryState` namedtuple (see below) with cumulative increase in memory for each line
-        obtained by summing repeated memory increase for a line if it's executed several times.
-        The list is sorted from the frame with the largest memory consumption to the frame with the smallest (can be negative if memory is released)
-    - `total`: total memory increase during the full tracing as a `Memory` named tuple (see below).
-        Line with memory release (negative consumption) are ignored if `ignore_released_memory` is `True` (default).
+    """ `MemorySummary` namedtuple otherwise with the fields:
+        - `sequential`: a list of `MemoryState` namedtuple (see below) computed from the provided `memory_trace`
+            by substracting the memory after executing each line from the memory before executing said line.
+        - `cumulative`: a list of `MemoryState` namedtuple (see below) with cumulative increase in memory for each line
+            obtained by summing repeated memory increase for a line if it's executed several times.
+            The list is sorted from the frame with the largest memory consumption to the frame with the smallest (can be negative if memory is released)
+        - `total`: total memory increase during the full tracing as a `Memory` named tuple (see below).
+            Line with memory release (negative consumption) are ignored if `ignore_released_memory` is `True` (default).
     """
 
     sequential: List[MemoryState]
@@ -182,38 +182,38 @@ MemoryTrace = List[UsedMemoryState]
 
 def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5, device_idx=None) -> int:
     """
-    measures peak cpu memory consumption of a given `function`
-    running the function for at least interval seconds
-    and at most 20 * interval seconds.
-    This function is heavily inspired by: `memory_usage`
-    of the package `memory_profiler`: https://github.com/pythonprofilers/memory_profiler/blob/895c4ac7a08020d66ae001e24067da6dcea42451/memory_profiler.py#L239
+        measures peak cpu memory consumption of a given `function`
+        running the function for at least interval seconds
+        and at most 20 * interval seconds.
+        This function is heavily inspired by: `memory_usage`
+        of the package `memory_profiler`: https://github.com/pythonprofilers/memory_profiler/blob/895c4ac7a08020d66ae001e24067da6dcea42451/memory_profiler.py#L239
 
-    Args:
-        - `function`: (`callable`): function() -> ...
-            function without any arguments to measure for which to measure the peak memory
+        Args:
+            - `function`: (`callable`): function() -> ...
+                function without any arguments to measure for which to measure the peak memory
 
-        - `interval`: (`float`, `optional`, defaults to `0.5`)
-            interval in second for which to measure the memory usage
+            - `interval`: (`float`, `optional`, defaults to `0.5`)
+                interval in second for which to measure the memory usage
 
-        - `device_idx`: (`int`, `optional`, defaults to `None`)
-            device id for which to measure gpu usage
+            - `device_idx`: (`int`, `optional`, defaults to `None`)
+                device id for which to measure gpu usage
 
-    Returns:
-        - `max_memory`: (`int`)
-            cosumed memory peak in Bytes
+        Returns:
+            - `max_memory`: (`int`)
+                cosumed memory peak in Bytes
     """
 
     def get_cpu_memory(process_id: int) -> int:
         """
-        measures current cpu memory usage of a given `process_id`
+            measures current cpu memory usage of a given `process_id`
 
-        Args:
-            - `process_id`: (`int`)
-                process_id for which to measure memory
+            Args:
+                - `process_id`: (`int`)
+                    process_id for which to measure memory
 
-        Returns
-            - `memory`: (`int`)
-                cosumed memory in Bytes
+            Returns
+                - `memory`: (`int`)
+                    cosumed memory in Bytes
         """
         process = psutil.Process(process_id)
         try:
@@ -234,8 +234,8 @@ def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5, device_i
         class MemoryMeasureProcess(Process):
 
             """
-            `MemoryMeasureProcess` inherits from `Process` and overwrites
-            its `run()` method. Used to measure the memory usage of a process
+                `MemoryMeasureProcess` inherits from `Process` and overwrites
+                its `run()` method. Used to measure the memory usage of a process
             """
 
             def __init__(self, process_id: int, child_connection: Connection, interval: float):
@@ -309,37 +309,37 @@ def start_memory_tracing(
     events_to_trace: str = "line",
     gpus_to_trace: Optional[List[int]] = None,
 ) -> MemoryTrace:
-    """Setup line-by-line tracing to record rss mem (RAM) at each line of a module or sub-module.
-    See `./benchmark.py` for usage examples.
-    Current memory consumption is returned using psutil and in particular is the RSS memory
-        "Resident Set Size” (the non-swapped physical memory the process is using).
-        See https://psutil.readthedocs.io/en/latest/#psutil.Process.memory_info
+    """ Setup line-by-line tracing to record rss mem (RAM) at each line of a module or sub-module.
+        See `./benchmark.py` for usage examples.
+        Current memory consumption is returned using psutil and in particular is the RSS memory
+            "Resident Set Size” (the non-swapped physical memory the process is using).
+            See https://psutil.readthedocs.io/en/latest/#psutil.Process.memory_info
 
-    Args:
-        - `modules_to_trace`: (None, string, list/tuple of string)
-            if None, all events are recorded
-            if string or list of strings: only events from the listed module/sub-module will be recorded (e.g. 'fairseq' or 'transformers.modeling_gpt2')
-        - `modules_not_to_trace`: (None, string, list/tuple of string)
-            if None, no module is avoided
-            if string or list of strings: events from the listed module/sub-module will not be recorded (e.g. 'torch')
-        - `events_to_trace`: string or list of string of events to be recorded (see official python doc for `sys.settrace` for the list of events)
-            default to line
-        - `gpus_to_trace`: (optional list, default None) list of GPUs to trace. Default to tracing all GPUs
+        Args:
+            - `modules_to_trace`: (None, string, list/tuple of string)
+                if None, all events are recorded
+                if string or list of strings: only events from the listed module/sub-module will be recorded (e.g. 'fairseq' or 'transformers.modeling_gpt2')
+            - `modules_not_to_trace`: (None, string, list/tuple of string)
+                if None, no module is avoided
+                if string or list of strings: events from the listed module/sub-module will not be recorded (e.g. 'torch')
+            - `events_to_trace`: string or list of string of events to be recorded (see official python doc for `sys.settrace` for the list of events)
+                default to line
+            - `gpus_to_trace`: (optional list, default None) list of GPUs to trace. Default to tracing all GPUs
 
-    Return:
-        - `memory_trace` is a list of `UsedMemoryState` for each event (default each line of the traced script).
-            - `UsedMemoryState` are named tuples with the following fields:
-                - 'frame': a `Frame` namedtuple (see below) storing information on the current tracing frame (current file, location in current file)
-                - 'cpu_memory': CPU RSS memory state *before* executing the line
-                - 'gpu_memory': GPU used memory *before* executing the line (sum for all GPUs or for only `gpus_to_trace` if provided)
+        Return:
+            - `memory_trace` is a list of `UsedMemoryState` for each event (default each line of the traced script).
+                - `UsedMemoryState` are named tuples with the following fields:
+                    - 'frame': a `Frame` namedtuple (see below) storing information on the current tracing frame (current file, location in current file)
+                    - 'cpu_memory': CPU RSS memory state *before* executing the line
+                    - 'gpu_memory': GPU used memory *before* executing the line (sum for all GPUs or for only `gpus_to_trace` if provided)
 
-    `Frame` is a namedtuple used by `UsedMemoryState` to list the current frame state.
-        `Frame` has the following fields:
-        - 'filename' (string): Name of the file currently executed
-        - 'module' (string): Name of the module currently executed
-        - 'line_number' (int): Number of the line currently executed
-        - 'event' (string): Event that triggered the tracing (default will be "line")
-        - 'line_text' (string): Text of the line in the python script
+        `Frame` is a namedtuple used by `UsedMemoryState` to list the current frame state.
+            `Frame` has the following fields:
+            - 'filename' (string): Name of the file currently executed
+            - 'module' (string): Name of the module currently executed
+            - 'line_number' (int): Number of the line currently executed
+            - 'event' (string): Event that triggered the tracing (default will be "line")
+            - 'line_text' (string): Text of the line in the python script
 
     """
     if is_psutil_available():
@@ -371,8 +371,8 @@ def start_memory_tracing(
     memory_trace = []
 
     def traceit(frame, event, args):
-        """Tracing method executed before running each line in a module or sub-module
-        Record memory allocated in a list with debugging information
+        """ Tracing method executed before running each line in a module or sub-module
+            Record memory allocated in a list with debugging information
         """
         global _is_memory_tracing_enabled
 
@@ -456,39 +456,39 @@ def start_memory_tracing(
 def stop_memory_tracing(
     memory_trace: Optional[MemoryTrace] = None, ignore_released_memory: bool = True
 ) -> Optional[MemorySummary]:
-    """Stop memory tracing cleanly and return a summary of the memory trace if a trace is given.
+    """ Stop memory tracing cleanly and return a summary of the memory trace if a trace is given.
 
-    Args:
-        - `memory_trace` (optional output of start_memory_tracing, default: None): memory trace to convert in summary
-        - `ignore_released_memory` (boolean, default: None): if True we only sum memory increase to compute total memory
+        Args:
+            - `memory_trace` (optional output of start_memory_tracing, default: None): memory trace to convert in summary
+            - `ignore_released_memory` (boolean, default: None): if True we only sum memory increase to compute total memory
 
-    Return:
-        - None if `memory_trace` is None
-        - `MemorySummary` namedtuple otherwise with the fields:
-            - `sequential`: a list of `MemoryState` namedtuple (see below) computed from the provided `memory_trace`
-                by substracting the memory after executing each line from the memory before executing said line.
-            - `cumulative`: a list of `MemoryState` namedtuple (see below) with cumulative increase in memory for each line
-                obtained by summing repeated memory increase for a line if it's executed several times.
-                The list is sorted from the frame with the largest memory consumption to the frame with the smallest (can be negative if memory is released)
-            - `total`: total memory increase during the full tracing as a `Memory` named tuple (see below).
-                Line with memory release (negative consumption) are ignored if `ignore_released_memory` is `True` (default).
+        Return:
+            - None if `memory_trace` is None
+            - `MemorySummary` namedtuple otherwise with the fields:
+                - `sequential`: a list of `MemoryState` namedtuple (see below) computed from the provided `memory_trace`
+                    by substracting the memory after executing each line from the memory before executing said line.
+                - `cumulative`: a list of `MemoryState` namedtuple (see below) with cumulative increase in memory for each line
+                    obtained by summing repeated memory increase for a line if it's executed several times.
+                    The list is sorted from the frame with the largest memory consumption to the frame with the smallest (can be negative if memory is released)
+                - `total`: total memory increase during the full tracing as a `Memory` named tuple (see below).
+                    Line with memory release (negative consumption) are ignored if `ignore_released_memory` is `True` (default).
 
-    `Memory` named tuple have fields
-        - `byte` (integer): number of bytes,
-        - `string` (string): same as human readable string (ex: "3.5MB")
+        `Memory` named tuple have fields
+            - `byte` (integer): number of bytes,
+            - `string` (string): same as human readable string (ex: "3.5MB")
 
-    `Frame` are namedtuple used to list the current frame state and have the following fields:
-        - 'filename' (string): Name of the file currently executed
-        - 'module' (string): Name of the module currently executed
-        - 'line_number' (int): Number of the line currently executed
-        - 'event' (string): Event that triggered the tracing (default will be "line")
-        - 'line_text' (string): Text of the line in the python script
+        `Frame` are namedtuple used to list the current frame state and have the following fields:
+            - 'filename' (string): Name of the file currently executed
+            - 'module' (string): Name of the module currently executed
+            - 'line_number' (int): Number of the line currently executed
+            - 'event' (string): Event that triggered the tracing (default will be "line")
+            - 'line_text' (string): Text of the line in the python script
 
-    `MemoryState` are namedtuples listing frame + CPU/GPU memory with the following fields:
-        - `frame` (`Frame`): the current frame (see above)
-        - `cpu`: CPU memory consumed at during the current frame as a `Memory` named tuple
-        - `gpu`: GPU memory consumed at during the current frame as a `Memory` named tuple
-        - `cpu_gpu`: CPU + GPU memory consumed at during the current frame as a `Memory` named tuple
+        `MemoryState` are namedtuples listing frame + CPU/GPU memory with the following fields:
+            - `frame` (`Frame`): the current frame (see above)
+            - `cpu`: CPU memory consumed at during the current frame as a `Memory` named tuple
+            - `gpu`: GPU memory consumed at during the current frame as a `Memory` named tuple
+            - `cpu_gpu`: CPU + GPU memory consumed at during the current frame as a `Memory` named tuple
     """
     global _is_memory_tracing_enabled
     _is_memory_tracing_enabled = False
@@ -499,19 +499,15 @@ def stop_memory_tracing(
 
         cumulative_memory_dict = defaultdict(lambda: [0, 0, 0])
 
-        for (
-            (frame, cpu_mem, gpu_mem),
-            (next_frame, next_cpu_mem, next_gpu_mem),
-        ) in zip(memory_trace[:-1], memory_trace[1:]):
+        for ((frame, cpu_mem, gpu_mem), (next_frame, next_cpu_mem, next_gpu_mem),) in zip(
+            memory_trace[:-1], memory_trace[1:]
+        ):
             cpu_mem_inc = next_cpu_mem - cpu_mem
             gpu_mem_inc = next_gpu_mem - gpu_mem
             cpu_gpu_mem_inc = cpu_mem_inc + gpu_mem_inc
             memory_diff_trace.append(
                 MemoryState(
-                    frame=frame,
-                    cpu=Memory(cpu_mem_inc),
-                    gpu=Memory(gpu_mem_inc),
-                    cpu_gpu=Memory(cpu_gpu_mem_inc),
+                    frame=frame, cpu=Memory(cpu_mem_inc), gpu=Memory(gpu_mem_inc), cpu_gpu=Memory(cpu_gpu_mem_inc),
                 )
             )
 
@@ -533,10 +529,7 @@ def stop_memory_tracing(
         )  # order by the total CPU + GPU memory increase
         cumulative_memory = list(
             MemoryState(
-                frame=frame,
-                cpu=Memory(cpu_mem_inc),
-                gpu=Memory(gpu_mem_inc),
-                cpu_gpu=Memory(cpu_gpu_mem_inc),
+                frame=frame, cpu=Memory(cpu_mem_inc), gpu=Memory(gpu_mem_inc), cpu_gpu=Memory(cpu_gpu_mem_inc),
             )
             for frame, (cpu_mem_inc, gpu_mem_inc, cpu_gpu_mem_inc) in cumulative_memory
         )
@@ -551,17 +544,15 @@ def stop_memory_tracing(
         total_memory = Memory(total_memory)
 
         return MemorySummary(
-            sequential=memory_diff_trace,
-            cumulative=cumulative_memory,
-            current=memory_curr_trace,
-            total=total_memory,
+            sequential=memory_diff_trace, cumulative=cumulative_memory, current=memory_curr_trace, total=total_memory,
         )
 
     return None
 
 
 def bytes_to_mega_bytes(memory_amount: int) -> int:
-    """Utility to convert a number of bytes (int) into a number of mega bytes (int)"""
+    """ Utility to convert a number of bytes (int) into a number of mega bytes (int)
+    """
     return memory_amount >> 20
 
 

src/transformers/commands/convert.py

@@ -1,9 +1,8 @@
 from argparse import ArgumentParser, Namespace
+from logging import getLogger
 
 from transformers.commands import BaseTransformersCLICommand
 
-from ..utils import logging
-
 
 def convert_command_factory(args: Namespace):
     """
@@ -53,7 +52,7 @@ class ConvertCommand(BaseTransformersCLICommand):
         finetuning_task_name: str,
         *args
     ):
-        self._logger = logging.get_logger("transformers-cli/converting")
+        self._logger = getLogger("transformers-cli/converting")
 
         self._logger.info("Loading model {}".format(model_type))
         self._model_type = model_type

src/transformers/commands/run.py

@@ -1,12 +1,11 @@
+import logging
 from argparse import ArgumentParser
 
 from transformers.commands import BaseTransformersCLICommand
 from transformers.pipelines import SUPPORTED_TASKS, Pipeline, PipelineDataFormat, pipeline
 
-from ..utils import logging
 
-
-logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
 def try_infer_format_from_ext(path: str):

src/transformers/commands/serving.py

@@ -1,3 +1,4 @@
+import logging
 from argparse import ArgumentParser, Namespace
 from typing import Any, List, Optional
 
@@ -5,15 +6,13 @@ from transformers import Pipeline
 from transformers.commands import BaseTransformersCLICommand
 from transformers.pipelines import SUPPORTED_TASKS, pipeline
 
-from ..utils import logging
-
 
 try:
-    from fastapi import Body, FastAPI, HTTPException
+    from uvicorn import run
+    from fastapi import FastAPI, HTTPException, Body
     from fastapi.routing import APIRoute
     from pydantic import BaseModel
     from starlette.responses import JSONResponse
-    from uvicorn import run
 
     _serve_dependencies_installed = True
 except (ImportError, AttributeError):

src/transformers/commands/train.py

@@ -1,12 +1,11 @@
 import os
 from argparse import ArgumentParser, Namespace
+from logging import getLogger
 
 from transformers import SingleSentenceClassificationProcessor as Processor
 from transformers import TextClassificationPipeline, is_tf_available, is_torch_available
 from transformers.commands import BaseTransformersCLICommand
 
-from ..utils import logging
-
 
 if not is_tf_available() and not is_torch_available():
     raise RuntimeError("At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training")
@@ -77,7 +76,7 @@ class TrainCommand(BaseTransformersCLICommand):
         train_parser.set_defaults(func=train_command_factory)
 
     def __init__(self, args: Namespace):
-        self.logger = logging.get_logger("transformers-cli/training")
+        self.logger = getLogger("transformers-cli/training")
 
         self.framework = "tf" if is_tf_available() else "torch"
 

src/transformers/commands/user.py

@@ -5,6 +5,7 @@ from getpass import getpass
 from typing import List, Union
 
 from requests.exceptions import HTTPError
+
 from transformers.commands import BaseTransformersCLICommand
 from transformers.hf_api import HfApi, HfFolder
 

src/transformers/configuration_albert.py

@@ -32,71 +32,71 @@ ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 class AlbertConfig(PretrainedConfig):
     r"""
-    This is the configuration class to store the configuration of a :class:`~transformers.AlbertModel`.
-    It is used to instantiate an ALBERT model according to the specified arguments, defining the model
-    architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
-    the ALBERT `xxlarge <https://huggingface.co/albert-xxlarge-v2>`__ architecture.
+        This is the configuration class to store the configuration of a :class:`~transformers.AlbertModel`.
+        It is used to instantiate an ALBERT model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the ALBERT `xxlarge <https://huggingface.co/albert-xxlarge-v2>`__ architecture.
 
-    Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
-    to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
-    for more information.
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
 
 
-    Args:
-        vocab_size (:obj:`int`, optional, defaults to 30000):
-            Vocabulary size of the ALBERT model. Defines the different tokens that
-            can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.AlbertModel`.
-        embedding_size (:obj:`int`, optional, defaults to 128):
-            Dimensionality of vocabulary embeddings.
-        hidden_size (:obj:`int`, optional, defaults to 4096):
-            Dimensionality of the encoder layers and the pooler layer.
-        num_hidden_layers (:obj:`int`, optional, defaults to 12):
-            Number of hidden layers in the Transformer encoder.
-        num_hidden_groups (:obj:`int`, optional, defaults to 1):
-            Number of groups for the hidden layers, parameters in the same group are shared.
-        num_attention_heads (:obj:`int`, optional, defaults to 64):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        intermediate_size (:obj:`int`, optional, defaults to 16384):
-            The dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
-        inner_group_num (:obj:`int`, optional, defaults to 1):
-            The number of inner repetition of attention and ffn.
-        hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "gelu_new"):
-            The non-linear activation function (function or string) in the encoder and pooler.
-            If string, "gelu", "relu", "swish" and "gelu_new" are supported.
-        hidden_dropout_prob (:obj:`float`, optional, defaults to 0):
-            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
-        attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0):
-            The dropout ratio for the attention probabilities.
-        max_position_embeddings (:obj:`int`, optional, defaults to 512):
-            The maximum sequence length that this model might ever be used with. Typically set this to something
-            large (e.g., 512 or 1024 or 2048).
-        type_vocab_size (:obj:`int`, optional, defaults to 2):
-            The vocabulary size of the `token_type_ids` passed into :class:`~transformers.AlbertModel`.
-        initializer_range (:obj:`float`, optional, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
-            The epsilon used by the layer normalization layers.
-        classifier_dropout_prob (:obj:`float`, optional, defaults to 0.1):
-            The dropout ratio for attached classifiers.
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 30000):
+                Vocabulary size of the ALBERT model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.AlbertModel`.
+            embedding_size (:obj:`int`, optional, defaults to 128):
+                Dimensionality of vocabulary embeddings.
+            hidden_size (:obj:`int`, optional, defaults to 4096):
+                Dimensionality of the encoder layers and the pooler layer.
+            num_hidden_layers (:obj:`int`, optional, defaults to 12):
+                Number of hidden layers in the Transformer encoder.
+            num_hidden_groups (:obj:`int`, optional, defaults to 1):
+                Number of groups for the hidden layers, parameters in the same group are shared.
+            num_attention_heads (:obj:`int`, optional, defaults to 64):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            intermediate_size (:obj:`int`, optional, defaults to 16384):
+                The dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+            inner_group_num (:obj:`int`, optional, defaults to 1):
+                The number of inner repetition of attention and ffn.
+            hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "gelu_new"):
+                The non-linear activation function (function or string) in the encoder and pooler.
+                If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            hidden_dropout_prob (:obj:`float`, optional, defaults to 0):
+                The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0):
+                The dropout ratio for the attention probabilities.
+            max_position_embeddings (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might ever be used with. Typically set this to something
+                large (e.g., 512 or 1024 or 2048).
+            type_vocab_size (:obj:`int`, optional, defaults to 2):
+                The vocabulary size of the `token_type_ids` passed into :class:`~transformers.AlbertModel`.
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
+                The epsilon used by the layer normalization layers.
+            classifier_dropout_prob (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for attached classifiers.
 
-    Example::
+        Example::
 
-        >>> from transformers import AlbertConfig, AlbertModel
-        >>> # Initializing an ALBERT-xxlarge style configuration
-        >>> albert_xxlarge_configuration = AlbertConfig()
+            >>> from transformers import AlbertConfig, AlbertModel
+            >>> # Initializing an ALBERT-xxlarge style configuration
+            >>> albert_xxlarge_configuration = AlbertConfig()
 
-        >>> # Initializing an ALBERT-base style configuration
-        >>> albert_base_configuration = AlbertConfig(
-        ...      hidden_size=768,
-        ...      num_attention_heads=12,
-        ...      intermediate_size=3072,
-        ...  )
+            >>> # Initializing an ALBERT-base style configuration
+            >>> albert_base_configuration = AlbertConfig(
+            ...      hidden_size=768,
+            ...      num_attention_heads=12,
+            ...      intermediate_size=3072,
+            ...  )
 
-        >>> # Initializing a model from the ALBERT-base style configuration
-        >>> model = AlbertModel(albert_xxlarge_configuration)
+            >>> # Initializing a model from the ALBERT-base style configuration
+            >>> model = AlbertModel(albert_xxlarge_configuration)
 
-        >>> # Accessing the model configuration
-        >>> configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
 
     model_type = "albert"

src/transformers/configuration_auto.py

@@ -15,6 +15,7 @@
 """ Auto Config class. """
 
 
+import logging
 from collections import OrderedDict
 
 from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
@@ -44,6 +45,9 @@ from .configuration_xlm_roberta import XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP
 from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig
 
 
+logger = logging.getLogger(__name__)
+
+
 ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
     (key, value)
     for pretrained_map in [
@@ -73,112 +77,43 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
 
 CONFIG_MAPPING = OrderedDict(
     [
-        (
-            "retribert",
-            RetriBertConfig,
-        ),
-        (
-            "t5",
-            T5Config,
-        ),
-        (
-            "mobilebert",
-            MobileBertConfig,
-        ),
-        (
-            "distilbert",
-            DistilBertConfig,
-        ),
-        (
-            "albert",
-            AlbertConfig,
-        ),
-        (
-            "camembert",
-            CamembertConfig,
-        ),
-        (
-            "xlm-roberta",
-            XLMRobertaConfig,
-        ),
+        ("retribert", RetriBertConfig,),
+        ("t5", T5Config,),
+        ("mobilebert", MobileBertConfig,),
+        ("distilbert", DistilBertConfig,),
+        ("albert", AlbertConfig,),
+        ("camembert", CamembertConfig,),
+        ("xlm-roberta", XLMRobertaConfig,),
         ("pegasus", PegasusConfig),
-        (
-            "marian",
-            MarianConfig,
-        ),
-        (
-            "mbart",
-            MBartConfig,
-        ),
-        (
-            "bart",
-            BartConfig,
-        ),
-        (
-            "reformer",
-            ReformerConfig,
-        ),
-        (
-            "longformer",
-            LongformerConfig,
-        ),
-        (
-            "roberta",
-            RobertaConfig,
-        ),
-        (
-            "flaubert",
-            FlaubertConfig,
-        ),
-        (
-            "bert",
-            BertConfig,
-        ),
-        (
-            "openai-gpt",
-            OpenAIGPTConfig,
-        ),
-        (
-            "gpt2",
-            GPT2Config,
-        ),
-        (
-            "transfo-xl",
-            TransfoXLConfig,
-        ),
-        (
-            "xlnet",
-            XLNetConfig,
-        ),
-        (
-            "xlm",
-            XLMConfig,
-        ),
-        (
-            "ctrl",
-            CTRLConfig,
-        ),
-        (
-            "electra",
-            ElectraConfig,
-        ),
-        (
-            "encoder-decoder",
-            EncoderDecoderConfig,
-        ),
+        ("marian", MarianConfig,),
+        ("mbart", MBartConfig,),
+        ("bart", BartConfig,),
+        ("reformer", ReformerConfig,),
+        ("longformer", LongformerConfig,),
+        ("roberta", RobertaConfig,),
+        ("flaubert", FlaubertConfig,),
+        ("bert", BertConfig,),
+        ("openai-gpt", OpenAIGPTConfig,),
+        ("gpt2", GPT2Config,),
+        ("transfo-xl", TransfoXLConfig,),
+        ("xlnet", XLNetConfig,),
+        ("xlm", XLMConfig,),
+        ("ctrl", CTRLConfig,),
+        ("electra", ElectraConfig,),
+        ("encoder-decoder", EncoderDecoderConfig,),
     ]
 )
 
 
 class AutoConfig:
     r"""
-    :class:`~transformers.AutoConfig` is a generic configuration class
-    that will be instantiated as one of the configuration classes of the library
-    when created with the :func:`~transformers.AutoConfig.from_pretrained` class method.
+        :class:`~transformers.AutoConfig` is a generic configuration class
+        that will be instantiated as one of the configuration classes of the library
+        when created with the :func:`~transformers.AutoConfig.from_pretrained` class method.
 
-    The :func:`~transformers.AutoConfig.from_pretrained` method takes care of returning the correct model class instance
-    based on the `model_type` property of the config object, or when it's missing,
-    falling back to using pattern matching on the `pretrained_model_name_or_path` string.
+        The :func:`~transformers.AutoConfig.from_pretrained` method takes care of returning the correct model class instance
+        based on the `model_type` property of the config object, or when it's missing,
+        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
     """
 
     def __init__(self):

src/transformers/configuration_bart.py

@@ -14,12 +14,14 @@
 # limitations under the License.
 """ BART configuration """
 
+
+import logging
+
 from .configuration_utils import PretrainedConfig
 from .file_utils import add_start_docstrings_to_callable
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 BART_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "facebook/bart-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/bart-base/config.json",
@@ -102,7 +104,7 @@ BART_CONFIG_ARGS_DOC = r"""
 @add_start_docstrings_to_callable(BART_CONFIG_ARGS_DOC)
 class BartConfig(PretrainedConfig):
     r"""
-    Configuration class for Bart. Parameters are renamed from the fairseq implementation
+        Configuration class for Bart. Parameters are renamed from the fairseq implementation
     """
     model_type = "bart"
 
@@ -141,14 +143,14 @@ class BartConfig(PretrainedConfig):
         **common_kwargs
     ):
         r"""
-        :class:`~transformers.BartConfig` is the configuration class for `BartModel`.
+            :class:`~transformers.BartConfig` is the configuration class for `BartModel`.
 
-        Examples::
+            Examples::
 
-            >>> from transformers import BartConfig, BartModel
+                >>> from transformers import BartConfig, BartModel
 
-            >>> config = BartConfig.from_pretrained('facebook/bart-large')
-            >>> model = BartModel(config)
+                >>> config = BartConfig.from_pretrained('facebook/bart-large')
+                >>> model = BartModel(config)
 
         """
         if "hidden_size" in common_kwargs:

src/transformers/configuration_bert.py

@@ -15,11 +15,13 @@
 # limitations under the License.
 """ BERT model configuration """
 
+
+import logging
+
 from .configuration_utils import PretrainedConfig
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "bert-base-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json",
@@ -50,59 +52,59 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 class BertConfig(PretrainedConfig):
     r"""
-    This is the configuration class to store the configuration of a :class:`~transformers.BertModel`.
-    It is used to instantiate an BERT model according to the specified arguments, defining the model
-    architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
-    the BERT `bert-base-uncased <https://huggingface.co/bert-base-uncased>`__ architecture.
+        This is the configuration class to store the configuration of a :class:`~transformers.BertModel`.
+        It is used to instantiate an BERT model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the BERT `bert-base-uncased <https://huggingface.co/bert-base-uncased>`__ architecture.
 
-    Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
-    to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
-    for more information.
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
 
 
-    Args:
-        vocab_size (:obj:`int`, optional, defaults to 30522):
-            Vocabulary size of the BERT model. Defines the different tokens that
-            can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.BertModel`.
-        hidden_size (:obj:`int`, optional, defaults to 768):
-            Dimensionality of the encoder layers and the pooler layer.
-        num_hidden_layers (:obj:`int`, optional, defaults to 12):
-            Number of hidden layers in the Transformer encoder.
-        num_attention_heads (:obj:`int`, optional, defaults to 12):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        intermediate_size (:obj:`int`, optional, defaults to 3072):
-            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
-        hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
-            The non-linear activation function (function or string) in the encoder and pooler.
-            If string, "gelu", "relu", "swish" and "gelu_new" are supported.
-        hidden_dropout_prob (:obj:`float`, optional, defaults to 0.1):
-            The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
-        attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0.1):
-            The dropout ratio for the attention probabilities.
-        max_position_embeddings (:obj:`int`, optional, defaults to 512):
-            The maximum sequence length that this model might ever be used with.
-            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
-        type_vocab_size (:obj:`int`, optional, defaults to 2):
-            The vocabulary size of the `token_type_ids` passed into :class:`~transformers.BertModel`.
-        initializer_range (:obj:`float`, optional, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
-            The epsilon used by the layer normalization layers.
-        gradient_checkpointing (:obj:`bool`, optional, defaults to False):
-            If True, use gradient checkpointing to save memory at the expense of slower backward pass.
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 30522):
+                Vocabulary size of the BERT model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.BertModel`.
+            hidden_size (:obj:`int`, optional, defaults to 768):
+                Dimensionality of the encoder layers and the pooler layer.
+            num_hidden_layers (:obj:`int`, optional, defaults to 12):
+                Number of hidden layers in the Transformer encoder.
+            num_attention_heads (:obj:`int`, optional, defaults to 12):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            intermediate_size (:obj:`int`, optional, defaults to 3072):
+                Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+            hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
+                The non-linear activation function (function or string) in the encoder and pooler.
+                If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            hidden_dropout_prob (:obj:`float`, optional, defaults to 0.1):
+                The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention probabilities.
+            max_position_embeddings (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            type_vocab_size (:obj:`int`, optional, defaults to 2):
+                The vocabulary size of the `token_type_ids` passed into :class:`~transformers.BertModel`.
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
+                The epsilon used by the layer normalization layers.
+            gradient_checkpointing (:obj:`bool`, optional, defaults to False):
+                If True, use gradient checkpointing to save memory at the expense of slower backward pass.
 
-    Example::
+        Example::
 
-        >>> from transformers import BertModel, BertConfig
+            >>> from transformers import BertModel, BertConfig
 
-        >>> # Initializing a BERT bert-base-uncased style configuration
-        >>> configuration = BertConfig()
+            >>> # Initializing a BERT bert-base-uncased style configuration
+            >>> configuration = BertConfig()
 
-        >>> # Initializing a model from the bert-base-uncased style configuration
-        >>> model = BertModel(configuration)
+            >>> # Initializing a model from the bert-base-uncased style configuration
+            >>> model = BertModel(configuration)
 
-        >>> # Accessing the model configuration
-        >>> configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
     model_type = "bert"
 

src/transformers/configuration_camembert.py

@@ -15,11 +15,13 @@
 # limitations under the License.
 """ CamemBERT configuration """
 
+
+import logging
+
 from .configuration_roberta import RobertaConfig
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "camembert-base": "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json",

src/transformers/configuration_ctrl.py

@@ -14,66 +14,68 @@
 # limitations under the License.
 """ Salesforce CTRL configuration """
 
+
+import logging
+
 from .configuration_utils import PretrainedConfig
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP = {"ctrl": "https://s3.amazonaws.com/models.huggingface.co/bert/ctrl-config.json"}
 
 
 class CTRLConfig(PretrainedConfig):
     """
-    This is the configuration class to store the configuration of a :class:`~transformers.CTRLModel`.
-    It is used to instantiate an CTRL model according to the specified arguments, defining the model
-    architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
-    the `ctrl <https://huggingface.co/ctrl>`__ architecture from SalesForce.
+        This is the configuration class to store the configuration of a :class:`~transformers.CTRLModel`.
+        It is used to instantiate an CTRL model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the `ctrl <https://huggingface.co/ctrl>`__ architecture from SalesForce.
 
-    Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
-    to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
-    for more information.
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
 
-    Args:
-        vocab_size (:obj:`int`, optional, defaults to 246534):
-            Vocabulary size of the CTRL model. Defines the different tokens that
-            can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.CTRLModel`.
-        n_positions (:obj:`int`, optional, defaults to 256):
-            The maximum sequence length that this model might ever be used with.
-            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
-        n_ctx (:obj:`int`, optional, defaults to 256):
-            Dimensionality of the causal mask (usually same as n_positions).
-        n_embd (:obj:`int`, optional, defaults to 1280):
-            Dimensionality of the embeddings and hidden states.
-        dff (:obj:`int`, optional, defaults to 8192):
-            Dimensionality of the inner dimension of the FFN.
-        n_layer (:obj:`int`, optional, defaults to 48):
-            Number of hidden layers in the Transformer encoder.
-        n_head (:obj:`int`, optional, defaults to 16):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        resid_pdrop (:obj:`float`, optional, defaults to 0.1):
-            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
-        embd_pdrop (:obj:`int`, optional, defaults to 0.1):
-            The dropout ratio for the embeddings.
-        attn_pdrop (:obj:`float`, optional, defaults to 0.1):
-            The dropout ratio for the attention.
-        layer_norm_epsilon (:obj:`float`, optional, defaults to 1e-6):
-            The epsilon to use in the layer normalization layers
-        initializer_range (:obj:`float`, optional, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 246534):
+                Vocabulary size of the CTRL model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.CTRLModel`.
+            n_positions (:obj:`int`, optional, defaults to 256):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            n_ctx (:obj:`int`, optional, defaults to 256):
+                Dimensionality of the causal mask (usually same as n_positions).
+            n_embd (:obj:`int`, optional, defaults to 1280):
+                Dimensionality of the embeddings and hidden states.
+            dff (:obj:`int`, optional, defaults to 8192):
+                Dimensionality of the inner dimension of the FFN.
+            n_layer (:obj:`int`, optional, defaults to 48):
+                Number of hidden layers in the Transformer encoder.
+            n_head (:obj:`int`, optional, defaults to 16):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            resid_pdrop (:obj:`float`, optional, defaults to 0.1):
+                The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+            embd_pdrop (:obj:`int`, optional, defaults to 0.1):
+                The dropout ratio for the embeddings.
+            attn_pdrop (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention.
+            layer_norm_epsilon (:obj:`float`, optional, defaults to 1e-6):
+                The epsilon to use in the layer normalization layers
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
 
-    Example::
+        Example::
 
-        >>> from transformers import CTRLModel, CTRLConfig
+            >>> from transformers import CTRLModel, CTRLConfig
 
-        >>> # Initializing a CTRL configuration
-        >>> configuration = CTRLConfig()
+            >>> # Initializing a CTRL configuration
+            >>> configuration = CTRLConfig()
 
-        >>> # Initializing a model from the configuration
-        >>> model = CTRLModel(configuration)
+            >>> # Initializing a model from the configuration
+            >>> model = CTRLModel(configuration)
 
-        >>> # Accessing the model configuration
-        >>> configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
 
     model_type = "ctrl"

src/transformers/configuration_distilbert.py

@@ -14,11 +14,13 @@
 # limitations under the License.
 """ DistilBERT model configuration """
 
+
+import logging
+
 from .configuration_utils import PretrainedConfig
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "distilbert-base-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json",
@@ -33,61 +35,61 @@ DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 class DistilBertConfig(PretrainedConfig):
     r"""
-    This is the configuration class to store the configuration of a :class:`~transformers.DistilBertModel`.
-    It is used to instantiate a DistilBERT model according to the specified arguments, defining the model
-    architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
-    the DistilBERT `distilbert-base-uncased <https://huggingface.co/distilbert-base-uncased>`__ architecture.
+        This is the configuration class to store the configuration of a :class:`~transformers.DistilBertModel`.
+        It is used to instantiate a DistilBERT model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the DistilBERT `distilbert-base-uncased <https://huggingface.co/distilbert-base-uncased>`__ architecture.
 
-    Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
-    to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
-    for more information.
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
 
 
-    Args:
-        vocab_size (:obj:`int`, optional, defaults to 30522):
-            Vocabulary size of the DistilBERT model. Defines the different tokens that
-            can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.BertModel`.
-        max_position_embeddings (:obj:`int`, optional, defaults to 512):
-            The maximum sequence length that this model might ever be used with.
-            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
-        sinusoidal_pos_embds (:obj:`boolean`, optional, defaults to :obj:`False`):
-            Whether to use sinusoidal positional embeddings.
-        n_layers (:obj:`int`, optional, defaults to 6):
-            Number of hidden layers in the Transformer encoder.
-        n_heads (:obj:`int`, optional, defaults to 12):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        dim (:obj:`int`, optional, defaults to 768):
-            Dimensionality of the encoder layers and the pooler layer.
-        hidden_dim (:obj:`int`, optional, defaults to 3072):
-            The size of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
-        dropout (:obj:`float`, optional, defaults to 0.1):
-            The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
-        attention_dropout (:obj:`float`, optional, defaults to 0.1):
-            The dropout ratio for the attention probabilities.
-        activation (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
-            The non-linear activation function (function or string) in the encoder and pooler.
-            If string, "gelu", "relu", "swish" and "gelu_new" are supported.
-        initializer_range (:obj:`float`, optional, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        qa_dropout (:obj:`float`, optional, defaults to 0.1):
-            The dropout probabilities used in the question answering model
-            :class:`~transformers.DistilBertForQuestionAnswering`.
-        seq_classif_dropout (:obj:`float`, optional, defaults to 0.2):
-            The dropout probabilities used in the sequence classification and the multiple choice model
-            :class:`~transformers.DistilBertForSequenceClassification`.
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 30522):
+                Vocabulary size of the DistilBERT model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.BertModel`.
+            max_position_embeddings (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            sinusoidal_pos_embds (:obj:`boolean`, optional, defaults to :obj:`False`):
+                Whether to use sinusoidal positional embeddings.
+            n_layers (:obj:`int`, optional, defaults to 6):
+                Number of hidden layers in the Transformer encoder.
+            n_heads (:obj:`int`, optional, defaults to 12):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            dim (:obj:`int`, optional, defaults to 768):
+                Dimensionality of the encoder layers and the pooler layer.
+            hidden_dim (:obj:`int`, optional, defaults to 3072):
+                The size of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+            dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
+            attention_dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention probabilities.
+            activation (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
+                The non-linear activation function (function or string) in the encoder and pooler.
+                If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            qa_dropout (:obj:`float`, optional, defaults to 0.1):
+                The dropout probabilities used in the question answering model
+                :class:`~transformers.DistilBertForQuestionAnswering`.
+            seq_classif_dropout (:obj:`float`, optional, defaults to 0.2):
+                The dropout probabilities used in the sequence classification and the multiple choice model
+                :class:`~transformers.DistilBertForSequenceClassification`.
 
-    Example::
+        Example::
 
-        >>> from transformers import DistilBertModel, DistilBertConfig
+            >>> from transformers import DistilBertModel, DistilBertConfig
 
-        >>> # Initializing a DistilBERT configuration
-        >>> configuration = DistilBertConfig()
+            >>> # Initializing a DistilBERT configuration
+            >>> configuration = DistilBertConfig()
 
-        >>> # Initializing a model from the configuration
-        >>> model = DistilBertModel(configuration)
+            >>> # Initializing a model from the configuration
+            >>> model = DistilBertModel(configuration)
 
-        >>> # Accessing the model configuration
-        >>> configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
     model_type = "distilbert"
 

src/transformers/configuration_dpr.py

@@ -14,11 +14,13 @@
 # limitations under the License.
 """ DPR model configuration """
 
+
+import logging
+
 from .configuration_bert import BertConfig
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 DPR_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "facebook/dpr-ctx_encoder-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-ctx_encoder-single-nq-base/config.json",
@@ -29,16 +31,16 @@ DPR_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 class DPRConfig(BertConfig):
     r"""
-    :class:`~transformers.DPRConfig` is the configuration class to store the configuration of a
-    `DPRModel`.
+        :class:`~transformers.DPRConfig` is the configuration class to store the configuration of a
+        `DPRModel`.
 
-    This is the configuration class to store the configuration of a `DPRContextEncoder`, `DPRQuestionEncoder`, or a `DPRReader`.
-    It is used to instantiate the components of the DPR model.
+        This is the configuration class to store the configuration of a `DPRContextEncoder`, `DPRQuestionEncoder`, or a `DPRReader`.
+        It is used to instantiate the components of the DPR model.
 
-    Args:
-        projection_dim (:obj:`int`, optional, defaults to 0):
-            Dimension of the projection for the context and question encoders.
-            If it is set to zero (default), then no projection is done.
+        Args:
+            projection_dim (:obj:`int`, optional, defaults to 0):
+                Dimension of the projection for the context and question encoders.
+                If it is set to zero (default), then no projection is done.
     """
     model_type = "dpr"
 

src/transformers/configuration_electra.py

@@ -15,11 +15,13 @@
 # limitations under the License.
 """ ELECTRA model configuration """
 
+
+import logging
+
 from .configuration_utils import PretrainedConfig
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "google/electra-small-generator": "https://s3.amazonaws.com/models.huggingface.co/bert/google/electra-small-generator/config.json",
@@ -33,82 +35,82 @@ ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 class ElectraConfig(PretrainedConfig):
     r"""
-    This is the configuration class to store the configuration of a :class:`~transformers.ElectraModel`.
-    It is used to instantiate an ELECTRA model according to the specified arguments, defining the model
-    architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
-    the ELECTRA `google/electra-small-discriminator <https://huggingface.co/google/electra-small-discriminator>`__
-    architecture.
+        This is the configuration class to store the configuration of a :class:`~transformers.ElectraModel`.
+        It is used to instantiate an ELECTRA model according to the specified arguments, defining the model
+        architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+        the ELECTRA `google/electra-small-discriminator <https://huggingface.co/google/electra-small-discriminator>`__
+        architecture.
 
-    Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
-    to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
-    for more information.
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
 
 
-    Args:
-        vocab_size (:obj:`int`, optional, defaults to 30522):
-            Vocabulary size of the ELECTRA model. Defines the different tokens that
-            can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.ElectraModel`.
-        embedding_size (:obj:`int`, optional, defaults to 128):
-            Dimensionality of the encoder layers and the pooler layer.
-        hidden_size (:obj:`int`, optional, defaults to 256):
-            Dimensionality of the encoder layers and the pooler layer.
-        num_hidden_layers (:obj:`int`, optional, defaults to 12):
-            Number of hidden layers in the Transformer encoder.
-        num_attention_heads (:obj:`int`, optional, defaults to 4):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        intermediate_size (:obj:`int`, optional, defaults to 1024):
-            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
-        hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
-            The non-linear activation function (function or string) in the encoder and pooler.
-            If string, "gelu", "relu", "swish" and "gelu_new" are supported.
-        hidden_dropout_prob (:obj:`float`, optional, defaults to 0.1):
-            The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
-        attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0.1):
-            The dropout ratio for the attention probabilities.
-        max_position_embeddings (:obj:`int`, optional, defaults to 512):
-            The maximum sequence length that this model might ever be used with.
-            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
-        type_vocab_size (:obj:`int`, optional, defaults to 2):
-            The vocabulary size of the `token_type_ids` passed into :class:`~transformers.ElectraModel`.
-        initializer_range (:obj:`float`, optional, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
-            The epsilon used by the layer normalization layers.
-        summary_type (:obj:`string`, optional, defaults to "first"):
-            Argument used when doing sequence summary. Used in for the multiple choice head in
-            :class:`~transformers.ElectraForMultipleChoice`.
-            Is one of the following options:
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 30522):
+                Vocabulary size of the ELECTRA model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.ElectraModel`.
+            embedding_size (:obj:`int`, optional, defaults to 128):
+                Dimensionality of the encoder layers and the pooler layer.
+            hidden_size (:obj:`int`, optional, defaults to 256):
+                Dimensionality of the encoder layers and the pooler layer.
+            num_hidden_layers (:obj:`int`, optional, defaults to 12):
+                Number of hidden layers in the Transformer encoder.
+            num_attention_heads (:obj:`int`, optional, defaults to 4):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            intermediate_size (:obj:`int`, optional, defaults to 1024):
+                Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+            hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
+                The non-linear activation function (function or string) in the encoder and pooler.
+                If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            hidden_dropout_prob (:obj:`float`, optional, defaults to 0.1):
+                The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention probabilities.
+            max_position_embeddings (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            type_vocab_size (:obj:`int`, optional, defaults to 2):
+                The vocabulary size of the `token_type_ids` passed into :class:`~transformers.ElectraModel`.
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
+                The epsilon used by the layer normalization layers.
+            summary_type (:obj:`string`, optional, defaults to "first"):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.ElectraForMultipleChoice`.
+                Is one of the following options:
 
-                - 'last' => take the last token hidden state (like XLNet)
-                - 'first' => take the first token hidden state (like Bert)
-                - 'mean' => take the mean of all tokens hidden states
-                - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2)
-                - 'attn' => Not implemented now, use multi-head attention
-        summary_use_proj (:obj:`boolean`, optional, defaults to :obj:`True`):
-            Argument used when doing sequence summary. Used in for the multiple choice head in
-            :class:`~transformers.ElectraForMultipleChoice`.
-            Add a projection after the vector extraction
-        summary_activation (:obj:`string` or :obj:`None`, optional, defaults to :obj:`None`):
-            Argument used when doing sequence summary. Used in for the multiple choice head in
-            :class:`~transformers.ElectraForMultipleChoice`.
-            'gelu' => add a gelu activation to the output, Other => no activation.
-        summary_last_dropout (:obj:`float`, optional, defaults to 0.0):
-            Argument used when doing sequence summary. Used in for the multiple choice head in
-            :class:`~transformers.ElectraForMultipleChoice`.
-            Add a dropout after the projection and activation
+                    - 'last' => take the last token hidden state (like XLNet)
+                    - 'first' => take the first token hidden state (like Bert)
+                    - 'mean' => take the mean of all tokens hidden states
+                    - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2)
+                    - 'attn' => Not implemented now, use multi-head attention
+            summary_use_proj (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.ElectraForMultipleChoice`.
+                Add a projection after the vector extraction
+            summary_activation (:obj:`string` or :obj:`None`, optional, defaults to :obj:`None`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.ElectraForMultipleChoice`.
+                'gelu' => add a gelu activation to the output, Other => no activation.
+            summary_last_dropout (:obj:`float`, optional, defaults to 0.0):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.ElectraForMultipleChoice`.
+                Add a dropout after the projection and activation
 
-    Example::
+        Example::
 
-        >>> from transformers import ElectraModel, ElectraConfig
+            >>> from transformers import ElectraModel, ElectraConfig
 
-        >>> # Initializing a ELECTRA electra-base-uncased style configuration
-        >>> configuration = ElectraConfig()
+            >>> # Initializing a ELECTRA electra-base-uncased style configuration
+            >>> configuration = ElectraConfig()
 
-        >>> # Initializing a model from the electra-base-uncased style configuration
-        >>> model = ElectraModel(configuration)
+            >>> # Initializing a model from the electra-base-uncased style configuration
+            >>> model = ElectraModel(configuration)
 
-        >>> # Accessing the model configuration
-        >>> configuration = model.config
+            >>> # Accessing the model configuration
+            >>> configuration = model.config
     """
     model_type = "electra"
 

src/transformers/configuration_encoder_decoder.py

@@ -15,57 +15,57 @@
 # limitations under the License.
 
 import copy
+import logging
 
 from .configuration_utils import PretrainedConfig
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 class EncoderDecoderConfig(PretrainedConfig):
     r"""
-    :class:`~transformers.EncoderDecoderConfig` is the configuration class to store the configuration of a `EncoderDecoderModel`.
+        :class:`~transformers.EncoderDecoderConfig` is the configuration class to store the configuration of a `EncoderDecoderModel`.
 
-    It is used to instantiate an Encoder Decoder model according to the specified arguments, defining the encoder and decoder configs.
-    Configuration objects inherit from  :class:`~transformers.PretrainedConfig`
-    and can be used to control the model outputs.
-    See the documentation for :class:`~transformers.PretrainedConfig` for more information.
+        It is used to instantiate an Encoder Decoder model according to the specified arguments, defining the encoder and decoder configs.
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig`
+        and can be used to control the model outputs.
+        See the documentation for :class:`~transformers.PretrainedConfig` for more information.
 
-    Args:
-        kwargs (`optional`):
-            Remaining dictionary of keyword arguments. Notably:
-                encoder (:class:`PretrainedConfig`, optional, defaults to `None`):
-                    An instance of a configuration object that defines the encoder config.
-                decoder (:class:`PretrainedConfig`, optional, defaults to `None`):
-                    An instance of a configuration object that defines the decoder config.
+        Args:
+            kwargs (`optional`):
+                Remaining dictionary of keyword arguments. Notably:
+                    encoder (:class:`PretrainedConfig`, optional, defaults to `None`):
+                        An instance of a configuration object that defines the encoder config.
+                    decoder (:class:`PretrainedConfig`, optional, defaults to `None`):
+                        An instance of a configuration object that defines the decoder config.
 
-    Example::
+        Example::
 
-        >>> from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel
+            >>> from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel
 
-        >>> # Initializing a BERT bert-base-uncased style configuration
-        >>> config_encoder = BertConfig()
-        >>> config_decoder = BertConfig()
+            >>> # Initializing a BERT bert-base-uncased style configuration
+            >>> config_encoder = BertConfig()
+            >>> config_decoder = BertConfig()
 
-        >>> config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)
+            >>> config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)
 
-        >>> # Initializing a Bert2Bert model from the bert-base-uncased style configurations
-        >>> model = EncoderDecoderModel(config=config)
+            >>> # Initializing a Bert2Bert model from the bert-base-uncased style configurations
+            >>> model = EncoderDecoderModel(config=config)
 
-        >>> # Accessing the model configuration
-        >>> config_encoder = model.config.encoder
-        >>> config_decoder  = model.config.decoder
-        >>> # set decoder config to causal lm
-        >>> config_decoder.is_decoder = True
-        >>> config_decoder.add_cross_attention = True
+            >>> # Accessing the model configuration
+            >>> config_encoder = model.config.encoder
+            >>> config_decoder  = model.config.decoder
+            >>> # set decoder config to causal lm
+            >>> config_decoder.is_decoder = True
+            >>> config_decoder.add_cross_attention = True
 
-        >>> # Saving the model, including its configuration
-        >>> model.save_pretrained('my-model')
+            >>> # Saving the model, including its configuration
+            >>> model.save_pretrained('my-model')
 
-        >>> # loading model and config from pretrained folder
-        >>> encoder_decoder_config = EncoderDecoderConfig.from_pretrained('my-model')
-        >>> model = EncoderDecoderModel.from_pretrained('my-model', config=encoder_decoder_config)
+            >>> # loading model and config from pretrained folder
+            >>> encoder_decoder_config = EncoderDecoderConfig.from_pretrained('my-model')
+            >>> model = EncoderDecoderModel.from_pretrained('my-model', config=encoder_decoder_config)
     """
     model_type = "encoder_decoder"