fix time plot

* add benchmark for all kinds of models

* improved import

* delete bogus files

* make style

.circleci/config.yml

@@ -73,7 +73,7 @@ jobs:
         steps:
             - checkout
             - run: sudo pip install .[tf,torch,docs]
-            - run: cd docs && make html
+            - run: cd docs && make html SPHINXOPTS="-W"
             - store_artifacts:
                 path: ./docs/_build
     deploy_doc:

.circleci/deploy.sh

@@ -17,7 +17,8 @@ function deploy_doc(){
 	fi
 }
 
-deploy_doc "master"
+# You can find the commit for each tag on https://github.com/huggingface/transformers/tags
+deploy_doc "master" master
 deploy_doc "b33a385" v1.0.0
 deploy_doc "fe02e45" v1.1.0
 deploy_doc "89fd345" v1.2.0
@@ -27,3 +28,11 @@ deploy_doc "3616209" v2.2.0
 deploy_doc "d0f8b9a" v2.3.0
 deploy_doc "6664ea9" v2.4.0
 deploy_doc "fb560dc" v2.5.0
+deploy_doc "b90745c" v2.5.1
+deploy_doc "fbc5bf1" v2.6.0
+deploy_doc "6f5a12a" v2.7.0
+deploy_doc "11c3257" v2.8.0
+deploy_doc "e7cfc1a" v2.9.0
+deploy_doc "7cb203f" v2.9.1
+deploy_doc "10d7239" v2.10.0 
+deploy_doc "b42586e" #v2.11.0 Latest stable release

README.md

@@ -59,7 +59,7 @@ Choose the right framework for every part of a model's lifetime
 | [Quick tour: Share your models ](#Quick-tour-of-model-sharing) | Upload and share your fine-tuned models with the community |
 | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers |
 | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers |
-| [Documentation][(v2.5.0)](https://huggingface.co/transformers/v2.5.0)[(v2.4.0/v2.4.1)](https://huggingface.co/transformers/v2.4.0)[(v2.3.0)](https://huggingface.co/transformers/v2.3.0)[(v2.2.0/v2.2.1/v2.2.2)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more |
+| Documentation [(master)](https://huggingface.co/transformers/master) [(stable)](https://huggingface.co/transformers/) [(v2.10.0)](https://huggingface.co/transformers/v2.10.0) [(v2.9.0/v2.9.1)](https://huggingface.co/transformers/v2.9.1) [(v2.8.0)](https://huggingface.co/transformers/v2.8.0) [(v2.7.0)](https://huggingface.co/transformers/v2.7.0) [(v2.6.0)](https://huggingface.co/transformers/v2.6.0) [(v2.5.0/v2.5.1)](https://huggingface.co/transformers/v2.5.1) [(v2.4.0/v2.4.1)](https://huggingface.co/transformers/v2.4.0)[(v2.3.0)](https://huggingface.co/transformers/v2.3.0)[(v2.2.0/v2.2.1/v2.2.2)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) | Full API documentation and more |
 
 ## Installation
 
@@ -169,7 +169,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training
 22. **[Other community models](https://huggingface.co/models)**, contributed by the [community](https://huggingface.co/users).
 23. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
 
-These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html).
+These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Pearson R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html).
 
 ## Online demo
 

docs/source/_static/css/huggingface.css

@@ -1,5 +1,45 @@
 /* Our DOM objects */
 
+/* Version control */
+
+.version-button {
+    background-color: #6670FF;
+    color: white;
+    border: none;
+    padding: 5px;
+    font-size: 15px;
+    cursor: pointer;
+}
+
+.version-button:hover, .version-button:focus {
+    background-color: #A6B0FF;
+}
+ 
+.version-dropdown {
+    display: none;
+    background-color: #6670FF;
+    min-width: 160px;
+    overflow: auto;
+    font-size: 15px;
+}
+  
+.version-dropdown a {
+    color: white;
+    padding: 3px 4px;
+    text-decoration: none;
+    display: block;
+}
+  
+.version-dropdown a:hover {
+    background-color: #A6B0FF;
+}
+  
+.version-show {
+    display: block;
+}
+
+/* Framework selector */
+
 .framework-selector {
     display: flex;
     flex-direction: row;
@@ -38,6 +78,7 @@
 
 /* The research field on top of the toc tree */
 .wy-side-nav-search{
+    padding-top: 0;
     background-color: #6670FF;
 }
 

docs/source/_static/js/custom.js

@@ -1,3 +1,26 @@
+// These two things need to be updated at each release for the version selector.
+// Last stable version
+const stableVersion = "v2.11.0"
+// Dictionary doc folder to label
+const versionMapping = {
+    "master": "master",
+    "": "v2.11.0 (stable)",
+    "v2.10.0": "v2.10.0",
+    "v2.9.1": "v2.9.0/v2.9.1",
+    "v2.8.0": "v2.8.0",
+    "v2.7.0": "v2.7.0",
+    "v2.6.0": "v2.6.0",
+    "v2.5.1": "v2.5.0/v2.5.1",
+    "v2.4.0": "v2.4.0/v2.4.1",
+    "v2.3.0": "v2.3.0",
+    "v2.2.0": "v2.2.0/v2.2.1/v2.2.2",
+    "v2.1.1": "v2.1.1",
+    "v2.0.0": "v2.0.0",
+    "v1.2.0": "v1.2.0",
+    "v1.1.0": "v1.1.0",
+    "v1.0.0": "v1.0.0"
+}
+
 function addIcon() {
     const huggingFaceLogo = "https://huggingface.co/landing/assets/transformers-docs/huggingface_logo.svg";
     const image = document.createElement("img");
@@ -58,6 +81,60 @@ function addGithubButton() {
     document.querySelector(".wy-side-nav-search .icon-home").insertAdjacentHTML('afterend', div);
 }
 
+function addVersionControl() {
+    // To grab the version currently in view, we parse the url
+    const parts = location.toString().split('/');
+    let versionIndex = parts.length - 2
+    // Main classes and models are nested so we need to go deeper
+    if (parts[versionIndex] == "main_classes" || parts[versionIndex] == "model_doc") {
+        versionIndex = parts.length - 3
+    } 
+    const version = parts[versionIndex];
+
+    // Menu with all the links,
+    const versionMenu = document.createElement("div");
+
+    const htmlLines = [];
+    for (const [key, value] of Object.entries(versionMapping)) {
+        var urlParts = (key == "") ? [] : [key];
+        urlParts = urlParts.concat(parts.slice(versionIndex));
+        htmlLines.push(`<a href="${urlParts.join('/')}">${value}</a>`);
+    }
+
+    versionMenu.classList.add("version-dropdown");
+    versionMenu.innerHTML = htmlLines.join('\n');
+    
+    // Button for version selection
+    const versionButton = document.createElement("div");
+    versionButton.classList.add("version-button");
+    let label = (version == "transformers") ? stableVersion : version
+    versionButton.innerText = label.concat(" ▼");
+
+    // Toggle the menu when we click on the button
+    versionButton.addEventListener("click", () => {
+        versionMenu.classList.toggle("version-show");
+    });
+
+    // Hide the menu when we click elsewhere
+    window.addEventListener("click", (event) => {
+        if (event.target != versionButton){
+            versionMenu.classList.remove('version-show');
+        }
+    });
+
+    // Container
+    const div = document.createElement("div");
+    div.appendChild(versionButton);
+    div.appendChild(versionMenu);
+    div.style.paddingTop = '25px';
+    div.style.backgroundColor = '#6670FF';
+    div.style.display = 'block';
+    div.style.textAlign = 'center';
+
+    const scrollDiv = document.querySelector(".wy-side-scroll");
+    scrollDiv.insertBefore(div, scrollDiv.children[1]);
+}
+
 function addHfMenu() {
     const div = `
     <div class="menu">
@@ -149,6 +226,7 @@ function parseGithubButtons (){"use strict";var e=window.document,t=e.location,o
 
 function onLoad() {
     addIcon();
+    addVersionControl();
     addCustomFooter();
     addGithubButton();
     parseGithubButtons();

docs/source/conf.py

@@ -187,8 +187,8 @@ epub_title = project
 epub_exclude_files = ['search.html']
 
 def setup(app):
-    app.add_stylesheet('css/huggingface.css')
-    app.add_stylesheet('css/code-snippets.css')
+    app.add_css_file('css/huggingface.css')
+    app.add_css_file('css/code-snippets.css')
     app.add_js_file('js/custom.js')
 
 # -- Extension configuration -------------------------------------------------

docs/source/glossary.rst

@@ -1,11 +1,41 @@
 Glossary
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^
+
+General terms
+-------------
+
+- autoencoding models: see MLM
+- autoregressive models: see CLM
+- CLM: causal language modeling, a pretraining task where the model reads the texts in order and has to predict the
+  next word. It's usually done by reading the whole sentence but using a mask inside the model to hide the future 
+  tokens at a certain timestep.
+- MLM: masked language modeling, a pretraining task where the model sees a corrupted version of the texts, usually done
+  by masking some tokens randomly, and has to predict the original text.
+- multimodal: a task taht combines texts with another kind of inputs (for instance images).
+- NLG: natural language generation, all tasks related to generating text ( for instance talk with transformers,
+  translation)
+- NLP: natural language processing, a generic way to say "deal with texts".
+- NLU: natural language understanding, all tasks related to understanding what is in a text (for instance classifying
+  the whole text, individual words)
+- pretrained model: a model that has been pretrained on some data (for instance all of Wikipedia). Pretraining methods
+  involve a self-supervised objective, which can be reading the text and trying to predict the next word (see CLM) or 
+  masking some words and trying to predict them (see MLM).
+- RNN: recurrent neural network, a type of model that uses a loop over a layer to process texts.
+- seq2seq or sequence-to-sequence: models that generate a new sequence from an input, like translation models, or
+  summarization models (such as :doc:`Bart </model_doc/bart>` or :doc:`T5 </model_doc/t5>`).
+- token: a part of a sentence, usually a word, but can also be a subword (non-common words are often split in subwords)
+  or a punctuation symbol.
+
+Model inputs
+------------
 
 Every model is different yet bears similarities with the others. Therefore most models use the same inputs, which are
 detailed here alongside usage examples.
 
+.. _input-ids:
+
 Input IDs
---------------------------
+~~~~~~~~~
 
 The input ids are often the only required parameters to be passed to the model as input. *They are token indices,
 numerical representations of tokens building the sequences that will be used as input by the model*.
@@ -24,24 +54,52 @@ The tokenizer takes care of splitting the sequence into tokens available in the
 
 ::
 
-    # Continuation of the previous script
     tokenized_sequence = tokenizer.tokenize(sequence)
-    assert tokenized_sequence == ['A', 'Titan', 'R', '##T', '##X', 'has', '24', '##GB', 'of', 'V', '##RA', '##M']
+    print(tokenized_sequence)
 
-These tokens can then be converted into IDs which are understandable by the model. Several methods are available for
-this, the recommended being `encode` or `encode_plus`, which leverage the Rust implementation of
+The tokens are either words or subwords. Here for instance, "VRAM" wasn't in the model vocabulary, so it's been split
+in "V", "RA" and "M". To indicate those tokens are not separate words but parts of the same word, a double-dash is
+added for "RA" and "M":
+
+::
+
+    ['A', 'Titan', 'R', '##T', '##X', 'has', '24', '##GB', 'of', 'V', '##RA', '##M']
+
+These tokens can then be converted into IDs which are understandable by the model. This can be done by directly feeding
+the sentence to the tokenizer, which leverages the Rust implementation of
 `huggingface/tokenizers <https://github.com/huggingface/tokenizers>`__ for peak performance.
 
 ::
 
-    # Continuation of the previous script
-    encoded_sequence = tokenizer.encode(sequence)
-    assert encoded_sequence == [101, 138, 18696, 155, 1942, 3190, 1144, 1572, 13745, 1104, 159, 9664, 2107, 102]
+    encoded_sequence = tokenizer(sequence)["input_ids"]
+    print(encoded_sequence)
 
-The `encode` and `encode_plus` methods automatically add "special tokens" which are special IDs the model uses.
+The tokenizer returns a dictionary with all the arguments necessary for its corresponding model to work properly. The
+token indices are under the key "input_ids":
+
+::
+
+    [101, 138, 18696, 155, 1942, 3190, 1144, 1572, 13745, 1104, 159, 9664, 2107, 102]
+
+Note that the tokenizer automatically adds "special tokens" (if the associated model rely on them) which are special
+IDs the model sometimes uses. If we decode the previous sequence of ids,
+
+::
+
+    tokenizer.decode(encoded_sequence)
+
+we will see 
+
+::
+
+    '[CLS] A Titan RTX has 24GB of VRAM [SEP]'
+
+because this is the way a :class:`~transformers.BertModel` is going to expect its inputs.
+
+.. _attention-mask:
 
 Attention mask
---------------------------
+~~~~~~~~~~~~~~
 
 The attention mask is an optional argument used when batching sequences together. This argument indicates to the
 model which tokens should be attended to, and which should not.
@@ -56,44 +114,55 @@ For example, consider these two sequences:
     sequence_a = "This is a short sequence."
     sequence_b = "This is a rather long sequence. It is at least longer than the sequence A."
 
-    encoded_sequence_a = tokenizer.encode(sequence_a)
-    assert len(encoded_sequence_a) == 8
+    encoded_sequence_a = tokenizer(sequence_a)["input_ids"]
+    encoded_sequence_b = tokenizer(sequence_b)["input_ids"]
+    
+    len(encoded_sequence_a), len(encoded_sequence_b)
 
-    encoded_sequence_b = tokenizer.encode(sequence_b)
-    assert len(encoded_sequence_b) == 19
-
-These two sequences have different lengths and therefore can't be put together in a same tensor as-is. The first
-sequence needs to be padded up to the length of the second one, or the second one needs to be truncated down to
-the length of the first one.
-
-In the first case, the list of IDs will be extended by the padding indices:
+The encoded versions have different lengths:
 
 ::
 
-    # Continuation of the previous script
-    padded_sequence_a = tokenizer.encode(sequence_a, max_length=19, pad_to_max_length=True)
+    (8, 19)
 
-    assert padded_sequence_a == [101, 1188, 1110, 170, 1603, 4954,  119, 102,    0,    0,    0,    0,    0,    0,    0,    0,   0,   0,   0]
-    assert encoded_sequence_b == [101, 1188, 1110, 170, 1897, 1263, 4954, 119, 1135, 1110, 1120, 1655, 2039, 1190, 1103, 4954, 138, 119, 102]
+Therefore, we can't be put then together in a same tensor as-is. The first sequence needs to be padded up to the length
+of the second one, or the second one needs to be truncated down to the length of the first one.
 
-These can then be converted into a tensor in PyTorch or TensorFlow. The attention mask is a binary tensor indicating
+In the first case, the list of IDs will be extended by the padding indices. We can pass a list to the tokenizer and ask
+it to pad like this:
+
+::
+
+    padded_sequences = tokenizer([sequence_a, sequence_b], padding=True)
+    padded_sequences["input_ids"]
+
+We can see that 0s have been added on the right of the first sentence to make it the same length as the second one:
+
+::
+
+    [[101, 1188, 1110, 170, 1603, 4954, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+     [101, 1188, 1110, 170, 1897, 1263, 4954, 119, 1135, 1110, 1120, 1655, 2039, 1190, 1103, 4954, 138, 119, 102]]
+
+This can then be converted into a tensor in PyTorch or TensorFlow. The attention mask is a binary tensor indicating
 the position of the padded indices so that the model does not attend to them. For the
 :class:`~transformers.BertTokenizer`, :obj:`1` indicate a value that should be attended to while :obj:`0` indicate
-a padded value.
-
-The method :func:`~transformers.PreTrainedTokenizer.encode_plus` may be used to obtain the attention mask directly:
+a padded value. This attention mask is in the dictionary returned by the tokenizer under the key "attention_mask":
 
 ::
 
-    # Continuation of the previous script
-    sequence_a_dict = tokenizer.encode_plus(sequence_a, max_length=19, pad_to_max_length=True)
+    padded_sequences["attention_mask"]
 
-    assert sequence_a_dict['input_ids'] == [101, 1188, 1110, 170, 1603, 4954, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
-    assert sequence_a_dict['attention_mask'] == [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+will give back
 
+::
+
+    [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
+
+.. _token-type-ids:
 
 Token Type IDs
---------------------------
+~~~~~~~~~~~~~~
 
 Some models' purpose is to do sequence classification or question answering. These require two different sequences to
 be encoded in the same input IDs. They are usually separated by special tokens, such as the classifier and separator
@@ -101,38 +170,51 @@ tokens. For example, the BERT model builds its two sequence input as such:
 
 ::
 
-    from transformers import BertTokenizer
-    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
+   # [CLS] SEQUENCE_A [SEP] SEQUENCE_B [SEP]
 
-    # [CLS] SEQ_A [SEP] SEQ_B [SEP]
-
-    sequence_a = "HuggingFace is based in NYC"
-    sequence_b = "Where is HuggingFace based?"
-
-    encoded_sequence = tokenizer.encode(sequence_a, sequence_b)
-    assert tokenizer.decode(encoded_sequence) == "[CLS] HuggingFace is based in NYC [SEP] Where is HuggingFace based? [SEP]"
-
-This is enough for some models to understand where one sequence ends and where another begins. However, other models
-such as BERT have an additional mechanism, which are the segment IDs. The Token Type IDs are a binary mask identifying
-the different sequences in the model.
-
-We can leverage :func:`~transformers.PreTrainedTokenizer.encode_plus` to output the Token Type IDs for us:
+We can use our tokenizer to automatically generate such a sentence by passing the two sequences as two arguments (and
+not a list like before) like this:
 
 ::
 
-    # Continuation of the previous script
-    encoded_dict = tokenizer.encode_plus(sequence_a, sequence_b)
+    from transformers import BertTokenizer
+    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
+    sequence_a = "HuggingFace is based in NYC"
+    sequence_b = "Where is HuggingFace based?"
 
-    assert encoded_dict['input_ids'] == [101, 20164, 10932, 2271, 7954, 1110, 1359, 1107, 17520, 102, 2777, 1110, 20164, 10932, 2271, 7954, 1359, 136, 102]
-    assert encoded_dict['token_type_ids'] == [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+    encoded_dict = tokenizer(sequence_a, sequence_b)
+    tokenizer.decode(encoded_dict["input_ids"])
+
+which will return:
+
+::
+
+    "[CLS] HuggingFace is based in NYC [SEP] Where is HuggingFace based? [SEP]"
+
+This is enough for some models to understand where one sequence ends and where another begins. However, other models
+such as BERT have an additional mechanism, which are the token type IDs (also called segment IDs). They are a binary
+mask identifying the different sequences in the model.
+
+The tokenizer returns in the dictionary under the key "token_type_ids":
+
+::
+
+    encoded_dict['token_type_ids']
+
+will return
+
+::
+
+    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]
 
 The first sequence, the "context" used for the question, has all its tokens represented by :obj:`0`, whereas the
 question has all its tokens represented by :obj:`1`. Some models, like :class:`~transformers.XLNetModel` use an
 additional token represented by a :obj:`2`.
 
+.. _position-ids:
 
 Position IDs
---------------------------
+~~~~~~~~~~~~
 
 The position IDs are used by the model to identify which token is at which position. Contrary to RNNs that have the
 position of each token embedded within them, transformers are unaware of the position of each token. The position
@@ -144,13 +226,24 @@ positional embeddings.
 Absolute positional embeddings are selected in the range ``[0, config.max_position_embeddings - 1]``. Some models
 use other types of positional embeddings, such as sinusoidal position embeddings or relative position embeddings.
 
+.. _feed-forward-chunking:
 
 Feed Forward Chunking
---------------------------
+~~~~~~~~~~~~~~~~~~~~~
 
-In transformers two feed forward layers usually follows the self attention layer in each residual attention block. The intermediate embedding size of the feed forward layers is often bigger than the hidden size of the model (*e.g.* for ``bert-base-uncased``). 
+In transformers two feed forward layers usually follows the self attention layer in each residual attention block.
+The intermediate embedding size of the feed forward layers is often bigger than the hidden size of the model (e.g.,
+for ``bert-base-uncased``). 
 
-For an input of size ``[batch_size, sequence_length]``, the memory required to store the intermediate feed forward embeddings ``[batch_size, sequence_length, config.intermediate_size]`` can account for a large fraction of the memory use. The authors of `Reformer: The Efficient Transformer <https://arxiv.org/abs/2001.04451>`_ noticed that since the computation is independent of the ``sequence_length`` dimension, it is mathematically equivalent to compute the output embeddings of both feed forward layers ``[batch_size, config.hidden_size]_0, ..., [batch_size, config.hidden_size]_n``  individually and concat them afterward to ``[batch_size, sequence_length, config.hidden_size]`` with ``n = sequence_length``, which trades increased computation time against reduced memory use, but yields a mathematically **equivalent** result.
+For an input of size ``[batch_size, sequence_length]``, the memory required to store the intermediate feed forward
+embeddings ``[batch_size, sequence_length, config.intermediate_size]`` can account for a large fraction of the memory
+use. The authors of `Reformer: The Efficient Transformer <https://arxiv.org/abs/2001.04451>`_ noticed that since the
+computation is independent of the ``sequence_length`` dimension, it is mathematically equivalent to compute the output
+embeddings of both feed forward layers ``[batch_size, config.hidden_size]_0, ..., [batch_size, config.hidden_size]_n`` 
+individually and concat them afterward to ``[batch_size, sequence_length, config.hidden_size]`` with
+``n = sequence_length``, which trades increased computation time against reduced memory use, but yields a
+mathematically **equivalent** result.
 
-For models employing the function :func:`~.transformers.apply_chunking_to_forward`, the ``chunk_size`` defines the number of output embeddings that are computed in parallel and thus defines the trade-off between memory and time complexity. 
-If ``chunk_size`` is set to 0, no feed forward chunking is done.
+For models employing the function :func:`~.transformers.apply_chunking_to_forward`, the ``chunk_size`` defines the
+number of output embeddings that are computed in parallel and thus defines the trade-off between memory and time
+complexity.  If ``chunk_size`` is set to 0, no feed forward chunking is done.

docs/source/index.rst

@@ -38,6 +38,16 @@ Choose the right framework for every part of a model's lifetime:
 Contents
 ---------------------------------
 
+The documentation is organized in five parts:
+
+- **GET STARTED** contains a quick tour, the installation instructions and some useful information about our philosophy
+  and a glossary.
+- **USING 🤗 TRANSFORMERS** contains general tutorials on how to use the library.
+- **ADVANCED GUIDES** contains more advanced guides that are more specific to a given script or part of the library.
+- **RESEARCH** focuses on tutorials that have less to do with how to use the library but more about general resarch in
+  transformers model
+- **PACKAGE REFERENCE** contains the documentation of each public class and function.
+
 The library currently contains PyTorch and Tensorflow implementations, pre-trained model weights, usage scripts and
 conversion utilities for the following models:
 
@@ -118,16 +128,17 @@ conversion utilities for the following models:
     :maxdepth: 2
     :caption: Get started
 
+    quicktour
     installation
-    quickstart
+    philosophy
     glossary
 
 .. toctree::
     :maxdepth: 2
-    :caption: Using Transformers
+    :caption: Using 🤗 Transformers
 
-    usage
-    summary
+    task_summary
+    model_summary
     serialization
     model_sharing
     multilingual
@@ -184,3 +195,4 @@ conversion utilities for the following models:
     model_doc/marian
     model_doc/longformer
     model_doc/retribert
+    model_doc/mobilebert

docs/source/main_classes/pipelines.rst

@@ -17,7 +17,7 @@ The pipeline abstraction
 The `pipeline` abstraction is a wrapper around all the other available pipelines. It is instantiated as any
 other pipeline but requires an additional argument which is the `task`.
 
-... autofunction:: transformers.pipeline
+.. autofunction:: transformers.pipeline
 
 
 The task specific pipelines

docs/source/migration.md

@@ -1,8 +1,8 @@
 # Migrating from previous packages
 
-## Migrating from pytorch-transformers to transformers
+## Migrating from pytorch-transformers to 🤗 Transformers
 
-Here is a quick summary of what you should take care of when migrating from `pytorch-transformers` to `transformers`.
+Here is a quick summary of what you should take care of when migrating from `pytorch-transformers` to 🤗 Transformers.
 
 ### Positional order of some models' keywords inputs (`attention_mask`, `token_type_ids`...) changed
 
@@ -14,17 +14,17 @@ If you used to call the models with positional inputs for keyword arguments, e.g
 
 ## Migrating from pytorch-pretrained-bert
 
-Here is a quick summary of what you should take care of when migrating from `pytorch-pretrained-bert` to `transformers`
+Here is a quick summary of what you should take care of when migrating from `pytorch-pretrained-bert` to 🤗 Transformers
 
 ### Models always output `tuples`
 
-The main breaking change when migrating from `pytorch-pretrained-bert` to `transformers` is that the models forward method always outputs a `tuple` with various elements depending on the model and the configuration parameters.
+The main breaking change when migrating from `pytorch-pretrained-bert` to 🤗 Transformers is that the models forward method always outputs a `tuple` with various elements depending on the model and the configuration parameters.
 
 The exact content of the tuples for each model are detailled in the models' docstrings and the [documentation](https://huggingface.co/transformers/).
 
 In pretty much every case, you will be fine by taking the first element of the output as the output you previously used in `pytorch-pretrained-bert`.
 
-Here is a `pytorch-pretrained-bert` to `transformers` conversion example for a `BertForSequenceClassification` classification model:
+Here is a `pytorch-pretrained-bert` to 🤗 Transformers conversion example for a `BertForSequenceClassification` classification model:
 
 ```python
 # Let's load our model
@@ -33,11 +33,11 @@ model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
 # If you used to have this line in pytorch-pretrained-bert:
 loss = model(input_ids, labels=labels)
 
-# Now just use this line in transformers to extract the loss from the output tuple:
+# Now just use this line in 🤗 Transformers to extract the loss from the output tuple:
 outputs = model(input_ids, labels=labels)
 loss = outputs[0]
 
-# In transformers you can also have access to the logits:
+# In 🤗 Transformers you can also have access to the logits:
 loss, logits = outputs[:2]
 
 # And even the attention weights if you configure the model to output them (and other outputs too, see the docstrings and documentation)
@@ -109,7 +109,7 @@ for batch in train_data:
     loss.backward()
     optimizer.step()
 
-### In Transformers, optimizer and schedules are splitted and instantiated like this:
+### In 🤗 Transformers, optimizer and schedules are splitted and instantiated like this:
 optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=False
 scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps)  # PyTorch scheduler
 ### and used like this:

docs/source/model_doc/auto.rst

@@ -1,11 +1,15 @@
 AutoModels
 -----------
 
-In many cases, the architecture you want to use can be guessed from the name or the path of the pretrained model you are supplying to the ``from_pretrained`` method.
+In many cases, the architecture you want to use can be guessed from the name or the path of the pretrained model you
+are supplying to the ``from_pretrained`` method.
 
-AutoClasses are here to do this job for you so that you automatically retrieve the relevant model given the name/path to the pretrained weights/config/vocabulary:
+AutoClasses are here to do this job for you so that you automatically retrieve the relevant model given the name/path
+to the pretrained weights/config/vocabulary:
 
-Instantiating one of ``AutoModel``, ``AutoConfig`` and ``AutoTokenizer`` will directly create a class of the relevant architecture (ex: ``model = AutoModel.from_pretrained('bert-base-cased')`` will create a instance of ``BertModel``).
+Instantiating one of ``AutoModel``, ``AutoConfig`` and ``AutoTokenizer`` will directly create a class of the relevant
+architecture (ex: ``model = AutoModel.from_pretrained('bert-base-cased')`` will create a instance of 
+:class:`~transformers.BertModel`).
 
 
 ``AutoConfig``
@@ -63,3 +67,43 @@ Instantiating one of ``AutoModel``, ``AutoConfig`` and ``AutoTokenizer`` will di
 .. autoclass:: transformers.AutoModelForTokenClassification
     :members:
 
+``TFAutoModel``
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFAutoModel
+    :members:
+
+
+``TFAutoModelForPreTraining``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFAutoModelForPreTraining
+    :members:
+
+
+``TFAutoModelWithLMHead``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFAutoModelWithLMHead
+    :members:
+
+
+``TFAutoModelForSequenceClassification``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFAutoModelForSequenceClassification
+    :members:
+
+
+``TFAutoModelForQuestionAnswering``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFAutoModelForQuestionAnswering
+    :members:
+
+
+``TFAutoModelForTokenClassification``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFAutoModelForTokenClassification
+    :members:

docs/source/model_doc/mobilebert.rst

@@ -0,0 +1,169 @@
+MobileBERT
+----------------------------------------------------
+
+Overview
+~~~~~~~~~~~~~~~~~~~~~
+
+The MobileBERT model was proposed in `MobileBERT: a Compact Task-Agnostic BERT
+for Resource-Limited Devices <https://arxiv.org/abs/2004.02984>`__
+by Zhiqing Sun, Hongkun Yu, Xiaodan Song, Renjie Liu, Yiming Yang, and Denny Zhou. It's a bidirectional transformer
+based on the BERT model, which is compressed and accelerated using several approaches.
+
+The abstract from the paper is the following:
+
+*Natural Language Processing (NLP) has recently achieved great success by using huge pre-trained models with hundreds
+of millions of parameters. However, these models suffer from heavy model sizes and high latency such that they cannot
+be deployed to resource-limited mobile devices. In this paper, we propose MobileBERT for compressing and accelerating
+the popular BERT model. Like the original BERT, MobileBERT is task-agnostic, that is, it can be generically applied
+to various downstream NLP tasks via simple fine-tuning. Basically, MobileBERT is a thin version of BERT_LARGE, while
+equipped with bottleneck structures and a carefully designed balance between self-attentions and feed-forward
+networks. To train MobileBERT, we first train a specially designed teacher model, an inverted-bottleneck incorporated
+BERT_LARGE model. Then, we conduct knowledge transfer from this teacher to MobileBERT. Empirical studies show that
+MobileBERT is 4.3x smaller and 5.5x faster than BERT_BASE while achieving competitive results on well-known
+benchmarks. On the natural language inference tasks of GLUE, MobileBERT achieves a GLUEscore o 77.7
+(0.6 lower than BERT_BASE), and 62 ms latency on a Pixel 4 phone. On the SQuAD v1.1/v2.0 question answering task,
+MobileBERT achieves a dev F1 score of 90.0/79.2 (1.5/2.1 higher than BERT_BASE).*
+
+Tips:
+
+- MobileBERT is a model with absolute position embeddings so it's usually advised to pad the inputs on
+  the right rather than the left.
+- MobileBERT is similar to BERT and therefore relies on the masked language modeling (MLM) objective.
+  It is therefore efficient at predicting masked tokens and at NLU in general, but is not optimal for
+  text generation. Models trained with a causal language modeling (CLM) objective are better in that regard.
+
+The original code can be found `here <https://github.com/google-research/mobilebert>`_.
+
+MobileBertConfig
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertConfig
+    :members:
+
+
+MobileBertTokenizer
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertTokenizer
+    :members: build_inputs_with_special_tokens, get_special_tokens_mask,
+        create_token_type_ids_from_sequences, save_vocabulary
+
+
+MobileBertTokenizerFast
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertTokenizerFast
+    :members:
+
+
+MobileBertModel
+~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertModel
+    :members:
+
+
+MobileBertForPreTraining
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertForPreTraining
+    :members:
+
+
+MobileBertForMaskedLM
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertForMaskedLM
+    :members:
+
+
+MobileBertForNextSentencePrediction
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertForNextSentencePrediction
+    :members:
+
+
+MobileBertForSequenceClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertForSequenceClassification
+    :members:
+
+
+MobileBertForMultipleChoice
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertForMultipleChoice
+    :members:
+
+
+MobileBertForTokenClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertForTokenClassification
+    :members:
+
+
+MobileBertForQuestionAnswering
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MobileBertForQuestionAnswering
+    :members:
+
+
+TFMobileBertModel
+~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFMobileBertModel
+    :members:
+
+
+TFMobileBertForPreTraining
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFMobileBertForPreTraining
+    :members:
+
+
+TFMobileBertForMaskedLM
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFMobileBertForMaskedLM
+    :members:
+
+
+TFMobileBertForNextSentencePrediction
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFMobileBertForNextSentencePrediction
+    :members:
+
+
+TFMobileBertForSequenceClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFMobileBertForSequenceClassification
+    :members:
+
+
+TFMobileBertForMultipleChoice
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFMobileBertForMultipleChoice
+    :members:
+
+
+TFMobileBertForTokenClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFMobileBertForTokenClassification
+    :members:
+
+
+TFMobileBertForQuestionAnswering
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFMobileBertForQuestionAnswering
+    :members:
+

docs/source/model_summary.rst

@@ -1,7 +1,7 @@
 Summary of the models
 ================================================
 
-This is a summary of the models available in the transformers library. It assumes you’re familiar with the original 
+This is a summary of the models available in 🤗 Transformers. It assumes you’re familiar with the original 
 `transformer model <https://arxiv.org/abs/1706.03762>`_. For a gentle introduction check the `annotated transformer 
 <http://nlp.seas.harvard.edu/2018/04/03/attention.html>`_. Here we focus on the high-level differences between the
 models. You can check them more in detail in their respective documentation. Also checkout the 

docs/source/philosophy.rst

@@ -0,0 +1,73 @@
+Philosophy
+==========
+
+🤗 Transformers is an opinionated library built for:
+
+- NLP researchers and educators seeking to use/study/extend large-scale transformers models
+- hands-on practitioners who want to fine-tune those models and/or serve them in production
+- engineers who just want to download a pretrained model and use it to solve a given NLP task.
+
+The library was designed with two strong goals in mind:
+
+- Be as easy and fast to use as possible:
+
+    - We strongly limited the number of user-facing abstractions to learn, in fact, there are almost no abstractions,
+      just three standard classes required to use each model: :doc:`configuration <main_classes/configuration>`, 
+      :doc:`models <main_classes/model>` and :doc:`tokenizer <main_classes/tokenizer>`.
+    - All of these classes can be initialized in a simple and unified way from pretrained instances by using a common
+      :obj:`from_pretrained()` instantiation method which will take care of downloading (if needed), caching and
+      loading the related class instance and associated data (configurations' hyper-parameters, tokenizers' vocabulary, 
+      and models' weights) from a pretrained checkpoint provided on 
+      `Hugging Face Hub <https://huggingface.co/models>`__ or your own saved checkpoint.
+    - On top of those three base classes, the library provides two APIs: :func:`~transformers.pipeline` for quickly
+      using a model (plus its associated tokenizer and configuration) on a given task and 
+      :func:`~transformers.Trainer`/:func:`~transformers.TFTrainer` to quickly train or fine-tune a given model.
+    - As a consequence, this library is NOT a modular toolbox of building blocks for neural nets. If you want to
+      extend/build-upon the library, just use regular Python/PyTorch/TensorFlow/Keras modules and inherit from the base
+      classes of the library to reuse functionalities like model loading/saving.
+
+- Provide state-of-the-art models with performances as close as possible to the original models:
+
+    - We provide at least one example for each architecture which reproduces a result provided by the official authors
+      of said architecture.
+    - The code is usually as close to the original code base as possible which means some PyTorch code may be not as
+      *pytorchic* as it could be as a result of being converted TensorFlow code and vice versa.
+
+A few other goals:
+
+- Expose the models' internals as consistently as possible:
+
+    - We give access, using a single API, to the full hidden-states and attention weights.
+    - Tokenizer and base model's API are standardized to easily switch between models.
+
+- Incorporate a subjective selection of promising tools for fine-tuning/investigating these models:
+
+    - A simple/consistent way to add new tokens to the vocabulary and embeddings for fine-tuning.
+    - Simple ways to mask and prune transformer heads.
+
+- Switch easily between PyTorch and TensorFlow 2.0, allowing training using one framwork and inference using another.
+
+Main concepts
+~~~~~~~~~~~~~
+
+The library is build around three types of classes for each model:
+
+- **Model classes**  such as :class:`~transformers.BertModel`, which are 30+ PyTorch models 
+  (`torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__) or Keras models 
+  (`tf.keras.Model <https://www.tensorflow.org/api_docs/python/tf/keras/Model>`__) that work with the pretrained
+  weights provided in the library.
+- **Configuration classes** such as :class:`~transformers.BertConfig`, which store all the parameters required to build
+  a model. You don't always need to instantiate these yourself. In particular, if you are using a pretrained model
+  without any modification, creating the model will automatically take care of instantiating the configuration (which
+  is part of the model).
+- **Tokenizer classes** such as :class:`~transformers.BertTokenizer`, which store the vocabulary for each model and
+  provide methods for encoding/decoding strings in a list of token embeddings indices to be fed to a model.
+
+All these classes can be instantiated from pretrained instances and saved locally using two methods:
+
+- :obj:`from_pretrained()` let you instantiate a model/configuration/tokenizer from a pretrained version either
+  provided by the library itself (the suported models are provided in the list :doc:`here <pretrained_models>`
+  or stored locally (or on a server) by the user,
+- :obj:`save_pretrained()` let you save a model/configuration/tokenizer locally so that it can be reloaded using
+  :obj:`from_pretrained()`.
+

docs/source/quickstart.md

@@ -1,222 +0,0 @@
-# Quickstart
-
-## Philosophy
-
-Transformers is an opinionated library built for NLP researchers seeking to use/study/extend large-scale transformers models.
-
-The library was designed with two strong goals in mind:
-
-- be as easy and fast to use as possible:
-
-  - we strongly limited the number of user-facing abstractions to learn, in fact, there are almost no abstractions, just three standard classes required to use each model: configuration, models and tokenizer,
-  - all of these classes can be initialized in a simple and unified way from pretrained instances by using a common `from_pretrained()` instantiation method which will take care of downloading (if needed), caching and loading the related class from a pretrained instance supplied in the library or your own saved instance.
-  - as a consequence, this library is NOT a modular toolbox of building blocks for neural nets. If you want to extend/build-upon the library, just use regular Python/PyTorch modules and inherit from the base classes of the library to reuse functionalities like model loading/saving.
-
-- provide state-of-the-art models with performances as close as possible to the original models:
-
-  - we provide at least one example for each architecture which reproduces a result provided by the official authors of said architecture,
-  - the code is usually as close to the original code base as possible which means some PyTorch code may be not as *pytorchic* as it could be as a result of being converted TensorFlow code.
-
-A few other goals:
-
-- expose the models' internals as consistently as possible:
-
-  - we give access, using a single API to the full hidden-states and attention weights,
-  - tokenizer and base model's API are standardized to easily switch between models.
-
-- incorporate a subjective selection of promising tools for fine-tuning/investigating these models:
-
-  - a simple/consistent way to add new tokens to the vocabulary and embeddings for fine-tuning,
-  - simple ways to mask and prune transformer heads.
-
-## Main concepts
-
-The library is build around three types of classes for each model:
-
-- **model classes**  e.g., `BertModel` which are 20+ PyTorch models (`torch.nn.Modules`) that work with the pretrained weights provided in the library. In TF2, these are `tf.keras.Model`.
-- **configuration classes** which store all the parameters required to build a model, e.g., `BertConfig`. You don't always need to instantiate these your-self. In particular, if you are using a pretrained model without any modification, creating the model will automatically take care of instantiating the configuration (which is part of the model)
-- **tokenizer classes** which store the vocabulary for each model and provide methods for encoding/decoding strings in a list of token embeddings indices to be fed to a model, e.g., `BertTokenizer`
-
-All these classes can be instantiated from pretrained instances and saved locally using two methods:
-
-- `from_pretrained()` let you instantiate a model/configuration/tokenizer from a pretrained version either provided by the library itself (currently 27 models are provided as listed [here](https://huggingface.co/transformers/pretrained_models.html)) or stored locally (or on a server) by the user,
-- `save_pretrained()` let you save a model/configuration/tokenizer locally so that it can be reloaded using `from_pretrained()`.
-
-We'll finish this quickstart tour by going through a few simple quick-start examples to see how we can instantiate and use these classes. The rest of the documentation is organized into two parts:
-
-- the **MAIN CLASSES** section details the common functionalities/method/attributes of the three main type of classes (configuration, model, tokenizer) plus some optimization related classes provided as utilities for training,
-- the **PACKAGE REFERENCE** section details all the variants of each class for each model architectures and, in particular, the input/output that you should expect when calling each of them.
-
-## Quick tour: Usage
-
-Here are two examples showcasing a few `Bert` and `GPT2` classes and pre-trained models.
-
-See the full API reference for examples of each model class.
-
-### BERT example
-
-Let's start by preparing a tokenized input (a list of token embeddings indices to be fed to Bert) from a text string using `BertTokenizer`
-
-```python
-import torch
-from transformers import BertTokenizer, BertModel, BertForMaskedLM
-
-# OPTIONAL: if you want to have more information on what's happening under the hood, activate the logger as follows
-import logging
-logging.basicConfig(level=logging.INFO)
-
-# Load pre-trained model tokenizer (vocabulary)
-tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-
-# Tokenize input
-text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
-tokenized_text = tokenizer.tokenize(text)
-
-# Mask a token that we will try to predict back with `BertForMaskedLM`
-masked_index = 8
-tokenized_text[masked_index] = '[MASK]'
-assert tokenized_text == ['[CLS]', 'who', 'was', 'jim', 'henson', '?', '[SEP]', 'jim', '[MASK]', 'was', 'a', 'puppet', '##eer', '[SEP]']
-
-# Convert token to vocabulary indices
-indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
-# Define sentence A and B indices associated to 1st and 2nd sentences (see paper)
-segments_ids = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]
-
-# Convert inputs to PyTorch tensors
-tokens_tensor = torch.tensor([indexed_tokens])
-segments_tensors = torch.tensor([segments_ids])
-```
-
-Let's see how we can use `BertModel` to encode our inputs in hidden-states:
-
-```python
-# Load pre-trained model (weights)
-model = BertModel.from_pretrained('bert-base-uncased')
-
-# Set the model in evaluation mode to deactivate the DropOut modules
-# This is IMPORTANT to have reproducible results during evaluation!
-model.eval()
-
-# If you have a GPU, put everything on cuda
-tokens_tensor = tokens_tensor.to('cuda')
-segments_tensors = segments_tensors.to('cuda')
-model.to('cuda')
-
-# Predict hidden states features for each layer
-with torch.no_grad():
-    # See the models docstrings for the detail of the inputs
-    outputs = model(tokens_tensor, token_type_ids=segments_tensors)
-    # Transformers models always output tuples.
-    # See the models docstrings for the detail of all the outputs
-    # In our case, the first element is the hidden state of the last layer of the Bert model
-    encoded_layers = outputs[0]
-# We have encoded our input sequence in a FloatTensor of shape (batch size, sequence length, model hidden dimension)
-assert tuple(encoded_layers.shape) == (1, len(indexed_tokens), model.config.hidden_size)
-```
-
-And how to use `BertForMaskedLM` to predict a masked token:
-
-```python
-# Load pre-trained model (weights)
-model = BertForMaskedLM.from_pretrained('bert-base-uncased')
-model.eval()
-
-# If you have a GPU, put everything on cuda
-tokens_tensor = tokens_tensor.to('cuda')
-segments_tensors = segments_tensors.to('cuda')
-model.to('cuda')
-
-# Predict all tokens
-with torch.no_grad():
-    outputs = model(tokens_tensor, token_type_ids=segments_tensors)
-    predictions = outputs[0]
-
-# confirm we were able to predict 'henson'
-predicted_index = torch.argmax(predictions[0, masked_index]).item()
-predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]
-assert predicted_token == 'henson'
-```
-
-### OpenAI GPT-2
-
-Here is a quick-start example using `GPT2Tokenizer` and `GPT2LMHeadModel` class with OpenAI's pre-trained model to predict the next token from a text prompt.
-
-First let's prepare a tokenized input from our text string using `GPT2Tokenizer`
-
-```python
-import torch
-from transformers import GPT2Tokenizer, GPT2LMHeadModel
-
-# OPTIONAL: if you want to have more information on what's happening, activate the logger as follows
-import logging
-logging.basicConfig(level=logging.INFO)
-
-# Load pre-trained model tokenizer (vocabulary)
-tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
-
-# Encode a text inputs
-text = "Who was Jim Henson ? Jim Henson was a"
-indexed_tokens = tokenizer.encode(text)
-
-# Convert indexed tokens in a PyTorch tensor
-tokens_tensor = torch.tensor([indexed_tokens])
-```
-
-Let's see how to use `GPT2LMHeadModel` to generate the next token following our text:
-
-```python
-# Load pre-trained model (weights)
-model = GPT2LMHeadModel.from_pretrained('gpt2')
-
-# Set the model in evaluation mode to deactivate the DropOut modules
-# This is IMPORTANT to have reproducible results during evaluation!
-model.eval()
-
-# If you have a GPU, put everything on cuda
-tokens_tensor = tokens_tensor.to('cuda')
-model.to('cuda')
-
-# Predict all tokens
-with torch.no_grad():
-    outputs = model(tokens_tensor)
-    predictions = outputs[0]
-
-# get the predicted next sub-word (in our case, the word 'man')
-predicted_index = torch.argmax(predictions[0, -1, :]).item()
-predicted_text = tokenizer.decode(indexed_tokens + [predicted_index])
-assert predicted_text == 'Who was Jim Henson? Jim Henson was a man'
-```
-
-Examples for each model class of each model architecture (Bert, GPT, GPT-2, Transformer-XL, XLNet and XLM) can be found in the [documentation](#documentation).
-
-#### Using the past
-
-GPT-2, as well as some other models (GPT, XLNet, Transfo-XL, CTRL), make use of a `past` or `mems` attribute which can be used to prevent re-computing the key/value pairs when using sequential decoding. It is useful when generating sequences as a big part of the attention mechanism benefits from previous computations.
-
-Here is a fully-working example using the `past` with `GPT2LMHeadModel` and argmax decoding (which should only be used as an example, as argmax decoding introduces a lot of repetition):
-
-```python
-from transformers import GPT2LMHeadModel, GPT2Tokenizer
-import torch
-
-tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
-model = GPT2LMHeadModel.from_pretrained('gpt2')
-
-generated = tokenizer.encode("The Manhattan bridge")
-context = torch.tensor([generated])
-past = None
-
-for i in range(100):
-    print(i)
-    output, past = model(context, past=past)
-    token = torch.argmax(output[..., -1, :])
-
-    generated += [token.tolist()]
-    context = token.unsqueeze(0)
-
-sequence = tokenizer.decode(generated)
-
-print(sequence)
-```
-
-The model only requires a single token as input as all the previous tokens' key/value pairs are contained in the `past`.

docs/source/quicktour.rst

@@ -0,0 +1,378 @@
+Quick tour
+==========
+
+Let's have a quick look at the 🤗 Transformers library features. The library downloads pretrained models for
+Natural Language Understanding (NLU) tasks, such as analyzing the sentiment of a text, and Natural Language Generation (NLG),
+such as completing a prompt with new text or translating in another language.
+
+First we will see how to easily leverage the pipeline API to quickly use those pretrained models at inference. Then, we
+will dig a little bit more and see how the library gives you access to those models and helps you preprocess your data.
+
+.. note::
+
+    All code examples presented in the documentation have a switch on the top left for Pytorch versus TensorFlow. If
+    not, the code is expected to work for both backends without any change needed.
+
+Getting started on a task with a pipeline
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The easiest way to use a pretrained model on a given task is to use :func:`~transformers.pipeline`. 🤗 Transformers
+provides the following tasks out of the box:
+
+- Sentiment analysis: is a text positive or negative?
+- Text generation (in English): provide a prompt and the model will generate what follows.
+- Name entity recognition (NER): in an input sentence, label each word with the entity it represents (person, place,
+  etc.)
+- Question answering: provide the model with some context and a question, extract the answer from the context.
+- Filling masked text: given a text with masked words (e.g., replaced by ``[MASK]``), fill the blanks.
+- Summarization: generate a summary of a long text.
+- Translation: translate a text in another language.
+- Feature extraction: return a tensor representation of the text.
+
+Let's see how this work for sentiment analysis (the other tasks are all covered in the
+:doc:`task summary </task_summary>`):
+
+::
+
+    from transformers import pipeline
+    classifier = pipeline('sentiment-analysis')
+
+When typing this command for the first time, a pretrained model and its tokenizer are downloaded and cached. We will
+look at both later on, but as an introduction the tokenizer's job is to preprocess the text for the model, which is
+then responsible for making predictions. The pipeline groups all of that together, and post-process the predictions to
+make them readable. For instance
+
+::
+
+    classifier('We are very happy to show you the 🤗 Transformers library.')
+
+will return something like this:
+
+::
+
+    [{'label': 'POSITIVE', 'score': 0.9997795224189758}]
+
+That's encouraging! You can use it on a list of sentences, which will be preprocessed then fed to the model as a
+`batch`:
+
+::
+
+    classifier(["We are very happy to show you the 🤗 Transformers library.",
+                "We hope you don't hate it."])
+
+returning a list of dictionaries like this one:
+
+::
+
+    [{'label': 'POSITIVE', 'score': 0.9997795224189758},
+     {'label': 'NEGATIVE', 'score': 0.5308589935302734}]
+
+You can see the second sentence has been classified as negative (it needs to be positive or negative) but its score is
+fairly neutral.
+
+By default, the model downloaded for this pipeline is called "distilbert-base-uncased-finetuned-sst-2-english". We can
+look at its `model page <https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english>`__ to get more
+information about it. It uses the :doc:`DistilBERT architecture </model_doc/distilbert>` and has been fine-tuned on a
+dataset called SST-2 for the sentiment analysis task.
+
+Let's say we want to use another model; for instance, one that has been trained on French data. We can search through
+the `model hub <https://huggingface.co/models>`__ that gathers models pretrained on a lot of data by research labs, but
+also community models (usually fine-tuned versions of those big models on a specific dataset). Applying the tags
+"French" and "text-classification" gives back a suggestion "nlptown/bert-base-multilingual-uncased-sentiment". Let's
+see how we can use it. 
+
+You can directly pass the name of the model to use to :func:`~transformers.pipeline`:
+
+::
+
+    classifier = pipeline('sentiment-analysis', model="nlptown/bert-base-multilingual-uncased-sentiment")
+
+This classifier can now deal with texts in English, French, but also Dutch, German, Italian and Spanish! You can also
+replace that name by a local folder where you have saved a pretrained model (see below). You can also pass a model
+object and its associated tokenizer.
+
+We will need two classes for this. The first is :class:`~transformers.AutoTokenizer`, which we will use to download the
+tokenizer associated to the model we picked and instantiate it. The second is
+:class:`~transformers.AutoModelForSequenceClassification` (or
+:class:`~transformers.TFAutoModelForSequenceClassification` if you are using TensorFlow), which we will use to download
+the model itself. Note that if we were using the library on an other task, the class of the model would change. The
+:doc:`task summary </task_summary>` tutorial summarizes which class is used for which task.
+
+::
+
+    ## PYTORCH CODE
+    from transformers import AutoTokenizer, AutoModelForSequenceClassification
+    ## TENSORFLOW CODE
+    from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
+
+Now, to download the models and tokenizer we found previously, we just have to use the 
+:func:`~transformers.AutoModelForSequenceClassification.from_pretrained` method (feel free to replace ``model_name`` by
+any other model from the model hub):
+
+::
+
+    ## PYTORCH CODE
+    model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
+    model = AutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    pipe = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
+    ## TENSORFLOW CODE
+    model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
+    model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    classifier = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
+
+If you don't find a model that has been pretrained on some data similar to yours, you will need to fine-tune a
+pretrained model on your data. We provide :doc:`example scripts </examples>` to do so. Once you're done, don't forget
+to share your fine-tuned model on the hub with the community, using :doc:`this tutorial </model_sharing>`.
+
+.. _pretrained-model:
+
+Under the hood: pretrained models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Let's now see what happens beneath the hood when using those pipelines. As we saw, the model and tokenizer are created
+using the :obj:`from_pretrained` method:
+
+::
+
+    ## PYTORCH CODE
+    from transformers import AutoTokenizer, AutoModelForSequenceClassification
+    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    model = AutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    ## TENSORFLOW CODE
+    from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
+    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+Using the tokenizer
+^^^^^^^^^^^^^^^^^^^
+
+We mentioned the tokenizer is responsible for the preprocessing of your texts. First, it will split a given text in
+words (or part of words, punctuation symbols, etc.) usually called `tokens`. There are multiple rules that can govern
+that process, which is why we need to instantiate the tokenizer using the name of the model, to make sure we use the
+same rules as when the model was pretrained.
+
+The second step is to convert those `tokens` into numbers, to be able to build a tensor out of them and feed them to
+the model. To do this, the tokenizer has a `vocab`, which is the part we download when we instantiate it with the
+:obj:`from_pretrained` method, since we need to use the same `vocab` as when the model was pretrained.
+
+To apply these steps on a given text, we can just feed it to our tokenizer:
+
+::
+
+    input = tokenizer("We are very happy to show you the 🤗 Transformers library.")
+    print(input)
+
+This returns a dictionary string to list of ints. It contains the `ids of the tokens <glossary.html#input-ids>`__,
+as mentioned before, but also additional arguments that will be useful to the model. Here for instance, we also have an
+`attention mask <glossary.html#attention-mask>`__ that the model will use to have a better understanding of the sequence:
+
+
+::
+    {'input_ids': [101, 2057, 2024, 2200, 3407, 2000, 2265, 2017, 1996, 100, 19081, 3075, 1012, 102],
+     'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
+
+You can pass a list of sentences directly to your tokenizer. If your goal is to send them through your model as a
+batch, you probably want to pad them all to the same length, truncate them to the maximum length the model can accept
+and get tensors back. You can specify all of that to the tokenizer:
+
+::
+
+    ## PYTORCH CODE
+    batch = tokenizer(
+        ["We are very happy to show you the 🤗 Transformers library.",
+         "We hope you don't hate it."],
+        padding=True, truncation=True, return_tensors="pt")
+    print(batch)
+    ## TENSORFLOW CODE
+    batch = tokenizer(
+        ["We are very happy to show you the 🤗 Transformers library.",
+         "We hope you don't hate it."],
+        padding=True, truncation=True, return_tensors="tf")
+    print(batch)
+
+The padding is automatically applied on the side the model expect it (in this case, on the right), with the
+padding token the model was pretrained with. The attention mask is also adapted to take the padding into account:
+
+::
+
+    {'input_ids': tensor([[  101,  2057,  2024,  2200,  3407,  2000,  2265,  2017,  1996,   100, 19081,  3075,  1012,   102],
+                          [  101,  2057,  3246,  2017,  2123,  1005,  1056,  5223,  2009,  1012,   102,     0,     0,     0]]), 
+     'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+                               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]])}
+
+You can learn more about tokenizers on their :doc:`doc page <main_classes/tokenizer>` (tutorial coming soon).
+
+Using the model
+^^^^^^^^^^^^^^^
+
+Once your input has been preprocessed by the tokenizer, you can directly send it to the model. As we mentioned, it will
+contain all the relevant information the model needs. If you're using a TensorFlow model, you can directly pass the
+dictionary keys to tensor, for a PyTorch model, you need to unpack the dictionary by adding :obj:`**`.
+
+::
+
+    ## PYTORCH CODE
+    outputs = model(**batch)
+    ## TENSORFLOW CODE
+    outputs = model(batch)
+
+In 🤗 Transformers, all outputs are tuples (with only one element potentially). Here, we get a tuple with just the
+final activations of the model.
+
+::
+
+    (tensor([[-4.1329,  4.3811],
+             [ 0.0818, -0.0418]]),)
+
+.. note::
+
+    All 🤗 Transformers models (PyTorch or TensorFlow) return the activations of the model *before* the final
+    activation function (like SoftMax) since this final activation function is often fused with the loss.
+
+Let's apply the SoftMax activation to get predictions.
+
+::
+
+    ## PYTORCH CODE
+    import torch.nn.functional as F
+    predictions = F.softmax(outputs[0], dim=-1)
+    print(predictions)
+    ## TENSORFLOW CODE
+    predictions = tf.nn.softmax(outputs[0], axis=-1)
+    print(predictions)
+
+We can see we get the numbers from before:
+
+::
+
+    tensor([[2.0060e-04, 9.9980e-01],
+            [5.3086e-01, 4.6914e-01]])
+
+If you have labels, you can provide them to the model, it will return a tuple with the loss and the final activations.
+
+::
+
+    ## PYTORCH CODE
+    import torch
+    outputs = model(**batch, labels = torch.tensor([1, 0])
+    ## TENSORFLOW CODE
+    import tensorflow as tf
+    outputs = model(batch, labels = tf.constant([1, 0])
+
+Models are standard `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__ or
+`tf.keras.Model <https://www.tensorflow.org/api_docs/python/tf/keras/Model>`__ so you can use them in your usual
+training loop. 🤗 Transformers also provides a :class:`~transformers.Trainer` (or :class:`~transformers.TFTrainer` if
+you are using TensorFlow) class to help with your training (taking care of things such as distributed training, mixed
+precision, etc.). See the training tutorial (coming soon) for more details.
+
+Once your model is fine-tuned, you can save it with its tokenizer the following way:
+
+::
+
+    tokenizer.save_pretrained(save_directory)
+    model.save_pretrained(save_directory)
+
+You can then load this model back using the :func:`~transformers.AutoModel.from_pretrained` method by passing the
+directory name instead of the model name. One cool feature of 🤗 Transformers is that you can easily switch between
+PyTorch and TensorFlow: any model saved as before can be loaded back either in PyTorch or TensorFlow. If you are
+loading a saved PyTorch model in a TensorFlow model, use :func:`~transformers.TFAutoModel.from_pretrained` like this:
+
+::
+
+    tokenizer = AutoTokenizer.from_pretrained(save_directory)
+    model = TFAutoModel.from_pretrained(save_directory, from_pt=True)
+
+and if you are loading a saved TensorFlow model in a PyTorch model, you should use the following code:
+
+::
+
+    tokenizer = AutoTokenizer.from_pretrained(save_directory)
+    model = AutoModel.from_pretrained(save_directory, from_tf=True)
+
+Lastly, you can also ask the model to return all hidden states and all attention weights if you need them:
+
+
+::
+
+    ## PYTORCH CODE
+    outputs = model(**batch, output_hidden_states=True, output_attentions=True)
+    all_hidden_states, all_attentions = outputs[-2:]
+    ## TENSORFLOW CODE
+    outputs = model(batch, output_hidden_states=True, output_attentions=True)
+    all_hidden_states, all_attentions = outputs[-2:]
+
+Accessing the code
+^^^^^^^^^^^^^^^^^^
+
+The :obj:`AutoModel` and :obj:`AutoTokenizer` classes are just shortcuts that will automatically work with any
+pretrained model. Behind the scenes, the library has one model class per combination of architecture plus class, so the
+code is easy to access and tweak if you need to.
+
+In our previous example, the model was called "distilbert-base-uncased-finetuned-sst-2-english", which means it's
+using the :doc:`DistilBERT </model_doc/distilbert>` architecture. The model automatically created is then a
+:class:`~transformers.DistilBertForSequenceClassification`. You can look at its documentation for all details relevant
+to that specific model, or browse the source code. This is how you would directly instantiate model and tokenizer
+without the auto magic:
+
+::
+
+    ## PYTORCH CODE
+    from transformers import DistilBertTokenizer, DistilBertForSequenceClassification
+    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    model = DistilBertForSequenceClassification.from_pretrained(model_name)
+    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+    ## TENSORFLOW CODE
+    from transformers import DistilBertTokenizer, TFDistilBertForSequenceClassification
+    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    model = TFDistilBertForSequenceClassification.from_pretrained(model_name)
+    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+
+Customizing the model
+^^^^^^^^^^^^^^^^^^^^^
+
+If you want to change how the model itself is built, you can define your custom configuration class. Each architecture
+comes with its own relevant configuration (in the case of DistilBERT, :class:`~transformers.DistilBertConfig`) which
+allows you to specify any of the hidden dimension, dropout rate etc. If you do core modifications, like changing the
+hidden size, you won't be able to use a pretrained model anymore and will need to train from scratch. You would then
+instantiate the model directly from this configuration.
+
+Here we use the predefined vocabulary of DistilBERT (hence load the tokenizer with the
+:func:`~transformers.DistilBertTokenizer.from_pretrained` method) and initialize the model from scratch (hence
+instantiate the model from the configuration instead of using the
+:func:`~transformers.DistilBertForSequenceClassification.from_pretrained` method).
+
+::
+
+    ## PYTORCH CODE
+    from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
+    config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
+    tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+    model = DistilBertForSequenceClassification(config)
+    ## TENSORFLOW CODE
+    from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
+    config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
+    tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+    model = TFDistilBertForSequenceClassification(config)
+
+For something that only changes the head of the model (for instance, the number of labels), you can still use a
+pretrained model for the body. For instance, let's define a classifier for 10 different labels using a pretrained body.
+We could create a configuration with all the default values and just change the number of labels, but more easily, you
+can directly pass any argument a configuration would take to the :func:`from_pretrained` method and it will update the
+default configuration with it:
+
+::
+
+    ## PYTORCH CODE
+    from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
+    model_name = "distilbert-base-uncased"
+    model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
+    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+    ## TENSORFLOW CODE
+    from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
+    model_name = "distilbert-base-uncased"
+    model = TFDistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
+    tokenizer = DistilBertTokenizer.from_pretrained(model_name)

docs/source/task_summary.rst

@@ -1,4 +1,4 @@
-Usage
+Summary of the tasks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 This page shows the most frequent use-cases when using the library. The models available allow for many different
@@ -217,9 +217,9 @@ Here is an example of question answering using a model and a tokenizer. The proc
     """
 
     questions = [
-        "How many pretrained models are available in Transformers?",
-        "What does Transformers provide?",
-        "Transformers provides interoperability between which frameworks?",
+        "How many pretrained models are available in 🤗 Transformers?",
+        "What does 🤗 Transformers provide?",
+        "🤗 Transformers provides interoperability between which frameworks?",
     ]
 
     for question in questions:
@@ -253,9 +253,9 @@ Here is an example of question answering using a model and a tokenizer. The proc
     """
 
     questions = [
-        "How many pretrained models are available in Transformers?",
-        "What does Transformers provide?",
-        "Transformers provides interoperability between which frameworks?",
+        "How many pretrained models are available in 🤗 Transformers?",
+        "What does 🤗 Transformers provide?",
+        "🤗 Transformers provides interoperability between which frameworks?",
     ]
 
     for question in questions:
@@ -280,13 +280,13 @@ This outputs the questions followed by the predicted answers:
 
 ::
 
-    Question: How many pretrained models are available in Transformers?
+    Question: How many pretrained models are available in 🤗 Transformers?
     Answer: over 32 +
 
-    Question: What does Transformers provide?
+    Question: What does 🤗 Transformers provide?
     Answer: general - purpose architectures
 
-    Question: Transformers provides interoperability between which frameworks?
+    Question: 🤗 Transformers provides interoperability between which frameworks?
     Answer: tensorflow 2 . 0 and pytorch
 
 

docs/source/torchscript.rst

@@ -12,7 +12,7 @@ According to Pytorch's documentation: "TorchScript is a way to create serializab
 Pytorch's two modules `JIT and TRACE <https://pytorch.org/docs/stable/jit.html>`_ allow the developer to export
 their model to be re-used in other programs, such as efficiency-oriented C++ programs.
 
-We have provided an interface that allows the export of `transformers` models to TorchScript so that they can
+We have provided an interface that allows the export of 🤗 Transformers models to TorchScript so that they can
 be reused in a different environment than a Pytorch-based python program. Here we explain how to use our models so that
 they can be exported, and what to be mindful of when using these models with TorchScript.
 

examples/README.md

@@ -1,6 +1,6 @@
 ## Examples
 
-Version 2.9 of `transformers` introduces a new [`Trainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py) class for PyTorch, and its equivalent [`TFTrainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer_tf.py) for TF 2.
+Version 2.9 of 🤗 Transformers introduces a new [`Trainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py) class for PyTorch, and its equivalent [`TFTrainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer_tf.py) for TF 2.
 Running the examples requires PyTorch 1.3.1+ or TensorFlow 2.0+.
 
 Here is the list of all our examples:

examples/benchmarking/plot_csv_file.py

@@ -1,14 +1,19 @@
 import csv
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import List, Optional
 
 import matplotlib.pyplot as plt
 import numpy as np
+from matplotlib.ticker import ScalarFormatter
 
 from transformers import HfArgumentParser
 
 
+def list_field(default=None, metadata=None):
+    return field(default_factory=lambda: default, metadata=metadata)
+
+
 @dataclass
 class PlotArguments:
     """
@@ -24,6 +29,9 @@ class PlotArguments:
         default=False,
         metadata={"help": "Whether the csv file has time results or memory results. Defaults to memory results."},
     )
+    no_log_scale: bool = field(
+        default=False, metadata={"help": "Disable logarithmic scale when plotting"},
+    )
     is_train: bool = field(
         default=False,
         metadata={
@@ -33,6 +41,25 @@ class PlotArguments:
     figure_png_file: Optional[str] = field(
         default=None, metadata={"help": "Filename under which the plot will be saved. If unused no plot is saved."},
     )
+    short_model_names: Optional[List[str]] = list_field(
+        default=None, metadata={"help": "List of model names that are used instead of the ones in the csv file."}
+    )
+
+
+def can_convert_to_int(string):
+    try:
+        int(string)
+        return True
+    except ValueError:
+        return False
+
+
+def can_convert_to_float(string):
+    try:
+        float(string)
+        return True
+    except ValueError:
+        return False
 
 
 class Plot:
@@ -46,16 +73,31 @@ class Plot:
                 model_name = row["model"]
                 self.result_dict[model_name]["bsz"].append(int(row["batch_size"]))
                 self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
-                self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = row[
-                    "result"
-                ]
+                if can_convert_to_int(row["result"]):
+                    # value is not None
+                    self.result_dict[model_name]["result"][
+                        (int(row["batch_size"]), int(row["sequence_length"]))
+                    ] = int(row["result"])
+                elif can_convert_to_float(row["result"]):
+                    # value is not None
+                    self.result_dict[model_name]["result"][
+                        (int(row["batch_size"]), int(row["sequence_length"]))
+                    ] = float(row["result"])
 
     def plot(self):
         fig, ax = plt.subplots()
         title_str = "Time usage" if self.args.is_time else "Memory usage"
         title_str = title_str + " for training" if self.args.is_train else title_str + " for inference"
 
-        for model_name in self.result_dict.keys():
+        if not self.args.no_log_scale:
+            # set logarithm scales
+            ax.set_xscale("log")
+            ax.set_yscale("log")
+
+        for axis in [ax.xaxis, ax.yaxis]:
+            axis.set_major_formatter(ScalarFormatter())
+
+        for model_name_idx, model_name in enumerate(self.result_dict.keys()):
             batch_sizes = sorted(list(set(self.result_dict[model_name]["bsz"])))
             sequence_lengths = sorted(list(set(self.result_dict[model_name]["seq_len"])))
             results = self.result_dict[model_name]["result"]
@@ -64,28 +106,33 @@ class Plot:
                 (batch_sizes, sequence_lengths) if self.args.plot_along_batch else (sequence_lengths, batch_sizes)
             )
 
-            plt.xlim(min(x_axis_array), max(x_axis_array))
+            label_model_name = (
+                model_name if self.args.short_model_names is None else self.args.short_model_names[model_name_idx]
+            )
 
             for inner_loop_value in inner_loop_array:
                 if self.args.plot_along_batch:
-                    y_axis_array = np.asarray([results[(x, inner_loop_value)] for x in x_axis_array], dtype=np.int)
+                    y_axis_array = np.asarray(
+                        [results[(x, inner_loop_value)] for x in x_axis_array if (x, inner_loop_value) in results],
+                        dtype=np.int,
+                    )
                 else:
-                    y_axis_array = np.asarray([results[(inner_loop_value, x)] for x in x_axis_array], dtype=np.float32)
-
-                ax.set_xscale("log", basex=2)
-                ax.set_yscale("log", basey=10)
+                    y_axis_array = np.asarray(
+                        [results[(inner_loop_value, x)] for x in x_axis_array if (inner_loop_value, x) in results],
+                        dtype=np.float32,
+                    )
 
                 (x_axis_label, inner_loop_label) = (
-                    ("batch_size", "sequence_length in #tokens")
-                    if self.args.plot_along_batch
-                    else ("sequence_length in #tokens", "batch_size")
+                    ("batch_size", "len") if self.args.plot_along_batch else ("in #tokens", "bsz")
                 )
 
-                x_axis_array = np.asarray(x_axis_array, np.int)
-                plt.scatter(x_axis_array, y_axis_array, label=f"{model_name} - {inner_loop_label}: {inner_loop_value}")
+                x_axis_array = np.asarray(x_axis_array, np.int)[: len(y_axis_array)]
+                plt.scatter(
+                    x_axis_array, y_axis_array, label=f"{label_model_name} - {inner_loop_label}: {inner_loop_value}"
+                )
                 plt.plot(x_axis_array, y_axis_array, "--")
 
-            title_str += f" {model_name} vs."
+            title_str += f" {label_model_name} vs."
 
         title_str = title_str[:-4]
         y_axis_label = "Time in s" if self.args.is_time else "Memory in MB"

examples/benchmarking/run_benchmark.py

@@ -1,5 +1,5 @@
 # coding=utf-8
-# Copyright 2018 The HuggingFace Inc. team.
+# Copyright 2020 The HuggingFace Inc. team.
 # Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

examples/benchmarking/run_benchmark_tf.py

@@ -0,0 +1,29 @@
+# coding=utf-8
+# Copyright 2018 The HuggingFace Inc. team.
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Benchmarking the library on inference and training in Tensorflow"""
+
+from transformers import HfArgumentParser, TensorflowBenchmark, TensorflowBenchmarkArguments
+
+
+def main():
+    parser = HfArgumentParser(TensorflowBenchmarkArguments)
+    benchmark_args = parser.parse_args_into_dataclasses()[0]
+    benchmark = TensorflowBenchmark(args=benchmark_args)
+    benchmark.run()
+
+
+if __name__ == "__main__":
+    main()

examples/benchmarking/time_xla_1.csv

@@ -0,0 +1,4 @@
+model,batch_size,sequence_length,result
+aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2,8,512,0.2032
+aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2,64,512,1.5279
+aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2,256,512,6.1837

examples/bert-loses-patience/README.md

@@ -0,0 +1,89 @@
+# Patience-based Early Exit
+
+Patience-based Early Exit (PABEE) is a plug-and-play inference method for pretrained language models.
+We have already implemented it on BERT and ALBERT. Basically, you can make your LM faster and more robust with PABEE. It can even improve the performance of ALBERT on GLUE. The only sacrifice is that the batch size can only be 1.
+Learn more in the paper ["BERT Loses Patience: Fast and Robust Inference with Early Exit"](https://arxiv.org/abs/2006.04152) and the official [GitHub repo](https://github.com/JetRunner/PABEE).
+
+![PABEE](https://github.com/JetRunner/PABEE/raw/master/bert-loses-patience.png)
+
+## Training
+
+You can fine-tune a pretrained language model (you can choose from BERT and ALBERT) and train the internal classifiers by:
+```bash
+export GLUE_DIR=/path/to/glue_data
+export TASK_NAME=MRPC
+
+python ./run_glue_with_pabee.py \
+  --model_type albert \
+  --model_name_or_path bert-base-uncased/albert-base-v2 \
+  --task_name $TASK_NAME \
+  --do_train \
+  --do_eval \
+  --do_lower_case \
+  --data_dir "$GLUE_DIR/$TASK_NAME" \
+  --max_seq_length 128 \
+  --per_gpu_train_batch_size 32 \
+  --per_gpu_eval_batch_size 32 \
+  --learning_rate 2e-5 \
+  --save_steps 50 \
+  --logging_steps 50 \
+  --num_train_epochs 5 \
+  --output_dir /path/to/save/ \
+  --evaluate_during_training
+```
+
+## Inference
+
+You can inference with different patience settings by:
+```bash
+export GLUE_DIR=/path/to/glue_data
+export TASK_NAME=MRPC
+
+python ./run_glue_with_pabee.py \
+  --model_type albert \
+  --model_name_or_path /path/to/save/ \
+  --task_name $TASK_NAME \
+  --do_eval \
+  --do_lower_case \
+  --data_dir "$GLUE_DIR/$TASK_NAME" \
+  --max_seq_length 128 \
+  --per_gpu_eval_batch_size 1 \
+  --learning_rate 2e-5 \
+  --logging_steps 50 \
+  --num_train_epochs 15 \
+  --output_dir /path/to/save/ \
+  --eval_all_checkpoints \
+  --patience 3,4,5,6,7,8
+```
+where `patience` can be a list of patience settings, separated by a comma. It will help determine which patience works best.
+
+When evaluating on a regression task (STS-B), you may add `--regression_threshold 0.1` to define the regression threshold.
+
+## Results
+On the GLUE dev set:
+
+| Model        | \#Param | Speed  | CoLA  | MNLI  | MRPC  | QNLI  | QQP   | RTE   | SST\-2 | STS\-B |
+|--------------|---------|--------|-------|-------|-------|-------|-------|-------|--------|--------|
+| ALBERT\-base | 12M     |        | 58\.9 | 84\.6 | 89\.5 | 91\.7 | 89\.6 | 78\.6 | 92\.8  | 89\.5  |
+| \+PABEE      | 12M     | 1\.57x | 61\.2 | 85\.1 | 90\.0 | 91\.8 | 89\.6 | 80\.1 | 93\.0  | 90\.1  |
+
+| Model         | \#Param | Speed\-up | MNLI  | SST\-2 | STS\-B |
+|---------------|---------|-----------|-------|--------|--------|
+| BERT\-base    | 108M    |           | 84\.5 | 92\.1  | 88\.9  |
+| \+PABEE       | 108M    | 1\.62x    | 83\.6 | 92\.0  | 88\.7  |
+| ALBERT\-large | 18M     |           | 86\.4 | 94\.9  | 90\.4  |
+| \+PABEE       | 18M     | 2\.42x    | 86\.8 | 95\.2  | 90\.6  |
+
+
+## Citation
+If you find this resource useful, please consider citing the following paper:
+```bibtex
+@misc{zhou2020bert,
+    title={BERT Loses Patience: Fast and Robust Inference with Early Exit},
+    author={Wangchunshu Zhou and Canwen Xu and Tao Ge and Julian McAuley and Ke Xu and Furu Wei},
+    year={2020},
+    eprint={2006.04152},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+```

examples/bert-loses-patience/pabee/modeling_pabee_albert.py

@@ -0,0 +1,310 @@
+# coding=utf-8
+# Copyright 2020 Google AI, Google Brain, the HuggingFace Inc. team and Microsoft Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch ALBERT model with Patience-based Early Exit. """
+
+import logging
+
+import torch
+import torch.nn as nn
+from torch.nn import CrossEntropyLoss, MSELoss
+
+from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_callable
+from transformers.modeling_albert import (
+    ALBERT_INPUTS_DOCSTRING,
+    ALBERT_START_DOCSTRING,
+    AlbertModel,
+    AlbertPreTrainedModel,
+    AlbertTransformer,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class AlbertTransformerWithPabee(AlbertTransformer):
+    def adaptive_forward(self, hidden_states, current_layer, attention_mask=None, head_mask=None):
+        if current_layer == 0:
+            hidden_states = self.embedding_hidden_mapping_in(hidden_states)
+        else:
+            hidden_states = hidden_states[0]
+
+        layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups)
+
+        # Index of the hidden group
+        group_idx = int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups))
+
+        layer_group_output = self.albert_layer_groups[group_idx](
+            hidden_states,
+            attention_mask,
+            head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group],
+        )
+        hidden_states = layer_group_output[0]
+
+        return (hidden_states,)
+
+
+@add_start_docstrings(
+    "The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top.",
+    ALBERT_START_DOCSTRING,
+)
+class AlbertModelWithPabee(AlbertModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.encoder = AlbertTransformerWithPabee(config)
+
+        self.init_weights()
+        self.patience = 0
+        self.inference_instances_num = 0
+        self.inference_layers_num = 0
+
+        self.regression_threshold = 0
+
+    def set_regression_threshold(self, threshold):
+        self.regression_threshold = threshold
+
+    def set_patience(self, patience):
+        self.patience = patience
+
+    def reset_stats(self):
+        self.inference_instances_num = 0
+        self.inference_layers_num = 0
+
+    def log_stats(self):
+        avg_inf_layers = self.inference_layers_num / self.inference_instances_num
+        message = f"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up = {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
+        print(message)
+
+    @add_start_docstrings_to_callable(ALBERT_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        output_dropout=None,
+        output_layers=None,
+        regression=False,
+    ):
+        r"""
+    Return:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.AlbertConfig`) and inputs:
+        last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during pre-training.
+
+            This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        """
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
+        if attention_mask is None:
+            attention_mask = torch.ones(input_shape, device=device)
+        if token_type_ids is None:
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
+
+        extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
+        extended_attention_mask = extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
+        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+
+        embedding_output = self.embeddings(
+            input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds
+        )
+        encoder_outputs = embedding_output
+
+        if self.training:
+            res = []
+            for i in range(self.config.num_hidden_layers):
+                encoder_outputs = self.encoder.adaptive_forward(
+                    encoder_outputs, current_layer=i, attention_mask=extended_attention_mask, head_mask=head_mask,
+                )
+
+                pooled_output = self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]))
+                logits = output_layers[i](output_dropout(pooled_output))
+                res.append(logits)
+        elif self.patience == 0:  # Use all layers for inference
+            encoder_outputs = self.encoder(encoder_outputs, extended_attention_mask, head_mask=head_mask)
+            pooled_output = self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]))
+            res = [output_layers[self.config.num_hidden_layers - 1](pooled_output)]
+        else:
+            patient_counter = 0
+            patient_result = None
+            calculated_layer_num = 0
+            for i in range(self.config.num_hidden_layers):
+                calculated_layer_num += 1
+                encoder_outputs = self.encoder.adaptive_forward(
+                    encoder_outputs, current_layer=i, attention_mask=extended_attention_mask, head_mask=head_mask,
+                )
+
+                pooled_output = self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]))
+                logits = output_layers[i](pooled_output)
+                if regression:
+                    labels = logits.detach()
+                    if patient_result is not None:
+                        patient_labels = patient_result.detach()
+                    if (patient_result is not None) and torch.abs(patient_result - labels) < self.regression_threshold:
+                        patient_counter += 1
+                    else:
+                        patient_counter = 0
+                else:
+                    labels = logits.detach().argmax(dim=1)
+                    if patient_result is not None:
+                        patient_labels = patient_result.detach().argmax(dim=1)
+                    if (patient_result is not None) and torch.all(labels.eq(patient_labels)):
+                        patient_counter += 1
+                    else:
+                        patient_counter = 0
+
+                patient_result = logits
+                if patient_counter == self.patience:
+                    break
+            res = [patient_result]
+            self.inference_layers_num += calculated_layer_num
+            self.inference_instances_num += 1
+
+        return res
+
+
+@add_start_docstrings(
+    """Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
+    the pooled output) e.g. for GLUE tasks. """,
+    ALBERT_START_DOCSTRING,
+)
+class AlbertForSequenceClassificationWithPabee(AlbertPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.albert = AlbertModelWithPabee(config)
+        self.dropout = nn.Dropout(config.classifier_dropout_prob)
+        self.classifiers = nn.ModuleList(
+            [nn.Linear(config.hidden_size, self.config.num_labels) for _ in range(config.num_hidden_layers)]
+        )
+
+        self.init_weights()
+
+    @add_start_docstrings_to_callable(ALBERT_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        labels=None,
+    ):
+        r"""
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in ``[0, ..., config.num_labels - 1]``.
+            If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
+            If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
+
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.AlbertConfig`) and inputs:
+        loss: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
+            Classification (or regression if config.num_labels==1) loss.
+        logits ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+
+        Examples::
+
+            from transformers import AlbertTokenizer
+            from pabee import AlbertForSequenceClassificationWithPabee
+            import torch
+
+            tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+            model = AlbertForSequenceClassificationWithPabee.from_pretrained('albert-base-v2')
+            input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+            labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+            outputs = model(input_ids, labels=labels)
+            loss, logits = outputs[:2]
+
+        """
+
+        logits = self.albert(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_dropout=self.dropout,
+            output_layers=self.classifiers,
+            regression=self.num_labels == 1,
+        )
+
+        outputs = (logits[-1],)
+
+        if labels is not None:
+            total_loss = None
+            total_weights = 0
+            for ix, logits_item in enumerate(logits):
+                if self.num_labels == 1:
+                    #  We are doing regression
+                    loss_fct = MSELoss()
+                    loss = loss_fct(logits_item.view(-1), labels.view(-1))
+                else:
+                    loss_fct = CrossEntropyLoss()
+                    loss = loss_fct(logits_item.view(-1, self.num_labels), labels.view(-1))
+                if total_loss is None:
+                    total_loss = loss
+                else:
+                    total_loss += loss * (ix + 1)
+                total_weights += ix + 1
+            outputs = (total_loss / total_weights,) + outputs
+
+        return outputs

examples/bert-loses-patience/pabee/modeling_pabee_bert.py

@@ -0,0 +1,342 @@
+# coding=utf-8
+# Copyright 2020 The Google AI Language Team Authors, The HuggingFace Inc. team and Microsoft Corporation.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch BERT model with Patience-based Early Exit. """
+
+
+import logging
+
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss, MSELoss
+
+from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_callable
+from transformers.modeling_bert import (
+    BERT_INPUTS_DOCSTRING,
+    BERT_START_DOCSTRING,
+    BertEncoder,
+    BertModel,
+    BertPreTrainedModel,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class BertEncoderWithPabee(BertEncoder):
+    def adaptive_forward(self, hidden_states, current_layer, attention_mask=None, head_mask=None):
+        layer_outputs = self.layer[current_layer](hidden_states, attention_mask, head_mask[current_layer])
+
+        hidden_states = layer_outputs[0]
+
+        return hidden_states
+
+
+@add_start_docstrings(
+    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.",
+    BERT_START_DOCSTRING,
+)
+class BertModelWithPabee(BertModel):
+    """
+
+    The model can behave as an encoder (with only self-attention) as well
+    as a decoder, in which case a layer of cross-attention is added between
+    the self-attention layers, following the architecture described in `Attention is all you need`_ by Ashish Vaswani,
+    Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
+
+    To behave as an decoder the model needs to be initialized with the
+    :obj:`is_decoder` argument of the configuration set to :obj:`True`; an
+    :obj:`encoder_hidden_states` is expected as an input to the forward pass.
+
+    .. _`Attention is all you need`:
+        https://arxiv.org/abs/1706.03762
+
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.encoder = BertEncoderWithPabee(config)
+
+        self.init_weights()
+        self.patience = 0
+        self.inference_instances_num = 0
+        self.inference_layers_num = 0
+
+        self.regression_threshold = 0
+
+    def set_regression_threshold(self, threshold):
+        self.regression_threshold = threshold
+
+    def set_patience(self, patience):
+        self.patience = patience
+
+    def reset_stats(self):
+        self.inference_instances_num = 0
+        self.inference_layers_num = 0
+
+    def log_stats(self):
+        avg_inf_layers = self.inference_layers_num / self.inference_instances_num
+        message = f"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up = {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
+        print(message)
+
+    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        output_dropout=None,
+        output_layers=None,
+        regression=False,
+    ):
+        r"""
+    Return:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
+        last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during pre-training.
+
+            This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        """
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
+        if attention_mask is None:
+            attention_mask = torch.ones(input_shape, device=device)
+        if token_type_ids is None:
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
+
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, device)
+
+        # If a 2D ou 3D attention mask is provided for the cross-attention
+        # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length]
+        if self.config.is_decoder and encoder_hidden_states is not None:
+            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
+            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
+            if encoder_attention_mask is None:
+                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
+            encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+        else:
+            encoder_extended_attention_mask = None
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+
+        embedding_output = self.embeddings(
+            input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds
+        )
+        encoder_outputs = embedding_output
+
+        if self.training:
+            res = []
+            for i in range(self.config.num_hidden_layers):
+                encoder_outputs = self.encoder.adaptive_forward(
+                    encoder_outputs, current_layer=i, attention_mask=extended_attention_mask, head_mask=head_mask
+                )
+
+                pooled_output = self.pooler(encoder_outputs)
+                logits = output_layers[i](output_dropout(pooled_output))
+                res.append(logits)
+        elif self.patience == 0:  # Use all layers for inference
+            encoder_outputs = self.encoder(
+                embedding_output,
+                attention_mask=extended_attention_mask,
+                head_mask=head_mask,
+                encoder_hidden_states=encoder_hidden_states,
+                encoder_attention_mask=encoder_extended_attention_mask,
+            )
+            pooled_output = self.pooler(encoder_outputs[0])
+            res = [output_layers[self.config.num_hidden_layers - 1](pooled_output)]
+        else:
+            patient_counter = 0
+            patient_result = None
+            calculated_layer_num = 0
+            for i in range(self.config.num_hidden_layers):
+                calculated_layer_num += 1
+                encoder_outputs = self.encoder.adaptive_forward(
+                    encoder_outputs, current_layer=i, attention_mask=extended_attention_mask, head_mask=head_mask
+                )
+
+                pooled_output = self.pooler(encoder_outputs)
+                logits = output_layers[i](pooled_output)
+                if regression:
+                    labels = logits.detach()
+                    if patient_result is not None:
+                        patient_labels = patient_result.detach()
+                    if (patient_result is not None) and torch.abs(patient_result - labels) < self.regression_threshold:
+                        patient_counter += 1
+                    else:
+                        patient_counter = 0
+                else:
+                    labels = logits.detach().argmax(dim=1)
+                    if patient_result is not None:
+                        patient_labels = patient_result.detach().argmax(dim=1)
+                    if (patient_result is not None) and torch.all(labels.eq(patient_labels)):
+                        patient_counter += 1
+                    else:
+                        patient_counter = 0
+
+                patient_result = logits
+                if patient_counter == self.patience:
+                    break
+            res = [patient_result]
+            self.inference_layers_num += calculated_layer_num
+            self.inference_instances_num += 1
+
+        return res
+
+
+@add_start_docstrings(
+    """Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
+    the pooled output) e.g. for GLUE tasks. """,
+    BERT_START_DOCSTRING,
+)
+class BertForSequenceClassificationWithPabee(BertPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.bert = BertModelWithPabee(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifiers = nn.ModuleList(
+            [nn.Linear(config.hidden_size, self.config.num_labels) for _ in range(config.num_hidden_layers)]
+        )
+
+        self.init_weights()
+
+    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        labels=None,
+    ):
+        r"""
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
+            If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
+            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
+            Classification (or regression if config.num_labels==1) loss.
+        logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+
+    Examples::
+
+        from transformers import BertTokenizer, BertForSequenceClassification
+        from pabee import BertForSequenceClassificationWithPabee
+        import torch
+
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertForSequenceClassificationWithPabee.from_pretrained('bert-base-uncased')
+
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+
+        loss, logits = outputs[:2]
+
+        """
+
+        logits = self.bert(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            output_dropout=self.dropout,
+            output_layers=self.classifiers,
+            regression=self.num_labels == 1,
+        )
+
+        outputs = (logits[-1],)
+
+        if labels is not None:
+            total_loss = None
+            total_weights = 0
+            for ix, logits_item in enumerate(logits):
+                if self.num_labels == 1:
+                    #  We are doing regression
+                    loss_fct = MSELoss()
+                    loss = loss_fct(logits_item.view(-1), labels.view(-1))
+                else:
+                    loss_fct = CrossEntropyLoss()
+                    loss = loss_fct(logits_item.view(-1, self.num_labels), labels.view(-1))
+                if total_loss is None:
+                    total_loss = loss
+                else:
+                    total_loss += loss * (ix + 1)
+                total_weights += ix + 1
+            outputs = (total_loss / total_weights,) + outputs
+
+        return outputs

examples/bert-loses-patience/run_glue_with_pabee.py

@@ -0,0 +1,703 @@
+# coding=utf-8
+# Copyright 2020 The Google AI Language Team Authors, The HuggingFace Inc. team and Microsoft Corporation.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Training and inference using the library models for sequence classification on GLUE (Bert, Albert) with PABEE."""
+
+
+import argparse
+import glob
+import json
+import logging
+import os
+import random
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
+from torch.utils.data.distributed import DistributedSampler
+from tqdm import tqdm, trange
+
+from pabee.modeling_pabee_albert import AlbertForSequenceClassificationWithPabee
+from pabee.modeling_pabee_bert import BertForSequenceClassificationWithPabee
+from transformers import (
+    WEIGHTS_NAME,
+    AdamW,
+    AlbertConfig,
+    AlbertTokenizer,
+    BertConfig,
+    BertTokenizer,
+    get_linear_schedule_with_warmup,
+)
+from transformers import glue_compute_metrics as compute_metrics
+from transformers import glue_convert_examples_to_features as convert_examples_to_features
+from transformers import glue_output_modes as output_modes
+from transformers import glue_processors as processors
+
+
+try:
+    from torch.utils.tensorboard import SummaryWriter
+except ImportError:
+    from tensorboardX import SummaryWriter
+
+
+logger = logging.getLogger(__name__)
+
+MODEL_CLASSES = {
+    "bert": (BertConfig, BertForSequenceClassificationWithPabee, BertTokenizer),
+    "albert": (AlbertConfig, AlbertForSequenceClassificationWithPabee, AlbertTokenizer),
+}
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def train(args, train_dataset, model, tokenizer):
+    """ Train the model """
+    if args.local_rank in [-1, 0]:
+        tb_writer = SummaryWriter()
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
+    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
+
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+    else:
+        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
+
+    # Prepare optimizer and schedule (linear warmup and decay)
+    no_decay = ["bias", "LayerNorm.weight"]
+    optimizer_grouped_parameters = [
+        {
+            "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
+            "weight_decay": args.weight_decay,
+        },
+        {"params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0},
+    ]
+
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
+    scheduler = get_linear_schedule_with_warmup(
+        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total
+    )
+
+    # Check if saved optimizer or scheduler states exist
+    if os.path.isfile(os.path.join(args.model_name_or_path, "optimizer.pt")) and os.path.isfile(
+        os.path.join(args.model_name_or_path, "scheduler.pt")
+    ):
+        # Load in optimizer and scheduler states
+        optimizer.load_state_dict(torch.load(os.path.join(args.model_name_or_path, "optimizer.pt")))
+        scheduler.load_state_dict(torch.load(os.path.join(args.model_name_or_path, "scheduler.pt")))
+
+    if args.fp16:
+        try:
+            from apex import amp
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
+
+    # multi-gpu training (should be after apex fp16 initialization)
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
+    # Distributed training (should be after apex fp16 initialization)
+    if args.local_rank != -1:
+        model = torch.nn.parallel.DistributedDataParallel(
+            model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True,
+        )
+
+    # Train!
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info(
+        "  Total train batch size (w. parallel, distributed & accumulation) = %d",
+        args.train_batch_size
+        * args.gradient_accumulation_steps
+        * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),
+    )
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    epochs_trained = 0
+    steps_trained_in_current_epoch = 0
+    # Check if continuing training from a checkpoint
+    if os.path.exists(args.model_name_or_path):
+        # set global_step to gobal_step of last saved checkpoint from model path
+        global_step = int(args.model_name_or_path.split("-")[-1].split("/")[0])
+        epochs_trained = global_step // (len(train_dataloader) // args.gradient_accumulation_steps)
+        steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps)
+
+        logger.info("  Continuing training from checkpoint, will skip to saved global_step")
+        logger.info("  Continuing training from epoch %d", epochs_trained)
+        logger.info("  Continuing training from global step %d", global_step)
+        logger.info(
+            "  Will skip the first %d steps in the first epoch", steps_trained_in_current_epoch,
+        )
+
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(
+        epochs_trained, int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0],
+    )
+    set_seed(args)  # Added here for reproductibility
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
+        for step, batch in enumerate(epoch_iterator):
+
+            # Skip past any already trained steps if resuming training
+            if steps_trained_in_current_epoch > 0:
+                steps_trained_in_current_epoch -= 1
+                continue
+
+            model.train()
+            batch = tuple(t.to(args.device) for t in batch)
+            inputs = {
+                "input_ids": batch[0],
+                "attention_mask": batch[1],
+                "labels": batch[3],
+            }
+            inputs["token_type_ids"] = batch[2]
+            outputs = model(**inputs)
+            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)
+
+            if args.n_gpu > 1:
+                loss = loss.mean()  # mean() to average on multi-gpu parallel training
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+
+            if args.fp16:
+                with amp.scale_loss(loss, optimizer) as scaled_loss:
+                    scaled_loss.backward()
+            else:
+                loss.backward()
+
+            tr_loss += loss.item()
+            if (step + 1) % args.gradient_accumulation_steps == 0:
+                if args.fp16:
+                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+                else:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
+                optimizer.step()
+                scheduler.step()  # Update learning rate schedule
+                model.zero_grad()
+                global_step += 1
+
+                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
+                    logs = {}
+                    if (
+                        args.local_rank == -1 and args.evaluate_during_training
+                    ):  # Only evaluate when single GPU otherwise metrics may not average well
+                        results = evaluate(args, model, tokenizer)
+                        for key, value in results.items():
+                            eval_key = "eval_{}".format(key)
+                            logs[eval_key] = value
+
+                    loss_scalar = (tr_loss - logging_loss) / args.logging_steps
+                    learning_rate_scalar = scheduler.get_lr()[0]
+                    logs["learning_rate"] = learning_rate_scalar
+                    logs["loss"] = loss_scalar
+                    logging_loss = tr_loss
+
+                    for key, value in logs.items():
+                        tb_writer.add_scalar(key, value, global_step)
+                    print(json.dumps({**logs, **{"step": global_step}}))
+
+                if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
+                    # Save model checkpoint
+                    output_dir = os.path.join(args.output_dir, "checkpoint-{}".format(global_step))
+                    if not os.path.exists(output_dir):
+                        os.makedirs(output_dir)
+                    model_to_save = (
+                        model.module if hasattr(model, "module") else model
+                    )  # Take care of distributed/parallel training
+                    model_to_save.save_pretrained(output_dir)
+                    tokenizer.save_pretrained(output_dir)
+
+                    torch.save(args, os.path.join(output_dir, "training_args.bin"))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+                    torch.save(optimizer.state_dict(), os.path.join(output_dir, "optimizer.pt"))
+                    torch.save(scheduler.state_dict(), os.path.join(output_dir, "scheduler.pt"))
+                    logger.info("Saving optimizer and scheduler states to %s", output_dir)
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+
+    if args.local_rank in [-1, 0]:
+        tb_writer.close()
+
+    return global_step, tr_loss / global_step
+
+
+def evaluate(args, model, tokenizer, prefix="", patience=0):
+
+    if args.model_type == "albert":
+        model.albert.set_regression_threshold(args.regression_threshold)
+        model.albert.set_patience(patience)
+        model.albert.reset_stats()
+    elif args.model_type == "bert":
+        model.bert.set_regression_threshold(args.regression_threshold)
+        model.bert.set_patience(patience)
+        model.bert.reset_stats()
+    else:
+        raise NotImplementedError()
+
+    # Loop to handle MNLI double evaluation (matched, mis-matched)
+    eval_task_names = ("mnli", "mnli-mm") if args.task_name == "mnli" else (args.task_name,)
+    eval_outputs_dirs = (args.output_dir, args.output_dir + "-MM") if args.task_name == "mnli" else (args.output_dir,)
+
+    results = {}
+    for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs):
+        eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True)
+
+        if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(eval_output_dir)
+
+        args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+        # Note that DistributedSampler samples randomly
+        eval_sampler = SequentialSampler(eval_dataset)
+        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
+
+        # multi-gpu eval
+        if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
+            model = torch.nn.DataParallel(model)
+
+        # Eval!
+        logger.info("***** Running evaluation {} *****".format(prefix))
+        logger.info("  Num examples = %d", len(eval_dataset))
+        logger.info("  Batch size = %d", args.eval_batch_size)
+        eval_loss = 0.0
+        nb_eval_steps = 0
+        preds = None
+        out_label_ids = None
+        for batch in tqdm(eval_dataloader, desc="Evaluating"):
+            model.eval()
+            batch = tuple(t.to(args.device) for t in batch)
+
+            with torch.no_grad():
+                inputs = {
+                    "input_ids": batch[0],
+                    "attention_mask": batch[1],
+                    "labels": batch[3],
+                }
+                inputs["token_type_ids"] = batch[2]
+                outputs = model(**inputs)
+                tmp_eval_loss, logits = outputs[:2]
+
+                eval_loss += tmp_eval_loss.mean().item()
+            nb_eval_steps += 1
+            if preds is None:
+                preds = logits.detach().cpu().numpy()
+                out_label_ids = inputs["labels"].detach().cpu().numpy()
+            else:
+                preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
+                out_label_ids = np.append(out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0)
+
+        eval_loss = eval_loss / nb_eval_steps
+        if args.output_mode == "classification":
+            preds = np.argmax(preds, axis=1)
+        elif args.output_mode == "regression":
+            preds = np.squeeze(preds)
+        result = compute_metrics(eval_task, preds, out_label_ids)
+        results.update(result)
+
+        output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt")
+        with open(output_eval_file, "w") as writer:
+            logger.info("***** Eval results {} *****".format(prefix))
+            for key in sorted(result.keys()):
+                logger.info("  %s = %s", key, str(result[key]))
+                print("  %s = %s" % (key, str(result[key])))
+                writer.write("%s = %s\n" % (key, str(result[key])))
+
+    if args.eval_all_checkpoints and patience != 0:
+        if args.model_type == "albert":
+            model.albert.log_stats()
+        elif args.model_type == "bert":
+            model.bert.log_stats()
+        else:
+            raise NotImplementedError()
+
+    return results
+
+
+def load_and_cache_examples(args, task, tokenizer, evaluate=False):
+    if args.local_rank not in [-1, 0] and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    processor = processors[task]()
+    output_mode = output_modes[task]
+    # Load data features from cache or dataset file
+    cached_features_file = os.path.join(
+        args.data_dir,
+        "cached_{}_{}_{}_{}".format(
+            "dev" if evaluate else "train",
+            list(filter(None, args.model_name_or_path.split("/"))).pop(),
+            str(args.max_seq_length),
+            str(task),
+        ),
+    )
+    if os.path.exists(cached_features_file) and not args.overwrite_cache:
+        logger.info("Loading features from cached file %s", cached_features_file)
+        features = torch.load(cached_features_file)
+    else:
+        logger.info("Creating features from dataset file at %s", args.data_dir)
+        label_list = processor.get_labels()
+        if task in ["mnli", "mnli-mm"] and args.model_type in ["roberta", "xlmroberta"]:
+            # HACK(label indices are swapped in RoBERTa pretrained model)
+            label_list[1], label_list[2] = label_list[2], label_list[1]
+        examples = (
+            processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
+        )
+        features = convert_examples_to_features(
+            examples, tokenizer, label_list=label_list, max_length=args.max_seq_length, output_mode=output_mode,
+        )
+        if args.local_rank in [-1, 0]:
+            logger.info("Saving features into cached file %s", cached_features_file)
+            torch.save(features, cached_features_file)
+
+    if args.local_rank == 0 and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    # Convert to Tensors and build dataset
+    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
+    all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)
+    all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)
+    if output_mode == "classification":
+        all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
+    elif output_mode == "regression":
+        all_labels = torch.tensor([f.label for f in features], dtype=torch.float)
+
+    dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels)
+    return dataset
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    # Required parameters
+    parser.add_argument(
+        "--data_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The input data dir. Should contain the .tsv files (or other data files) for the task.",
+    )
+    parser.add_argument(
+        "--model_type",
+        default=None,
+        type=str,
+        required=True,
+        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        default=None,
+        type=str,
+        required=True,
+        help="Path to pre-trained model or shortcut name.",
+    )
+    parser.add_argument(
+        "--task_name",
+        default=None,
+        type=str,
+        required=True,
+        help="The name of the task to train selected in the list: " + ", ".join(processors.keys()),
+    )
+    parser.add_argument(
+        "--output_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The output directory where the model predictions and checkpoints will be written.",
+    )
+    parser.add_argument(
+        "--patience", default="0", type=str, required=False,
+    )
+    parser.add_argument(
+        "--regression_threshold", default=0, type=float, required=False,
+    )
+
+    # Other parameters
+    parser.add_argument(
+        "--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name",
+    )
+    parser.add_argument(
+        "--tokenizer_name",
+        default="",
+        type=str,
+        help="Pretrained tokenizer name or path if not the same as model_name",
+    )
+    parser.add_argument(
+        "--cache_dir",
+        default="",
+        type=str,
+        help="Where do you want to store the pre-trained models downloaded from s3",
+    )
+    parser.add_argument(
+        "--max_seq_length",
+        default=128,
+        type=int,
+        help="The maximum total input sequence length after tokenization. Sequences longer "
+        "than this will be truncated, sequences shorter will be padded.",
+    )
+    parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
+    parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.")
+    parser.add_argument(
+        "--evaluate_during_training", action="store_true", help="Run evaluation during training at each logging step.",
+    )
+    parser.add_argument(
+        "--do_lower_case", action="store_true", help="Set this flag if you are using an uncased model.",
+    )
+
+    parser.add_argument(
+        "--per_gpu_train_batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.",
+    )
+    parser.add_argument(
+        "--per_gpu_eval_batch_size", default=1, type=int, help="Batch size per GPU/CPU for evaluation.",
+    )
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Number of updates steps to accumulate before performing a backward/update pass.",
+    )
+    parser.add_argument(
+        "--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.",
+    )
+    parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
+    parser.add_argument(
+        "--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.",
+    )
+    parser.add_argument(
+        "--max_steps",
+        default=-1,
+        type=int,
+        help="If > 0: set total number of training steps to perform. Override num_train_epochs.",
+    )
+    parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")
+
+    parser.add_argument("--logging_steps", type=int, default=500, help="Log every X updates steps.")
+    parser.add_argument(
+        "--save_steps", type=int, default=500, help="Save checkpoint every X updates steps.",
+    )
+    parser.add_argument(
+        "--eval_all_checkpoints",
+        action="store_true",
+        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number",
+    )
+    parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
+    parser.add_argument(
+        "--overwrite_output_dir", action="store_true", help="Overwrite the content of the output directory",
+    )
+    parser.add_argument(
+        "--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets",
+    )
+    parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
+
+    parser.add_argument(
+        "--fp16",
+        action="store_true",
+        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
+    )
+    parser.add_argument(
+        "--fp16_opt_level",
+        type=str,
+        default="O1",
+        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+        "See details at https://nvidia.github.io/apex/amp.html",
+    )
+    parser.add_argument(
+        "--local_rank", type=int, default=-1, help="For distributed training: local_rank",
+    )
+    parser.add_argument("--server_ip", type=str, default="", help="For distant debugging.")
+    parser.add_argument("--server_port", type=str, default="", help="For distant debugging.")
+    args = parser.parse_args()
+
+    if (
+        os.path.exists(args.output_dir)
+        and os.listdir(args.output_dir)
+        and args.do_train
+        and not args.overwrite_output_dir
+    ):
+        raise ValueError(
+            "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
+                args.output_dir
+            )
+        )
+
+    # Setup distant debugging if needed
+    if args.server_ip and args.server_port:
+        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
+        import ptvsd
+
+        print("Waiting for debugger attach")
+        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
+        ptvsd.wait_for_attach()
+
+    # Setup CUDA, GPU & distributed training
+    if args.local_rank == -1 or args.no_cuda:
+        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+        args.n_gpu = torch.cuda.device_count()
+    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+        torch.cuda.set_device(args.local_rank)
+        device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend="nccl")
+        args.n_gpu = 1
+    args.device = device
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
+    )
+    logger.warning(
+        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+        args.local_rank,
+        device,
+        args.n_gpu,
+        bool(args.local_rank != -1),
+        args.fp16,
+    )
+
+    # Set seed
+    set_seed(args)
+
+    # Prepare GLUE task
+    args.task_name = args.task_name.lower()
+    if args.task_name not in processors:
+        raise ValueError("Task not found: %s" % (args.task_name))
+    processor = processors[args.task_name]()
+    args.output_mode = output_modes[args.task_name]
+    label_list = processor.get_labels()
+    num_labels = len(label_list)
+
+    if args.patience != "0" and args.per_gpu_eval_batch_size != 1:
+        raise ValueError("The eval batch size must be 1 with PABEE inference on.")
+
+    # Load pretrained model and tokenizer
+    if args.local_rank not in [-1, 0]:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    args.model_type = args.model_type.lower()
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(
+        args.config_name if args.config_name else args.model_name_or_path,
+        num_labels=num_labels,
+        finetuning_task=args.task_name,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+    tokenizer = tokenizer_class.from_pretrained(
+        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+        do_lower_case=args.do_lower_case,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+    model = model_class.from_pretrained(
+        args.model_name_or_path,
+        from_tf=bool(".ckpt" in args.model_name_or_path),
+        config=config,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+
+    if args.local_rank == 0:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    model.to(args.device)
+
+    print("Total Model Parameters:", sum(param.numel() for param in model.parameters()))
+    output_layers_param_num = sum(param.numel() for param in model.classifiers.parameters())
+    print("Output Layers Parameters:", output_layers_param_num)
+    single_output_layer_param_num = sum(param.numel() for param in model.classifiers[0].parameters())
+    print(
+        "Added Output Layers Parameters:", output_layers_param_num - single_output_layer_param_num,
+    )
+
+    logger.info("Training/evaluation parameters %s", args)
+
+    # Training
+    if args.do_train:
+        train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)
+        global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+
+    # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
+        # Create output directory if needed
+        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(args.output_dir)
+
+        logger.info("Saving model checkpoint to %s", args.output_dir)
+        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
+        # They can then be reloaded using `from_pretrained()`
+        model_to_save = (
+            model.module if hasattr(model, "module") else model
+        )  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(args.output_dir)
+        tokenizer.save_pretrained(args.output_dir)
+
+        # Good practice: save your training arguments together with the trained model
+        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
+
+        # Load a trained model and vocabulary that you have fine-tuned
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
+        model.to(args.device)
+
+    # Evaluation
+    results = {}
+    if args.do_eval and args.local_rank in [-1, 0]:
+        patience_list = [int(x) for x in args.patience.split(",")]
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        checkpoints = [args.output_dir]
+        if args.eval_all_checkpoints:
+            checkpoints = list(
+                os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME, recursive=True))
+            )
+            logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN)  # Reduce logging
+        logger.info("Evaluate the following checkpoints: %s", checkpoints)
+
+        for checkpoint in checkpoints:
+
+            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
+            prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""
+
+            model = model_class.from_pretrained(checkpoint)
+            model.to(args.device)
+
+            print(f"Evaluation for checkpoint {prefix}")
+            for patience in patience_list:
+                result = evaluate(args, model, tokenizer, prefix=prefix, patience=patience)
+                result = dict((k + "_{}".format(global_step), v) for k, v in result.items())
+                results.update(result)
+    return results
+
+
+if __name__ == "__main__":
+    main()

examples/bert-loses-patience/test_run_glue_with_pabee.py

@@ -0,0 +1,48 @@
+import argparse
+import logging
+import sys
+import unittest
+from unittest.mock import patch
+
+import run_glue_with_pabee
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+logger = logging.getLogger()
+
+
+def get_setup_file():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f")
+    args = parser.parse_args()
+    return args.f
+
+
+class PabeeTests(unittest.TestCase):
+    def test_run_glue(self):
+        stream_handler = logging.StreamHandler(sys.stdout)
+        logger.addHandler(stream_handler)
+
+        testargs = """
+            run_glue_with_pabee.py
+            --model_type albert
+            --model_name_or_path albert-base-v2
+            --data_dir ./tests/fixtures/tests_samples/MRPC/
+            --task_name mrpc
+            --do_train
+            --do_eval
+            --output_dir ./tests/fixtures/tests_samples/temp_dir
+            --per_gpu_train_batch_size=2
+            --per_gpu_eval_batch_size=1
+            --learning_rate=2e-5
+            --max_steps=50
+            --warmup_steps=2
+            --overwrite_output_dir
+            --seed=42
+            --max_seq_length=128
+            """.split()
+        with patch.object(sys, "argv", testargs):
+            result = run_glue_with_pabee.main()
+            for value in result.values():
+                self.assertGreaterEqual(value, 0.75)

examples/lightning_base.py

@@ -8,6 +8,7 @@ from typing import Any, Dict
 import numpy as np
 import pytorch_lightning as pl
 import torch
+from pytorch_lightning.utilities import rank_zero_info, rank_zero_only
 
 from transformers import (
     AdamW,
@@ -60,10 +61,9 @@ class BaseTransformer(pl.LightningModule):
         model=None,
         **config_kwargs
     ):
-        "Initialize a model."
-
+        """Initialize a model, tokenizer and config."""
         super().__init__()
-        self.hparams = hparams
+        self.hparams = hparams  # TODO: move to self.save_hyperparameters()
         self.step_count = 0
         self.tfmr_ckpts = {}
         self.output_dir = Path(self.hparams.output_dir)
@@ -84,8 +84,8 @@ class BaseTransformer(pl.LightningModule):
             )
         else:
             self.tokenizer: PreTrainedTokenizer = tokenizer
+        self.model_type = MODEL_MODES[mode]
         if model is None:
-            self.model_type = MODEL_MODES[mode]
             self.model = self.model_type.from_pretrained(
                 self.hparams.model_name_or_path,
                 from_tf=bool(".ckpt" in self.hparams.model_name_or_path),
@@ -93,18 +93,13 @@ class BaseTransformer(pl.LightningModule):
                 cache_dir=cache_dir,
             )
         else:
-            self.model_type = None
             self.model = model
 
     def load_hf_checkpoint(self, *args, **kwargs):
         self.model = self.model_type.from_pretrained(*args, **kwargs)
 
-    def is_logger(self):
-        return self.trainer.proc_rank <= 0
-
     def configure_optimizers(self):
         "Prepare optimizer and schedule (linear warmup and decay)"
-
         model = self.model
         no_decay = ["bias", "LayerNorm.weight"]
         optimizer_grouped_parameters = [
@@ -137,7 +132,7 @@ class BaseTransformer(pl.LightningModule):
     def test_step(self, batch, batch_nb):
         return self.validation_step(batch, batch_nb)
 
-    def test_end(self, outputs):
+    def test_epoch_end(self, outputs):
         return self.validation_end(outputs)
 
     def train_dataloader(self):
@@ -208,6 +203,7 @@ class BaseTransformer(pl.LightningModule):
         parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
         parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
         parser.add_argument("--warmup_steps", default=500, type=int, help="Linear warmup over warmup_steps.")
+        parser.add_argument("--num_workers", default=4, type=int, help="kwarg passed to DataLoader")
         parser.add_argument(
             "--num_train_epochs", default=3, type=int, help="Total number of training epochs to perform."
         )
@@ -217,28 +213,26 @@ class BaseTransformer(pl.LightningModule):
 
 
 class LoggingCallback(pl.Callback):
+    @rank_zero_only
     def on_validation_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule):
-        logger.info("***** Validation results *****")
-        if pl_module.is_logger():
-            metrics = trainer.callback_metrics
-            # Log results
+        rank_zero_info("***** Validation results *****")
+        metrics = trainer.callback_metrics
+        # Log results
+        for key in sorted(metrics):
+            if key not in ["log", "progress_bar"]:
+                rank_zero_info("{} = {}\n".format(key, str(metrics[key])))
+
+    @rank_zero_only
+    def on_test_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule):
+        logger.info("***** Test results *****")
+        metrics = trainer.callback_metrics
+        # Log and save results to file
+        output_test_results_file = os.path.join(pl_module.hparams.output_dir, "test_results.txt")
+        with open(output_test_results_file, "w") as writer:
             for key in sorted(metrics):
                 if key not in ["log", "progress_bar"]:
                     logger.info("{} = {}\n".format(key, str(metrics[key])))
-
-    def on_test_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule):
-        logger.info("***** Test results *****")
-
-        if pl_module.is_logger():
-            metrics = trainer.callback_metrics
-
-            # Log and save results to file
-            output_test_results_file = os.path.join(pl_module.hparams.output_dir, "test_results.txt")
-            with open(output_test_results_file, "w") as writer:
-                for key in sorted(metrics):
-                    if key not in ["log", "progress_bar"]:
-                        logger.info("{} = {}\n".format(key, str(metrics[key])))
-                        writer.write("{} = {}\n".format(key, str(metrics[key])))
+                    writer.write("{} = {}\n".format(key, str(metrics[key])))
 
 
 def add_generic_args(parser, root_dir) -> None:

examples/longform-qa/eli5_app.py

@@ -1,11 +1,11 @@
-import numpy as np
-import torch
-
 import faiss
 import nlp
+import numpy as np
+import torch
+from elasticsearch import Elasticsearch
+
 import streamlit as st
 import transformers
-from elasticsearch import Elasticsearch
 from eli5_utils import (
     embed_questions_for_retrieval,
     make_qa_s2s_model,

examples/longform-qa/eli5_utils.py

@@ -4,17 +4,17 @@ import os  # noqa: F401
 from random import choice, randint
 from time import time
 
+import faiss  # noqa: F401
+import nlp  # noqa: F401
 import numpy as np
 import pandas as pd
 import torch
 import torch.utils.checkpoint as checkpoint
+from elasticsearch import Elasticsearch  # noqa: F401
+from elasticsearch.helpers import bulk, streaming_bulk  # noqa: F401
 from torch.utils.data import DataLoader, Dataset, RandomSampler, SequentialSampler
 from tqdm import tqdm
 
-import faiss  # noqa: F401
-import nlp  # noqa: F401
-from elasticsearch import Elasticsearch  # noqa: F401
-from elasticsearch.helpers import bulk, streaming_bulk  # noqa: F401
 from transformers import AdamW, AutoModel, AutoModelForSeq2SeqLM, AutoTokenizer, get_linear_schedule_with_warmup
 
 

examples/requirements.txt

@@ -5,6 +5,11 @@ psutil
 sacrebleu
 rouge-score
 tensorflow_datasets
-pytorch-lightning==0.7.6
+pytorch-lightning==0.8.1
 matplotlib
 git-python==1.0.3
+faiss
+streamlit
+elasticsearch
+pandas
+nlp

examples/summarization/callbacks.py

@@ -19,12 +19,11 @@ logger = logging.getLogger(__name__)
 
 
 class Seq2SeqLoggingCallback(pl.Callback):
+    @rank_zero_only
     def _write_logs(
         self, trainer: pl.Trainer, pl_module: pl.LightningModule, type_path: str, save_generations=True
     ) -> None:
         logger.info(f"***** {type_path} results at step {trainer.global_step:05d} *****")
-        if not pl_module.is_logger():
-            return
         metrics = trainer.callback_metrics
         trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ["log", "progress_bar", "preds"]})
         # Log results

examples/summarization/distillation.py

@@ -271,6 +271,7 @@ class SummarizationDistiller(SummarizationModule):
 
 class T5SummarizationDistiller(SummarizationDistiller):
     def pre_init(self, hparams):
+        raise NotImplementedError("T5 Distillation does not work yet")
         teacher = T5ForConditionalGeneration.from_pretrained(hparams.teacher)
         n_layer = hparams.student_decoder_layers
         assert n_layer == hparams.student_encoder_layers  # TODO(SS): relax this

examples/summarization/finetune.py

@@ -85,7 +85,7 @@ class SummarizationModule(BaseTransformer):
         if self.hparams.freeze_encoder:
             freeze_params(self.model.model.encoder)  # TODO: this will break for t5
         self.hparams.git_sha = get_git_info()["repo_sha"]
-        self.num_workers = 4 if self.hparams.gpus <= 1 else None  # passing num_workers breaks lightning for multigpu
+        self.num_workers = hparams.num_workers
 
     def freeze_embeds(self):
         """Freeze token embeddings and positional embeddings for bart, just token embeddings for t5."""
@@ -126,7 +126,7 @@ class SummarizationModule(BaseTransformer):
     def validation_step(self, batch, batch_idx) -> Dict:
         return self._generative_step(batch)
 
-    def validation_end(self, outputs, prefix="val") -> Dict:
+    def validation_epoch_end(self, outputs, prefix="val") -> Dict:
         self.step_count += 1
         losses = {k: torch.stack([x[k] for x in outputs]).mean() for k in self.loss_names}
         loss = losses["loss"]
@@ -144,14 +144,12 @@ class SummarizationModule(BaseTransformer):
         self.metrics[prefix].append(metrics)
         pickle_save(self.metrics, self.metrics_save_path)
 
-    def _generative_step(self, batch):
+    def _generative_step(self, batch: dict) -> dict:
         pad_token_id = self.tokenizer.pad_token_id
         source_ids, source_mask, y = SummarizationDataset.trim_seq2seq_batch(batch, pad_token_id)
-        # TODO(SS): task specific params
-
         t0 = time.time()
         generated_ids = self.model.generate(input_ids=source_ids, attention_mask=source_mask, use_cache=True,)
-        gen_time = time.time() - t0
+        gen_time = (time.time() - t0) / source_ids.shape[0]
         preds = self.ids_to_clean_text(generated_ids)
         target = self.ids_to_clean_text(y)
         loss_tensors = self._step(batch)
@@ -164,24 +162,8 @@ class SummarizationModule(BaseTransformer):
     def test_step(self, batch, batch_idx):
         return self._generative_step(batch)
 
-    def test_end(self, outputs):
-        return self.validation_end(outputs, prefix="test")
-
     def test_epoch_end(self, outputs):
-        output_test_predictions_file = os.path.join(self.hparams.output_dir, "test_predictions.txt")
-        output_test_targets_file = os.path.join(self.hparams.output_dir, "test_targets.txt")
-        # write predictions and targets for later rouge evaluation.
-        with open(output_test_predictions_file, "w+") as p_writer, open(output_test_targets_file, "w+") as t_writer:
-            for output_batch in outputs:
-                p_writer.writelines(s + "\n" for s in output_batch["preds"])
-                t_writer.writelines(s + "\n" for s in output_batch["target"])
-            p_writer.close()
-            t_writer.close()
-
-        return self.test_end(outputs)
-
-    def validation_epoch_end(self, outputs):
-        self.validation_end(outputs, "val")
+        return self.validation_epoch_end(outputs, prefix="test")
 
     def get_dataset(self, type_path) -> SummarizationDataset:
         n_obs = self.n_obs[type_path]
@@ -310,6 +292,7 @@ def main(args, model=None) -> SummarizationModule:
         logger=logger,
         # TODO: early stopping callback seems messed up
     )
+    pickle_save(model.hparams, model.output_dir / "hparams.pkl")
     if not args.do_predict:
         return model
 

examples/summarization/finetune.sh

@@ -1,7 +1,3 @@
-export OUTPUT_DIR=bart_cnn_finetune
-
-# Make output directory if it doesn't exist
-mkdir -p $OUTPUT_DIR
 
 # Add parent directory to python path to access lightning_base.py
 export PYTHONPATH="../":"${PYTHONPATH}"
@@ -9,6 +5,7 @@ export PYTHONPATH="../":"${PYTHONPATH}"
 
 # --model_name_or_path=t5-base for t5
 
+# the proper usage is documented in the README
 python finetune.py \
     --model_name_or_path=facebook/bart-large \
     --learning_rate=3e-5 \

examples/summarization/finetune_bart_tiny.sh

@@ -16,14 +16,13 @@ mkdir -p $OUTPUT_DIR
 export PYTHONPATH="../":"${PYTHONPATH}"
 python finetune.py \
 --data_dir=cnn_tiny/ \
---model_type=bart \
 --model_name_or_path=sshleifer/bart-tiny-random \
 --learning_rate=3e-5 \
 --train_batch_size=2 \
 --eval_batch_size=2 \
 --output_dir=$OUTPUT_DIR \
 --num_train_epochs=1  \
---n_gpu=0 \
+--gpus=0 \
 --do_train $@
 
 rm -rf cnn_tiny

examples/summarization/finetune_t5.sh

@@ -11,7 +11,6 @@ export PYTHONPATH="../":"${PYTHONPATH}"
 python finetune.py \
 --data_dir=./cnn-dailymail/cnn_dm \
 --model_name_or_path=t5-large \
---model_type=t5
 --learning_rate=3e-5 \
 --train_batch_size=4 \
 --eval_batch_size=4 \

examples/summarization/run_eval.py

@@ -26,6 +26,7 @@ def generate_summaries(
     examples: list, out_file: str, model_name: str, batch_size: int = 8, device: str = DEFAULT_DEVICE, fp16=False,
 ) -> None:
     fout = Path(out_file).open("w", encoding="utf-8")
+    model_name = str(model_name)
     model = AutoModelForSeq2SeqLM.from_pretrained(model_name).to(device)
     if fp16:
         model = model.half()

examples/summarization/test_summarization_examples.py

@@ -24,6 +24,7 @@ logger = logging.getLogger()
 FP16_EVER = False
 CHEAP_ARGS = {
     "logger": "default",
+    "num_workers": 2,
     "alpha_hid": 0,
     "freeze_embeds": True,
     "enc_only": False,
@@ -79,7 +80,8 @@ def _dump_articles(path: Path, articles: list):
         f.write("\n".join(articles))
 
 
-BDIR = Path("~/transformers_fork/examples/summarization/bart/").absolute()
+MSG = "T5 is broken at the moment"
+T5_TINY = "patrickvonplaten/t5-tiny-random"
 
 
 def make_test_data_dir():
@@ -92,7 +94,6 @@ def make_test_data_dir():
     return tmp_dir
 
 
-@unittest.skip("These wont' pass until hidden_states kwarg is merged.")
 class TestSummarizationDistiller(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -108,47 +109,22 @@ class TestSummarizationDistiller(unittest.TestCase):
             freeze_encoder=True,
             gpus=2,
             sortish_sampler=False,
-        )
-        self._bart_distiller_cli(updates)
-
-    @unittest.skipUnless(torch.cuda.is_available(), "skipping fp16 test")
-    def test_bdc_fp16(self):
-        updates = dict(
-            student_encoder_layers=2,
-            student_decoder_layers=1,
-            alpha_hid=3.0,
-            freeze_encoder=True,
-            gpus=1,
-            fp16=FP16_EVER,
             fp16_opt_level="O1",
+            fp16=FP16_EVER,
         )
         self._bart_distiller_cli(updates)
 
-    @unittest.skipUnless(torch.cuda.is_available(), "skipping fp16 test")
-    def test_bdc_t5_eval_fp16(self):
+    def test_bdc_t5_train(self):
         updates = dict(
             fp16=FP16_EVER,
-            gpus=1,
+            gpus=1 if torch.cuda.is_available() else 0,
             model_type="t5",
-            model_name_or_path="patrickvonplaten/t5-tiny-random",
-            do_train=False,
-            do_predict=True,
-            tokenizer_name=None,
-            no_teacher=True,
-        )
-        self._bart_distiller_cli(updates, check_contents=False)
-
-    @unittest.skipUnless(torch.cuda.is_available(), "skipping fp16 test")
-    def test_bdc_t5_train_fp16(self):
-        updates = dict(
-            fp16=FP16_EVER,
-            gpus=1,
-            model_type="t5",
-            model_name_or_path="patrickvonplaten/t5-tiny-random",
+            model_name_or_path=T5_TINY,
             do_train=True,
             do_predict=True,
-            tokenizer_name="patrickvonplaten/t5-tiny-random",
+            tokenizer_name=T5_TINY,
             no_teacher=True,
+            alpha_hid=2.0,
         )
         self._bart_distiller_cli(updates)
 
@@ -161,7 +137,6 @@ class TestSummarizationDistiller(unittest.TestCase):
         self._bart_distiller_cli(updates)
 
     def test_bdc_checkpointing(self):
-
         updates = dict(
             student_encoder_layers=2,
             student_decoder_layers=1,
@@ -184,32 +159,8 @@ class TestSummarizationDistiller(unittest.TestCase):
 
         evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
 
-    def test_bdc_t5(self):
-        updates = dict(
-            student_encoder_layers=1,
-            student_decoder_layers=1,
-            alpha_hid=2.0,
-            teacher="patrickvonplaten/t5-tiny-random",
-            model_type="t5",
-            model_name_or_path="patrickvonplaten/t5-tiny-random",
-            tokenizer_name="patrickvonplaten/t5-tiny-random",
-        )
-        self._bart_distiller_cli(updates)
-
-    def test_bdc_t5_eval(self):
-        updates = dict(
-            model_type="t5",
-            model_name_or_path="patrickvonplaten/t5-tiny-random",
-            do_train=False,
-            do_predict=True,
-            tokenizer_name="patrickvonplaten/t5-tiny-random",
-            no_teacher=True,
-        )
-        self._bart_distiller_cli(updates, check_contents=False)
-
     def _bart_distiller_cli(self, updates, check_contents=True):
         default_updates = dict(
-            model_type="bart",
             train_batch_size=1,
             eval_batch_size=2,
             num_train_epochs=2,
@@ -237,21 +188,14 @@ class TestSummarizationDistiller(unittest.TestCase):
         self.assertIn(ckpt_name, contents)
         self.assertIn("metrics.pkl", contents)
         self.assertIn("test_generations.txt", contents)
-        self.assertIn("val_generations_1.txt", contents)
-        self.assertIn("val_1_results.txt", contents)
+        self.assertIn("val_generations_00001.txt", contents)
+        self.assertIn("val_results_00001.txt", contents)
         self.assertIn("test_results.txt", contents)
-        # self.assertEqual(len(contents), 15)
 
         metrics = pickle_load(Path(output_dir) / "metrics.pkl")
-        import pandas as pd
-
-        val_df = pd.DataFrame(metrics["val"])
-        train_df = pd.DataFrame(metrics["train"])
-        test_df = pd.DataFrame(metrics["test"])
-        desired_n_evals = args_d["num_train_epochs"] * 2 + 1
-        self.assertEqual(val_df.shape[0], desired_n_evals)  #
-        self.assertEqual(test_df.shape[1], val_df.shape[1])
-        self.assertEqual(train_df.shape[0], 0)
+        desired_n_evals = int(args_d["num_train_epochs"] * (1 / args_d["val_check_interval"]) + 1)
+        self.assertEqual(len(metrics["val"]), desired_n_evals)
+        self.assertEqual(len(metrics["train"]), 0)  # doesn't get logged here
         return model
 
 
@@ -281,9 +225,8 @@ class TestBartExamples(unittest.TestCase):
         output_dir = tempfile.mkdtemp(prefix="output_")
         args_d.update(
             data_dir=tmp_dir,
-            model_type="t5",
-            model_name_or_path="patrickvonplaten/t5-tiny-random",
-            tokenizer_name=None,  # "patrickvonplaten/t5-tiny-random",
+            model_name_or_path=T5_TINY,
+            tokenizer_name=None,  # T5_TINY,
             train_batch_size=2,
             eval_batch_size=2,
             gpus=0,

examples/summarization/utils.py

@@ -13,8 +13,6 @@ from torch import nn
 from torch.utils.data import Dataset, Sampler
 from tqdm import tqdm
 
-from transformers import BartTokenizer
-
 
 def encode_file(
     tokenizer,
@@ -47,8 +45,10 @@ def encode_file(
             max_length=max_length,
             pad_to_max_length=pad_to_max_length,
             add_prefix_space=True,
+            truncation=True,
             return_tensors=return_tensors,
         )
+        assert tokenized.input_ids.shape[1] == max_length
         examples.append(tokenized)
     torch.save(lmap(dict, examples), cache_path.open("wb"))
     return examples
@@ -85,7 +85,7 @@ class SummarizationDataset(Dataset):
         prefix="",
     ):
         super().__init__()
-        tok_name = "T5" if not isinstance(tokenizer, BartTokenizer) else ""
+        tok_name = tokenizer.__class__.__name__.lower().rstrip("tokenizer")
         self.source = encode_file(
             tokenizer,
             os.path.join(data_dir, type_path + ".source"),
@@ -94,16 +94,10 @@ class SummarizationDataset(Dataset):
             prefix=prefix,
             tok_name=tok_name,
         )
-        if type_path == "train":
-            tgt_path = os.path.join(data_dir, type_path + ".target")
-        else:
-            tgt_path = os.path.join(data_dir, type_path + ".target")
-
+        tgt_path = os.path.join(data_dir, type_path + ".target")
         self.target = encode_file(
             tokenizer, tgt_path, max_target_length, overwrite_cache=overwrite_cache, tok_name=tok_name
         )
-        self.source = encode_file(tokenizer, os.path.join(data_dir, type_path + ".source"), max_source_length)
-        self.target = encode_file(tokenizer, os.path.join(data_dir, type_path + ".target"), max_target_length)
         if n_obs is not None:
             self.source = self.source[:n_obs]
             self.target = self.target[:n_obs]

examples/text-classification/README.md

@@ -52,9 +52,12 @@ Some of these results are significantly different from the ones reported on the
 of GLUE benchmark on the website. For QQP and WNLI, please refer to [FAQ #12](https://gluebenchmark.com/faq) on the webite.
 
 Before running any one of these GLUE tasks you should download the
-[GLUE data](https://gluebenchmark.com/tasks) by running
-[this script](https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e)
-and unpack it to some directory `$GLUE_DIR`.
+[GLUE data](https://gluebenchmark.com/tasks) by running the following lines at the root of the repo
+```
+python utils/download_glue_data.py --data_dir /path/to/glue --tasks all
+```
+
+after replacing *path/to/glue* with a value that you like. Then you can run
 
 ```bash
 export GLUE_DIR=/path/to/glue

examples/text-classification/run_pl_glue.py

@@ -108,7 +108,7 @@ class GLUETransformer(BaseTransformer):
 
         return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}
 
-    def _eval_end(self, outputs):
+    def _eval_end(self, outputs) -> tuple:
         val_loss_mean = torch.stack([x["val_loss"] for x in outputs]).mean().detach().cpu().item()
         preds = np.concatenate([x["pred"] for x in outputs], axis=0)
 
@@ -132,20 +132,14 @@ class GLUETransformer(BaseTransformer):
         logs = ret["log"]
         return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}
 
-    def test_epoch_end(self, outputs):
-        # updating to test_epoch_end instead of deprecated test_end
+    def test_epoch_end(self, outputs) -> dict:
         ret, predictions, targets = self._eval_end(outputs)
-
-        # Converting to the dic required by pl
-        # https://github.com/PyTorchLightning/pytorch-lightning/blob/master/\
-        # pytorch_lightning/trainer/logging.py#L139
         logs = ret["log"]
         # `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
         return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}
 
     @staticmethod
     def add_model_specific_args(parser, root_dir):
-        # Add NER specific options
         BaseTransformer.add_model_specific_args(parser, root_dir)
         parser.add_argument(
             "--max_seq_length",

model_cards/a-ware/bart-squadv2/README.md

@@ -0,0 +1,76 @@
+---
+datasets:
+- squad_v2
+---
+
+# BART-LARGE finetuned on SQuADv2
+
+This is bart-large model finetuned on SQuADv2 dataset for question answering task
+
+## Model details
+BART was propsed in the [paper](https://arxiv.org/abs/1910.13461) **BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension**.
+BART is a seq2seq model intended for both NLG and NLU tasks. 
+
+To use BART for question answering tasks, we feed the complete document into the encoder and decoder, and use the top
+hidden state of the decoder as a representation for each
+word. This representation is used to classify the token. As given in the paper bart-large achives comparable to ROBERTa on SQuAD.
+Another notable thing about BART is that it can handle sequences with upto 1024 tokens.
+
+| Param               | #Value |
+|---------------------|--------|
+| encoder layers      | 12     |
+| decoder layers      | 12     |
+| hidden size         | 4096   |
+| num attetion heads  | 16     |
+| on disk size        | 1.63GB |
+
+
+## Model training
+This model was trained with following parameters using simpletransformers wrapper:
+```
+train_args = {
+    'learning_rate': 1e-5,
+    'max_seq_length': 512,
+    'doc_stride': 512,
+    'overwrite_output_dir': True,
+    'reprocess_input_data': False,
+    'train_batch_size': 8,
+    'num_train_epochs': 2,
+    'gradient_accumulation_steps': 2,
+    'no_cache': True,
+    'use_cached_eval_features': False,
+    'save_model_every_epoch': False,
+    'output_dir': "bart-squadv2",
+    'eval_batch_size': 32,
+    'fp16_opt_level': 'O2',
+    }
+```
+
+[You can even train your own model using this colab notebook](https://colab.research.google.com/drive/1I5cK1M_0dLaf5xoewh6swcm5nAInfwHy?usp=sharing)
+
+## Results
+```{"correct": 6832, "similar": 4409, "incorrect": 632, "eval_loss": -14.950117511952177}```
+
+## Model in Action  🚀
+```python3
+from transformers import BartTokenizer, BartForQuestionAnswering
+import torch
+
+tokenizer = BartTokenizer.from_pretrained('a-ware/bart-squadv2')
+model = BartForQuestionAnswering.from_pretrained('a-ware/bart-squadv2')
+
+question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
+encoding = tokenizer.encode_plus(question, text, return_tensors='pt')
+input_ids = encoding['input_ids']
+attention_mask = encoding['attention_mask']
+
+start_scores, end_scores = model(input_ids, attention_mask=attention_mask, output_attentions=False)[:2]
+
+all_tokens = tokenizer.convert_ids_to_tokens(input_ids[0])
+answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])
+answer = tokenizer.convert_tokens_to_ids(answer.split())
+answer = tokenizer.decode(answer)
+#answer => 'a nice puppet' 
+```
+
+> Created with ❤️ by A-ware UG [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/aware-ai)

model_cards/a-ware/roberta-large-squad-classification/README.md

@@ -0,0 +1,48 @@
+---
+datasets:
+- squad_v2
+---
+
+# Roberta-LARGE finetuned on SQuADv2
+
+This is roberta-large model finetuned on SQuADv2 dataset for question answering answerability classification
+
+## Model details
+This model is simply an Sequenceclassification model with two inputs (context and question) in a list.
+The result is either [1] for answerable or [0] if it is not answerable.
+It was trained over 4 epochs on squadv2 dataset and can be used to filter out which context is good to give into the QA model to avoid bad answers.
+
+## Model training
+This model was trained with following parameters using simpletransformers wrapper:
+```
+train_args = {
+    'learning_rate': 1e-5,
+    'max_seq_length': 512,
+    'overwrite_output_dir': True,
+    'reprocess_input_data': False,
+    'train_batch_size': 4,
+    'num_train_epochs': 4,
+    'gradient_accumulation_steps': 2,
+    'no_cache': True,
+    'use_cached_eval_features': False,
+    'save_model_every_epoch': False,
+    'output_dir': "bart-squadv2",
+    'eval_batch_size': 8,
+    'fp16_opt_level': 'O2',
+    }
+```
+
+## Results
+```{"accuracy": 90.48%}```
+## Model in Action  🚀
+```python3
+from simpletransformers.classification import ClassificationModel
+
+model = ClassificationModel('roberta', 'a-ware/roberta-large-squadv2', num_labels=2, args=train_args)
+
+predictions, raw_outputs = model.predict([["my dog is an year old. he loves to go into the rain", "how old is my dog ?"]])
+print(predictions)
+==> [1]
+```
+
+> Created with ❤️ by A-ware UG [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/aware-ai)

model_cards/a-ware/xlmroberta-squadv2/README.md

@@ -0,0 +1,59 @@
+---
+datasets:
+- squad_v2
+---
+
+# XLM-ROBERTA-LARGE finetuned on SQuADv2
+
+This is xlm-roberta-large model finetuned on SQuADv2 dataset for question answering task
+
+## Model details
+XLM-Roberta was propsed in the [paper](https://arxiv.org/pdf/1911.02116.pdf) **XLM-R: State-of-the-art cross-lingual understanding through self-supervision
+
+## Model training
+This model was trained with following parameters using simpletransformers wrapper:
+```
+train_args = {
+    'learning_rate': 1e-5,
+    'max_seq_length': 512,
+    'doc_stride': 512,
+    'overwrite_output_dir': True,
+    'reprocess_input_data': False,
+    'train_batch_size': 8,
+    'num_train_epochs': 2,
+    'gradient_accumulation_steps': 2,
+    'no_cache': True,
+    'use_cached_eval_features': False,
+    'save_model_every_epoch': False,
+    'output_dir': "bart-squadv2",
+    'eval_batch_size': 32,
+    'fp16_opt_level': 'O2',
+    }
+```
+
+## Results
+```{"correct": 6961, "similar": 4359, "incorrect": 553, "eval_loss": -12.177856394381962}```
+
+## Model in Action  🚀
+```python3
+from transformers import XLMRobertaTokenizer, XLMRobertaForQuestionAnswering
+import torch
+
+tokenizer = XLMRobertaTokenizer.from_pretrained('a-ware/xlmroberta-squadv2')
+model = XLMRobertaForQuestionAnswering.from_pretrained('a-ware/xlmroberta-squadv2')
+
+question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
+encoding = tokenizer.encode_plus(question, text, return_tensors='pt')
+input_ids = encoding['input_ids']
+attention_mask = encoding['attention_mask']
+
+start_scores, end_scores = model(input_ids, attention_mask=attention_mask, output_attentions=False)[:2]
+
+all_tokens = tokenizer.convert_ids_to_tokens(input_ids[0])
+answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])
+answer = tokenizer.convert_tokens_to_ids(answer.split())
+answer = tokenizer.decode(answer)
+#answer => 'a nice puppet' 
+```
+
+> Created with ❤️ by A-ware UG [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/aware-ai)

model_cards/ahotrod/electra_large_discriminator_squad2_512/README.md

@@ -0,0 +1,58 @@
+## ELECTRA_large_discriminator language model fine-tuned on SQuAD2.0
+
+### with the following results:
+
+```
+  "exact": 87.09677419354838,
+  "f1": 89.98343832723452,
+  "total": 11873,
+  "HasAns_exact": 84.66599190283401,
+  "HasAns_f1": 90.44759839056285,
+  "HasAns_total": 5928,
+  "NoAns_exact": 89.52060555088309,
+  "NoAns_f1": 89.52060555088309,
+  "NoAns_total": 5945,
+  "best_exact": 87.09677419354838,
+  "best_exact_thresh": 0.0,
+  "best_f1": 89.98343832723432,
+  "best_f1_thresh": 0.0
+```
+### from script:
+```
+python ${EXAMPLES}/run_squad.py \
+  --model_type electra \
+  --model_name_or_path google/electra-large-discriminator \
+  --do_train \
+  --do_eval \
+  --train_file ${SQUAD}/train-v2.0.json \
+  --predict_file ${SQUAD}/dev-v2.0.json \
+  --version_2_with_negative \
+  --do_lower_case \
+  --num_train_epochs 3 \
+  --warmup_steps 306 \
+  --weight_decay 0.01 \
+  --learning_rate 3e-5 \
+  --max_grad_norm 0.5 \
+  --adam_epsilon 1e-6 \
+  --max_seq_length 512 \
+  --doc_stride 128 \
+  --per_gpu_train_batch_size 8 \
+  --gradient_accumulation_steps 16 \
+  --per_gpu_eval_batch_size 128 \
+  --fp16 \
+  --fp16_opt_level O1 \
+  --threads 12 \
+  --logging_steps 50 \
+  --save_steps 1000 \
+  --overwrite_output_dir \
+  --output_dir ${MODEL_PATH}
+```
+### using the following system & software:
+```
+Transformers: 2.11.0
+PyTorch: 1.5.0
+TensorFlow: 2.2.0
+Python: 3.8.1
+OS/Platform: Linux-5.3.0-59-generic-x86_64-with-glibc2.10
+CPU/GPU: Intel i9-9900K / NVIDIA Titan RTX 24GB
+```

model_cards/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616/README.md

@@ -0,0 +1,20 @@
+# BERT L-10 H-512 fine-tuned on MLM (CORD-19 2020/06/16)
+
+BERT model with [10 Transformer layers and hidden embedding of size 512](https://huggingface.co/google/bert_uncased_L-10_H-512_A-8), referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962), fine-tuned for MLM on CORD-19 dataset (as released on 2020/06/16).
+
+## Training the model
+
+```bash
+python run_language_modeling.py
+    --model_type bert
+    --model_name_or_path google/bert_uncased_L-10_H-512_A-8
+    --do_train
+    --train_data_file {cord19-200616-dataset}
+    --mlm
+    --mlm_probability 0.2
+    --line_by_line
+    --block_size 512
+    --per_device_train_batch_size 10
+    --learning_rate 3e-5
+    --num_train_epochs 2
+    --output_dir bert_uncased_L-10_H-512_A-8_cord19-200616

model_cards/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2/README.md

@@ -0,0 +1,26 @@
+---
+datasets:
+- squad_v2
+---
+
+# BERT L-10 H-512 CORD-19 (2020/06/16) fine-tuned on SQuAD v2.0
+
+BERT model with [10 Transformer layers and hidden embedding of size 512](https://huggingface.co/google/bert_uncased_L-10_H-512_A-8), referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962), [fine-tuned for MLM](https://huggingface.co/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616) on CORD-19 dataset (as released on 2020/06/16) and fine-tuned for QA on SQuAD v2.0.
+
+## Training the model
+
+```bash
+python run_squad.py
+    --model_type bert
+    --model_name_or_path aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616
+    --train_file 'train-v2.0.json'
+    --predict_file 'dev-v2.0.json'
+    --do_train
+    --do_eval
+    --do_lower_case
+    --version_2_with_negative
+    --max_seq_length 384
+    --per_gpu_train_batch_size 10
+    --learning_rate 3e-5
+    --num_train_epochs 2
+    --output_dir bert_uncased_L-10_H-512_A-8_cord19-200616_squad2

model_cards/aodiniz/bert_uncased_L-2_H-512_A-8_cord19-200616/README.md

@@ -0,0 +1,20 @@
+# BERT L-2 H-512 fine-tuned on MLM (CORD-19 2020/06/16)
+
+BERT model with [2 Transformer layers and hidden embedding of size 512](https://huggingface.co/google/bert_uncased_L-2_H-512_A-8), referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962), fine-tuned for MLM on CORD-19 dataset (as released on 2020/06/16).
+
+## Training the model
+
+```bash
+python run_language_modeling.py
+    --model_type bert
+    --model_name_or_path google/bert_uncased_L-2_H-512_A-8
+    --do_train
+    --train_data_file {cord19-200616-dataset}
+    --mlm
+    --mlm_probability 0.2
+    --line_by_line
+    --block_size 512
+    --per_device_train_batch_size 20
+    --learning_rate 3e-5
+    --num_train_epochs 2
+    --output_dir bert_uncased_L-2_H-512_A-8_cord19-200616

model_cards/aodiniz/bert_uncased_L-4_H-256_A-4_cord19-200616/README.md

@@ -0,0 +1,20 @@
+# BERT L-4 H-256 fine-tuned on MLM (CORD-19 2020/06/16)
+
+BERT model with [4 Transformer layers and hidden embedding of size 256](https://huggingface.co/google/bert_uncased_L-4_H-256_A-4), referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962), fine-tuned for MLM on CORD-19 dataset (as released on 2020/06/16).
+
+## Training the model
+
+```bash
+python run_language_modeling.py
+    --model_type bert
+    --model_name_or_path google/bert_uncased_L-4_H-256_A-4
+    --do_train
+    --train_data_file {cord19-200616-dataset}
+    --mlm
+    --mlm_probability 0.2
+    --line_by_line
+    --block_size 256
+    --per_device_train_batch_size 20
+    --learning_rate 3e-5
+    --num_train_epochs 2
+    --output_dir bert_uncased_L-4_H-256_A-4_cord19-200616

model_cards/canwenxu/BERT-of-Theseus-MNLI/README.md

@@ -20,3 +20,5 @@ We provide a 6-layer pretrained model on MNLI as a general-purpose model, which
 | BERT-base       | 83.5 | 89.5 | 91.2 | 89.8 | 71.1 | 91.5  | 88.9  |
 | DistillBERT     | 79.0 | 87.5 | 85.3 | 84.9 | 59.9 | 90.7  | 81.2  |
 | BERT-of-Theseus | 82.1 | 87.5 | 88.8 | 88.8 | 70.1 | 91.8  | 87.8  |
+
+Please Note: this checkpoint is for [Intermediate-Task Transfer Learning](https://arxiv.org/abs/2005.00628) so it does not include the classification head for MNLI! Please fine-tune it before use (like DistilBERT).

model_cards/deepset/roberta-base-squad2-covid/README.md

@@ -4,7 +4,7 @@
 **Language model:** deepset/roberta-base-squad2  
 **Language:** English  
 **Downstream-task:** Extractive QA  
-**Training data:** [SQuAD-style CORD-19 annotations](https://github.com/deepset-ai/COVID-QA/tree/master/data/question-answering)  
+**Training data:** [SQuAD-style CORD-19 annotations from 23rd April](https://github.com/deepset-ai/COVID-QA/blob/master/data/question-answering/200423_covidQA.json)  
 **Code:**  See [example](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering_crossvalidation.py) in [FARM](https://github.com/deepset-ai/FARM)  
 **Infrastructure**: Tesla v100
 

model_cards/fran-martinez/scibert_scivocab_cased_ner_jnlpba/README.md

@@ -0,0 +1,161 @@
+---
+language: scientific english
+---
+
+# SciBERT finetuned on JNLPA for NER downstream task
+## Language Model
+ [SciBERT](https://arxiv.org/pdf/1903.10676.pdf) is a pretrained language model based on BERT and trained by the 
+ [Allen Institute for AI](https://allenai.org/) on papers from the corpus of 
+ [Semantic Scholar](https://www.semanticscholar.org/). 
+ Corpus size is 1.14M papers, 3.1B tokens. SciBERT has its own vocabulary (scivocab) that's built to best match 
+ the training corpus.
+ 
+## Downstream task
+[`allenai/scibert_scivocab_cased`](https://huggingface.co/allenai/scibert_scivocab_cased#) has been finetuned for Named Entity 
+Recognition (NER) dowstream task. The code to train the NER can be found [here](https://github.com/fran-martinez/bio_ner_bert).
+
+### Data
+The corpus used to fine-tune the NER is [BioNLP / JNLPBA shared task](http://www.geniaproject.org/shared-tasks/bionlp-jnlpba-shared-task-2004).
+
+- Training data consist of 2,000 PubMed abstracts with term/word annotation. This corresponds to 18,546 samples (senteces).
+- Evaluation data consist of 404 PubMed abstracts with term/word annotation. This corresponds to 3,856 samples (sentences).
+
+The classes (at word level) and its distribution (number of examples for each class) for training and evaluation datasets are shown below:
+ 
+| Class Label         | # training examples| # evaluation examples|
+|:--------------|--------------:|----------------:|
+|O              |   382,963     |     81,647      |
+|B-protein      |    30,269     |      5,067      |
+|I-protein      |    24,848     |      4,774      |
+|B-cell_type    |     6,718     |      1,921      |
+|I-cell_type    |     8,748     |      2,991      |
+|B-DNA          |     9,533     |      1,056      |
+|I-DNA          |    15,774     |      1,789      |
+|B-cell_line    |     3,830     |        500      |
+|I-cell_line    |     7,387     |       9,89      |
+|B-RNA          |       951     |        118      |
+|I-RNA          |     1,530     |        187      |
+
+### Model
+An exhaustive hyperparameter search was done.
+The hyperparameters that provided the best results are:
+
+- Max length sequence: 128
+- Number of epochs: 6
+- Batch size: 32
+- Dropout: 0.3
+- Optimizer: Adam
+
+The used learning rate was 5e-5 with a decreasing linear schedule. A warmup was used at the beggining of the training
+with a ratio of steps equal to 0.1 from the total training steps.
+
+The model from the epoch with the best F1-score was selected, in this case, the model from epoch 5.
+
+
+### Evaluation
+The following table shows the evaluation metrics calculated at span/entity level:
+
+|          |   precision|    recall|  f1-score|   
+|:---------|-----------:|---------:|---------:|
+cell_line   |  0.5205   | 0.7100   | 0.6007   | 
+cell_type   |  0.7736   | 0.7422   | 0.7576   |
+protein     |  0.6953   | 0.8459   | 0.7633   |
+DNA         |  0.6997   | 0.7894   | 0.7419   | 
+RNA         |  0.6985   | 0.8051   | 0.7480   | 
+|           |          |          |
+**micro avg**   |  0.6984   | 0.8076  |  0.7490|
+**macro avg**   | 0.7032   | 0.8076   | 0.7498 |
+
+The macro F1-score is equal to 0.7498, compared to the value provided by the Allen Institute for AI in their
+[paper](https://arxiv.org/pdf/1903.10676.pdf), which is equal to 0.7728. This drop in performance could be due to 
+several reasons, but one hypothesis could be the fact that the authors used an additional conditional random field, 
+while this model uses a regular classification layer with softmax activation on top of SciBERT model.
+
+At word level, this model achieves a precision of 0.7742, a recall of 0.8536 and a F1-score of 0.8093.
+
+### Model usage in inference
+Use the pipeline:
+````python
+from transformers import pipeline
+
+text = "Mouse thymus was used as a source of glucocorticoid receptor from normal CS lymphocytes."
+
+nlp_ner = pipeline("ner",
+                   model='fran-martinez/scibert_scivocab_cased_ner_jnlpba',
+                   tokenizer='fran-martinez/scibert_scivocab_cased_ner_jnlpba')
+
+nlp_ner(text)
+
+"""
+Output:
+---------------------------
+[
+{'word': 'glucocorticoid', 
+'score': 0.9894881248474121, 
+'entity': 'B-protein'}, 
+ 
+{'word': 'receptor', 
+'score': 0.989505410194397, 
+'entity': 'I-protein'}, 
+
+{'word': 'normal', 
+'score': 0.7680378556251526, 
+'entity': 'B-cell_type'}, 
+
+{'word': 'cs', 
+'score': 0.5176806449890137, 
+'entity': 'I-cell_type'}, 
+
+{'word': 'lymphocytes', 
+'score': 0.9898491501808167, 
+'entity': 'I-cell_type'}
+]
+"""
+````
+Or load model and tokenizer as follows:
+````python
+import torch
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+
+# Example
+text = "Mouse thymus was used as a source of glucocorticoid receptor from normal CS lymphocytes."
+
+# Load model
+tokenizer = AutoTokenizer.from_pretrained("fran-martinez/scibert_scivocab_cased_ner_jnlpba")
+model = AutoModelForTokenClassification.from_pretrained("fran-martinez/scibert_scivocab_cased_ner_jnlpba")
+
+# Get input for BERT
+input_ids = torch.tensor(tokenizer.encode(text)).unsqueeze(0)
+
+# Predict
+with torch.no_grad():
+  outputs = model(input_ids)
+
+# From the output let's take the first element of the tuple.
+# Then, let's get rid of [CLS] and [SEP] tokens (first and last)
+predictions = outputs[0].argmax(axis=-1)[0][1:-1]
+
+# Map label class indexes to string labels.
+for token, pred in zip(tokenizer.tokenize(text), predictions):
+  print(token, '->', model.config.id2label[pred.numpy().item()])
+
+"""
+Output:
+---------------------------
+mouse -> O
+thymus -> O
+was -> O
+used -> O
+as -> O
+a -> O
+source -> O
+of -> O
+glucocorticoid -> B-protein
+receptor -> I-protein
+from -> O
+normal -> B-cell_type
+cs -> I-cell_type
+lymphocytes -> I-cell_type
+. -> O
+"""
+````

model_cards/giganticode/StackOBERTflow-comments-small-v1/README.md

@@ -0,0 +1,39 @@
+# StackOBERTflow-comments-small
+
+StackOBERTflow is a RoBERTa model trained on StackOverflow comments.
+A Byte-level BPE tokenizer with dropout was used (using the `tokenizers` package).
+
+The model is *small*, i.e. has only 6-layers and the maximum sequence length was restricted to 256 tokens. 
+The model was trained for 6 epochs on several GBs of comments from the StackOverflow corpus.
+
+## Quick start: masked language modeling prediction
+
+```python
+from transformers import pipeline
+from pprint import pprint
+
+COMMENT = "You really should not do it this way, I would use <mask> instead."
+
+fill_mask = pipeline(
+    "fill-mask",
+    model="giganticode/StackOBERTflow-comments-small-v1",
+    tokenizer="giganticode/StackOBERTflow-comments-small-v1"
+)
+
+pprint(fill_mask(COMMENT))
+# [{'score': 0.019997311756014824,
+#   'sequence': '<s> You really should not do it this way, I would use jQuery instead.</s>',
+#   'token': 1738},
+#  {'score': 0.01693696901202202,
+#   'sequence': '<s> You really should not do it this way, I would use arrays instead.</s>',
+#   'token': 2844},
+#  {'score': 0.013411642983555794,
+#   'sequence': '<s> You really should not do it this way, I would use CSS instead.</s>',
+#   'token': 2254},
+#  {'score': 0.013224546797573566,
+#   'sequence': '<s> You really should not do it this way, I would use it instead.</s>',
+#   'token': 300},
+#  {'score': 0.011984303593635559,
+#   'sequence': '<s> You really should not do it this way, I would use classes instead.</s>',
+#   'token': 1779}]
+```

model_cards/google/mobilebert-uncased/README.md

@@ -0,0 +1,32 @@
+---
+language: english
+thumbnail: https://huggingface.co/front/thumbnails/google.png
+
+license: apache-2.0
+---
+
+## MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices
+
+MobileBERT is a thin version of BERT_LARGE, while equipped with bottleneck structures and a carefully designed balance
+between self-attentions and feed-forward networks.
+
+This checkpoint is the original MobileBert Optimized Uncased English: 
+[uncased_L-24_H-128_B-512_A-4_F-4_OPT](https://storage.googleapis.com/cloud-tpu-checkpoints/mobilebert/uncased_L-24_H-128_B-512_A-4_F-4_OPT.tar.gz) 
+checkpoint.
+
+## How to use MobileBERT in `transformers`
+
+```python
+from transformers import pipeline
+
+fill_mask = pipeline(
+	"fill-mask",
+	model="google/mobilebert-uncased",
+	tokenizer="google/mobilebert-uncased"
+)
+
+print(
+	fill_mask(f"HuggingFace is creating a {fill_mask.tokenizer.mask_token} that the community uses to solve NLP tasks.")
+)
+
+```

model_cards/google/reformer-crime-and-punishment/README.md

@@ -10,8 +10,8 @@ The ReformerLM model was trained in flax using colab notebook proposed by author
 The model is a language model that operates on small sub-word units. Text can be generated as follows:
 
 ```python
-model = ReformerModelWithLMHead.from_pretrained("patrickvonplaten/reformer-crime-and-punish")
-tok = ReformerTokenizer.from_pretrained("patrickvonplaten/reformer-crime-and-punish")
+model = ReformerModelWithLMHead.from_pretrained("google/reformer-crime-and-punishment")
+tok = ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment")
 tok.decode(model.generate(tok.encode("A few months later", return_tensors="pt"), do_sample=True,temperature=0.7, max_length=100)[0])
 
 # gives:'A few months later on was more than anything in the flat. 

model_cards/microsoft/MiniLM-L12-H384-uncased/README.md

@@ -0,0 +1,43 @@
+---
+thumbnail: https://huggingface.co/front/thumbnails/microsoft.png
+tags:
+- text-classification
+license: mit
+---
+
+## MiniLM: Small and Fast Pre-trained Models for Language Understanding and Generation
+
+MiniLM is a distilled model from the paper "[MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers](https://arxiv.org/abs/2002.10957)".
+
+Please find the information about preprocessing, training and full details of the MiniLM in the [original MiniLM repository](https://github.com/microsoft/unilm/blob/master/minilm/).
+
+Please note: This checkpoint can be an inplace substitution for BERT and it needs to be fine-tuned before use!
+
+### English Pre-trained Models
+We release the **uncased** **12**-layer model with **384** hidden size distilled from an in-house pre-trained [UniLM v2](/unilm) model in BERT-Base size.
+
+- MiniLMv1-L12-H384-uncased: 12-layer, 384-hidden, 12-heads, 33M parameters, 2.7x faster than BERT-Base
+
+#### Fine-tuning on NLU tasks
+
+We present the dev results on SQuAD 2.0 and several GLUE benchmark tasks.
+
+| Model                                             | #Param | SQuAD 2.0 | MNLI-m | SST-2 | QNLI | CoLA | RTE  | MRPC | QQP  |
+|---------------------------------------------------|--------|-----------|--------|-------|------|------|------|------|------|
+| [BERT-Base](https://arxiv.org/pdf/1810.04805.pdf) | 109M   | 76.8      | 84.5   | 93.2  | 91.7 | 58.9 | 68.6 | 87.3 | 91.3 |
+| **MiniLM-L12xH384**                               | 33M    | 81.7      | 85.7   | 93.0  | 91.5 | 58.5 | 73.3 | 89.5 | 91.3 |
+
+### Citation
+
+If you find MiniLM useful in your research, please cite the following paper:
+
+``` latex
+@misc{wang2020minilm,
+    title={MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers},
+    author={Wenhui Wang and Furu Wei and Li Dong and Hangbo Bao and Nan Yang and Ming Zhou},
+    year={2020},
+    eprint={2002.10957},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+```

model_cards/microsoft/Multilingual-MiniLM-L12-H384/README.md

@@ -0,0 +1,89 @@
+---
+thumbnail: https://huggingface.co/front/thumbnails/microsoft.png
+tags:
+- text-classification
+license: mit
+---
+
+## MiniLM: Small and Fast Pre-trained Models for Language Understanding and Generation
+
+MiniLM is a distilled model from the paper "[MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers](https://arxiv.org/abs/2002.10957)".
+
+Please find the information about preprocessing, training and full details of the MiniLM in the [original MiniLM repository](https://github.com/microsoft/unilm/blob/master/minilm/).
+
+Please note: This checkpoint uses `BertModel` with `XLMRobertaTokenizer` so `AutoTokenizer` won't work with this checkpoint!
+
+### Multilingual Pretrained Model
+- Multilingual-MiniLMv1-L12-H384: 12-layer, 384-hidden, 12-heads, 21M Transformer parameters, 96M embedding parameters
+
+Multilingual MiniLM uses the same tokenizer as XLM-R. But the Transformer architecture of our model is the same as BERT. We provide the fine-tuning code on XNLI based on [huggingface/transformers](https://github.com/huggingface/transformers). Please replace `run_xnli.py` in transformers with [ours](https://github.com/microsoft/unilm/blob/master/minilm/examples/run_xnli.py) to fine-tune multilingual MiniLM.  
+
+We evaluate the multilingual MiniLM on cross-lingual natural language inference benchmark (XNLI) and cross-lingual question answering benchmark (MLQA).
+
+#### Cross-Lingual Natural Language Inference - [XNLI](https://arxiv.org/abs/1809.05053)
+
+We evaluate our model on cross-lingual transfer from English to other languages. Following [Conneau et al. (2019)](https://arxiv.org/abs/1911.02116), we select the best single model on the joint dev set of all the languages.
+
+| Model                                                                                       | #Layers | #Hidden | #Transformer Parameters | Average | en   | fr   | es   | de   | el   | bg   | ru   | tr   | ar   | vi   | th   | zh   | hi   | sw   | ur   |
+|---------------------------------------------------------------------------------------------|---------|---------|-------------------------|---------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|
+| [mBERT](https://github.com/google-research/bert)                                            | 12      | 768     | 85M                     | 66.3    | 82.1 | 73.8 | 74.3 | 71.1 | 66.4 | 68.9 | 69.0 | 61.6 | 64.9 | 69.5 | 55.8 | 69.3 | 60.0 | 50.4 | 58.0 |
+| [XLM-100](https://github.com/facebookresearch/XLM#pretrained-cross-lingual-language-models) | 16      | 1280    | 315M                    | 70.7    | 83.2 | 76.7 | 77.7 | 74.0 | 72.7 | 74.1 | 72.7 | 68.7 | 68.6 | 72.9 | 68.9 | 72.5 | 65.6 | 58.2 | 62.4 |
+| [XLM-R Base](https://arxiv.org/abs/1911.02116)                                              | 12      | 768     | 85M                     | 74.5    | 84.6 | 78.4 | 78.9 | 76.8 | 75.9 | 77.3 | 75.4 | 73.2 | 71.5 | 75.4 | 72.5 | 74.9 | 71.1 | 65.2 | 66.5 |
+| **mMiniLM-L12xH384**                                                                        | 12      | 384     | 21M                     | 71.1    | 81.5 | 74.8 | 75.7 | 72.9 | 73.0 | 74.5 | 71.3 | 69.7 | 68.8 | 72.1 | 67.8 | 70.0 | 66.2 | 63.3 | 64.2 |
+
+This example code fine-tunes **12**-layer multilingual MiniLM on XNLI.
+
+```bash
+# run fine-tuning on XNLI
+DATA_DIR=/{path_of_data}/
+OUTPUT_DIR=/{path_of_fine-tuned_model}/
+MODEL_PATH=/{path_of_pre-trained_model}/
+
+python ./examples/run_xnli.py --model_type minilm \
+ --output_dir ${OUTPUT_DIR} --data_dir ${DATA_DIR} \
+ --model_name_or_path microsoft/Multilingual-MiniLM-L12-H384 \
+ --tokenizer_name xlm-roberta-base \
+ --config_name ${MODEL_PATH}/multilingual-minilm-l12-h384-config.json \
+ --do_train \
+ --do_eval \
+ --max_seq_length 128 \
+ --per_gpu_train_batch_size 128 \
+ --learning_rate 5e-5 \
+ --num_train_epochs 5 \
+ --per_gpu_eval_batch_size 32 \
+ --weight_decay 0.001 \
+ --warmup_steps 500 \
+ --save_steps 1500 \
+ --logging_steps 1500 \
+ --eval_all_checkpoints \
+ --language en \
+ --fp16 \
+ --fp16_opt_level O2
+```
+
+#### Cross-Lingual Question Answering - [MLQA](https://arxiv.org/abs/1910.07475)
+
+Following [Lewis et al. (2019b)](https://arxiv.org/abs/1910.07475), we adopt SQuAD 1.1 as training data and use MLQA English development data for early stopping.
+
+| Model F1 Score                                                                             | #Layers | #Hidden | #Transformer Parameters | Average | en   | es   | de   | ar   | hi   | vi   | zh   |
+|--------------------------------------------------------------------------------------------|---------|---------|-------------------------|---------|------|------|------|------|------|------|------|
+| [mBERT](https://github.com/google-research/bert)                                           | 12      | 768     | 85M                     | 57.7    | 77.7 | 64.3 | 57.9 | 45.7 | 43.8 | 57.1 | 57.5 |
+| [XLM-15](https://github.com/facebookresearch/XLM#pretrained-cross-lingual-language-models) | 12      | 1024    | 151M                    | 61.6    | 74.9 | 68.0 | 62.2 | 54.8 | 48.8 | 61.4 | 61.1 |
+| [XLM-R Base](https://arxiv.org/abs/1911.02116) (Reported)                                  | 12      | 768     | 85M                     | 62.9    | 77.8 | 67.2 | 60.8 | 53.0 | 57.9 | 63.1 | 60.2 |
+| [XLM-R Base](https://arxiv.org/abs/1911.02116) (Our fine-tuned)                            | 12      | 768     | 85M                     | 64.9    | 80.3 | 67.0 | 62.7 | 55.0 | 60.4 | 66.5 | 62.3 |
+| **mMiniLM-L12xH384**                                                                       | 12      | 384     | 21M                     | 63.2    | 79.4 | 66.1 | 61.2 | 54.9 | 58.5 | 63.1 | 59.0 |
+
+### Citation
+
+If you find MiniLM useful in your research, please cite the following paper:
+
+``` latex
+@misc{wang2020minilm,
+    title={MiniLM: Deep Self-Attention Distillation for Task-Agnostic Compression of Pre-Trained Transformers},
+    author={Wenhui Wang and Furu Wei and Li Dong and Hangbo Bao and Nan Yang and Ming Zhou},
+    year={2020},
+    eprint={2002.10957},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+```

model_cards/mrm8488/t5-base-finetuned-emotion/README.md

@@ -0,0 +1,81 @@
+--
+language: english
+---
+
+# T5-base fine-tuned for Emotion Recognition 😂😢😡😃😯
+
+
+[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) base fine-tuned on [emotion recognition](https://github.com/dair-ai/emotion_dataset) dataset for **Emotion Recognition** downstream task.
+
+## Details of T5
+
+The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract:
+
+Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
+
+![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67)
+
+## Details of the downstream task (Sentiment Recognition) - Dataset 📚
+
+[Elvis Saravia](https://twitter.com/omarsar0) has gathered a great [dataset](https://github.com/dair-ai/emotion_dataset) for emotion recognition. It allows to classifiy the text into one of the following **6** emotions:
+
+ - sadness 😢
+ - joy 😃
+ - love 🥰
+ - anger 😡
+ - fear 😱
+ - surprise 😯
+
+## Model fine-tuning 🏋️‍
+
+The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him!
+
+## Test set metrics 🧾
+
+               |precision | recall  | f1-score |support|
+    |----------|----------|---------|----------|-------|
+    |anger     |      0.93|     0.92|      0.93|    275|
+    |fear      |      0.91|     0.87|      0.89|    224|
+    |joy       |      0.97|     0.94|      0.95|    695|
+    |love      |      0.80|     0.91|      0.85|    159|
+    |sadness   |      0.97|     0.97|      0.97|    521|
+    |surpirse  |      0.73|     0.89|      0.80|     66|
+    |----------|----------|---------|----------|-------|
+    |accuracy|            |         |      0.93|   2000|
+    |macro avg|       0.89|     0.92|      0.90|   2000|
+    |weighted avg|    0.94|     0.93|      0.93|   2000|
+    
+    
+                 
+    
+
+
+
+## Model in Action 🚀
+
+```python
+from transformers import AutoTokenizer, AutoModelWithLMHead
+
+tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-emotion")
+
+model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-emotion")
+
+def get_emotion(text):
+  input_ids = tokenizer.encode(text + '</s>', return_tensors='pt')
+
+  output = model.generate(input_ids=input_ids,
+               max_length=2)
+  
+  dec = [tokenizer.decode(ids) for ids in output]
+  label = dec[0]
+  return label
+  
+ get_emotion("i feel as if i havent blogged in ages are at least truly blogged i am doing an update cute") # Output: 'joy'
+ 
+ get_emotion("i have a feeling i kinda lost my best friend") # Output: 'sadness'
+
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/t5-base-finetuned-imdb-sentiment/README.md

@@ -0,0 +1,68 @@
+--
+language: english
+datasets:
+- imdb
+---
+
+# T5-base fine-tuned for Sentiment Anlalysis 🎞️👍👎
+
+
+[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) base fine-tuned on [IMDB](https://huggingface.co/datasets/imdb) dataset for **Sentiment Analysis** downstream task.
+
+## Details of T5
+
+The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract:
+
+Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
+
+![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67)
+
+## Details of the downstream task (Sentiment analysis) - Dataset 📚
+
+[IMDB](https://huggingface.co/datasets/imdb)
+
+This is a dataset for binary sentiment classification containing substantially more data than previous benchmark datasets. We provide a set of **25,000** highly polar movie reviews for training, and **25,000** for testing.
+
+## Model fine-tuning 🏋️‍
+
+The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him!
+
+## Test set metrics 🧾
+
+               |precision | recall  | f1-score |support|
+    |----------|----------|---------|----------|-------|
+    |negative  |     0.95 |     0.95|      0.95|  12500|
+    |positive  |     0.95 |     0.95|      0.95|  12500|
+    |----------|----------|---------|----------|-------|
+    |accuracy|            |         |      0.95|  25000|
+    |macro avg|       0.95|     0.95|      0.95|  25000|
+    |weighted avg|    0.95|     0.95|     0.95 |  25000|
+
+
+## Model in Action 🚀
+
+```python
+from transformers import AutoTokenizer, AutoModelWithLMHead
+
+tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-imdb-sentiment")
+
+model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-imdb-sentiment")
+
+def get_sentiment(text):
+  input_ids = tokenizer.encode(text + '</s>', return_tensors='pt')
+
+  output = model.generate(input_ids=input_ids,
+               max_length=2)
+  
+  dec = [tokenizer.decode(ids) for ids in output]
+  label = dec[0]
+  return label
+  
+get_sentiment("I dislike a lot that film")
+
+# Output: 'negative'
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/nyu-mll/roberta-base-100M-1/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-base-100M-2/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-base-100M-3/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-base-10M-1/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-base-10M-2/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-base-10M-3/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-base-1B-1/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-base-1B-2/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-base-1B-3/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-med-small-1M-1/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-med-small-1M-2/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta-med-small-1M-3/README.md

@@ -0,0 +1 @@
+../roberta_1M_to_1B/README.md

model_cards/nyu-mll/roberta_1M_to_1B/README.md

@@ -0,0 +1,49 @@
+# RoBERTa Pretrained on Smaller Datasets
+
+We pretrain RoBERTa on smaller datasets (1M, 10M, 100M, 1B tokens). We release 3 models with lowest perplexities for each pretraining data size out of 25 runs (or 10 in the case of 1B tokens). The pretraining data reproduces that of BERT: We combine English Wikipedia and a reproduction of BookCorpus using texts from smashwords in a ratio of approximately 3:1.
+
+### Hyperparameters and Validation Perplexity
+
+The hyperparameters and validation perplexities corresponding to each model are as follows:
+
+| Model Name               | Training Size | Model Size | Max Steps | Batch Size | Validation Perplexity |
+|--------------------------|---------------|------------|-----------|------------|-----------------------|
+| [roberta-base-1B-1][link-roberta-base-1B-1]        | 1B            | BASE       | 100K      | 512        | 3.93                  |
+| [roberta-base-1B-2][link-roberta-base-1B-2]        | 1B            | BASE       | 31K       | 1024       | 4.25                  |
+| [roberta-base-1B-3][link-roberta-base-1B-3]        | 1B            | BASE       | 31K       | 4096       | 3.84                  |
+| [roberta-base-100M-1][link-roberta-base-100M-1]      | 100M          | BASE       | 100K      | 512        | 4.99                  |
+| [roberta-base-100M-2][link-roberta-base-100M-2]      | 100M          | BASE       | 31K       | 1024       | 4.61                  |
+| [roberta-base-100M-3][link-roberta-base-100M-3]      | 100M          | BASE       | 31K       | 512        | 5.02                  |
+| [roberta-base-10M-1][link-roberta-base-10M-1]       | 10M           | BASE       | 10K       | 1024       | 11.31                 |
+| [roberta-base-10M-2][link-roberta-base-10M-2]       | 10M           | BASE       | 10K       | 512        | 10.78                 |
+| [roberta-base-10M-3][link-roberta-base-10M-3]       | 10M           | BASE       | 31K       | 512        | 11.58                 |
+| [roberta-med-small-1M-1][link-roberta-med-small-1M-1]   | 1M            | MED-SMALL  | 100K      | 512        | 153.38                |
+| [roberta-med-small-1M-2][link-roberta-med-small-1M-2]   | 1M            | MED-SMALL  | 10K       | 512        | 134.18                |
+| [roberta-med-small-1M-3][link-roberta-med-small-1M-3]   | 1M            | MED-SMALL  | 31K       | 512        | 139.39                |
+
+The hyperparameters corresponding to model sizes mentioned above are as follows:
+
+| Model Size | L  | AH | HS  | FFN  | P    |
+|------------|----|----|-----|------|------|
+| BASE       | 12 | 12 | 768 | 3072 | 125M |
+| MED-SMALL  | 6  | 8  | 512 | 2048 | 45M  |
+
+(AH = number of attention heads; HS = hidden size; FFN = feedforward network dimension; P = number of parameters.)
+
+For other hyperparameters, we select:
+- Peak Learning rate: 5e-4
+- Warmup Steps: 6% of max steps
+- Dropout: 0.1
+
+[link-roberta-med-small-1M-1]: https://huggingface.co/nyu-mll/roberta-med-small-1M-1
+[link-roberta-med-small-1M-2]: https://huggingface.co/nyu-mll/roberta-med-small-1M-2
+[link-roberta-med-small-1M-3]: https://huggingface.co/nyu-mll/roberta-med-small-1M-3
+[link-roberta-base-10M-1]: https://huggingface.co/nyu-mll/roberta-base-10M-1
+[link-roberta-base-10M-2]: https://huggingface.co/nyu-mll/roberta-base-10M-2
+[link-roberta-base-10M-3]: https://huggingface.co/nyu-mll/roberta-base-10M-3
+[link-roberta-base-100M-1]: https://huggingface.co/nyu-mll/roberta-base-100M-1
+[link-roberta-base-100M-2]: https://huggingface.co/nyu-mll/roberta-base-100M-2
+[link-roberta-base-100M-3]: https://huggingface.co/nyu-mll/roberta-base-100M-3
+[link-roberta-base-1B-1]: https://huggingface.co/nyu-mll/roberta-base-1B-1
+[link-roberta-base-1B-2]: https://huggingface.co/nyu-mll/roberta-base-1B-2
+[link-roberta-base-1B-3]: https://huggingface.co/nyu-mll/roberta-base-1B-3

notebooks/02-transformers.ipynb

@@ -221,7 +221,7 @@
    },
    "source": [
     "The code you saw in the previous section introduced all the steps required to do simple model invocation.\n",
-    "For more day-to-day usage, transformers provides you higher-level methods which will makes your NLP journey easier\n",
+    "For more day-to-day usage, transformers provides you higher-level methods which will makes your NLP journey easier.\n",
     "Let's improve our previous example"
    ]
   },
@@ -268,7 +268,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "As you can see above, the methode `encode_plus` provides a convenient way to generate all the required parameters\n",
+    "As you can see above, the method `encode_plus` provides a convenient way to generate all the required parameters\n",
     "that will go through the model. \n",
     "\n",
     "Moreover, you might have noticed it generated some additional tensors: \n",

notebooks/README.md

@@ -1,13 +1,13 @@
-# Transformers Notebooks
+# 🤗 Transformers Notebooks
 
 You can find here a list of the official notebooks provided by Hugging Face.
 
 Also, we would like to list here interesting content created by the community. 
-If you wrote some notebook(s) leveraging transformers and would like be listed here, please open a 
+If you wrote some notebook(s) leveraging 🤗 Transformers and would like be listed here, please open a 
 Pull Request so it can be included under the Community notebooks. 
 
 
-## Hugging Face's notebooks :hugs:
+## Hugging Face's notebooks 🤗
 
 | Notebook     |      Description      |   |
 |:----------|:-------------|------:|
@@ -37,3 +37,4 @@ Pull Request so it can be included under the Community notebooks.
 | [Fine-tune DistilBert for Multiclass Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb) | How to fine-tune DistilBert for multiclass classification with PyTorch | [Abhishek Kumar Mishra](https://github.com/abhimishra91) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb)|
 |[Fine-tune BERT for Multi-label Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|How to fine-tune BERT for multi-label classification using PyTorch|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|
 |[Fine-tune T5 for Summarization](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|How to fine-tune T5 for summarization in PyTorch and track experiments with WandB|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|
+|[Speed up Fine-Tuning in Transformers with Dynamic Padding / Bucketing](https://github.com/ELS-RD/transformers-notebook/blob/master/Divide_Hugging_Face_Transformers_training_time_by_2_or_more.ipynb)|How to speed up fine-tuning by a factor of 2 using dynamic padding / bucketing|[Michael Benesty](https://github.com/pommedeterresautee) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1CBfRU1zbfu7-ijiOqAAQUA-RJaxfcJoO?usp=sharing)|

setup.cfg

@@ -5,12 +5,15 @@ include_trailing_comma = True
 known_first_party = transformers
 known_third_party =
     absl
+    elasticsearch
     fairseq
+    faiss
     fastprogress
     git
     h5py
     matplotlib
     MeCab
+    nlp
     nltk
     numpy
     packaging

setup.py

@@ -85,7 +85,8 @@ extras["serving"] = ["pydantic", "uvicorn", "fastapi", "starlette"]
 extras["all"] = extras["serving"] + ["tensorflow", "torch"]
 
 extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "psutil"]
-extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme"]
+# sphinx-rtd-theme==0.5.0 introduced big changes in the style.
+extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3"]
 extras["quality"] = [
     "black",
     "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
@@ -108,7 +109,7 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.8.0-rc1",
+        "tokenizers == 0.8.0-rc3",
         # dataclasses for Python versions that don't have it
         "dataclasses;python_version<'3.7'",
         # utilities from PyPA to e.g. compare versions

src/transformers/__init__.py

@@ -34,6 +34,7 @@ from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config
 from .configuration_longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig
 from .configuration_marian import MarianConfig
 from .configuration_mmbt import MMBTConfig
+from .configuration_mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig
 from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
 from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig
 from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
@@ -77,6 +78,9 @@ from .file_utils import (
     add_end_docstrings,
     add_start_docstrings,
     cached_path,
+    is_apex_available,
+    is_psutil_available,
+    is_py3nvml_available,
     is_tf_available,
     is_torch_available,
     is_torch_tpu_available,
@@ -129,6 +133,7 @@ from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast
 from .tokenization_flaubert import FlaubertTokenizer
 from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
 from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast
+from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFast
 from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
 from .tokenization_reformer import ReformerTokenizer
 from .tokenization_retribert import RetriBertTokenizer, RetriBertTokenizerFast
@@ -188,6 +193,21 @@ if is_torch_available():
         MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
     )
 
+    from .modeling_mobilebert import (
+        MobileBertPreTrainedModel,
+        MobileBertModel,
+        MobileBertForPreTraining,
+        MobileBertForSequenceClassification,
+        MobileBertForQuestionAnswering,
+        MobileBertForMaskedLM,
+        MobileBertForNextSentencePrediction,
+        MobileBertForMultipleChoice,
+        MobileBertForTokenClassification,
+        load_tf_weights_in_mobilebert,
+        MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        MobileBertLayer,
+    )
+
     from .modeling_bert import (
         BertPreTrainedModel,
         BertModel,
@@ -333,6 +353,7 @@ if is_torch_available():
         ElectraForMaskedLM,
         ElectraForTokenClassification,
         ElectraPreTrainedModel,
+        ElectraForMultipleChoice,
         ElectraForSequenceClassification,
         ElectraForQuestionAnswering,
         ElectraModel,
@@ -380,7 +401,8 @@ if is_torch_available():
     from .data.datasets import GlueDataset, TextDataset, LineByLineTextDataset, GlueDataTrainingArguments
 
     # Benchmarks
-    from .benchmark import PyTorchBenchmark, PyTorchBenchmarkArguments
+    from .benchmark.benchmark import PyTorchBenchmark
+    from .benchmark.benchmark_args import PyTorchBenchmarkArguments
 
 # TensorFlow
 if is_tf_available():
@@ -494,6 +516,20 @@ if is_tf_available():
         TFGPT2PreTrainedModel,
     )
 
+    from .modeling_tf_mobilebert import (
+        TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        TFMobileBertModel,
+        TFMobileBertPreTrainedModel,
+        TFMobileBertForPreTraining,
+        TFMobileBertForSequenceClassification,
+        TFMobileBertForQuestionAnswering,
+        TFMobileBertForMaskedLM,
+        TFMobileBertForNextSentencePrediction,
+        TFMobileBertForMultipleChoice,
+        TFMobileBertForTokenClassification,
+        TFMobileBertMainLayer,
+    )
+
     from .modeling_tf_openai import (
         TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFOpenAIGPTDoubleHeadsModel,
@@ -576,6 +612,10 @@ if is_tf_available():
     # Trainer
     from .trainer_tf import TFTrainer
 
+    # Benchmarks
+    from .benchmark.benchmark_tf import TensorflowBenchmark
+    from .benchmark.benchmark_args_tf import TensorflowBenchmarkArguments
+
 
 if not is_tf_available() and not is_torch_available():
     logger.warning(

src/transformers/benchmark/__init__.py

@@ -1,10 +0,0 @@
-# flake8: noqa
-# There's no way to ignore "F401 '...' imported but unused" warnings in this
-# module, but to preserve other warnings. So, don't check this module at all.
-
-from ..file_utils import is_torch_available
-
-
-if is_torch_available():
-    from .benchmark_args import PyTorchBenchmarkArguments
-    from .benchmark import PyTorchBenchmark

src/transformers/benchmark/benchmark.py

@@ -20,16 +20,24 @@
 
 import logging
 import timeit
+from typing import Callable, Optional
 
 from transformers import (
     MODEL_MAPPING,
     MODEL_WITH_LM_HEAD_MAPPING,
     PretrainedConfig,
+    is_py3nvml_available,
     is_torch_available,
-    is_torch_tpu_available,
 )
 
-from .benchmark_utils import Benchmark, Memory, measure_peak_memory_cpu, start_memory_tracing, stop_memory_tracing
+from .benchmark_utils import (
+    Benchmark,
+    Memory,
+    MemorySummary,
+    measure_peak_memory_cpu,
+    start_memory_tracing,
+    stop_memory_tracing,
+)
 
 
 if is_torch_available():
@@ -37,6 +45,10 @@ if is_torch_available():
     from .benchmark_args import PyTorchBenchmarkArguments
 
 
+if is_py3nvml_available():
+    import py3nvml.py3nvml as nvml
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -50,220 +62,196 @@ class PyTorchBenchmark(Benchmark):
     def framework_version(self):
         return torch.__version__
 
-    def train(self, model_name, batch_size, sequence_length, trace_memory=False):
-        try:
-            config = self.config_dict[model_name]
+    def _inference_speed(self, model_name: str, batch_size: int, sequence_length: int) -> float:
+        _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
+        return self._measure_speed(_inference)
 
-            if self.args.torchscript:
-                config.torchscript = True
+    def _inference_memory(
+        self, model_name: str, batch_size: int, sequence_length: int
+    ) -> [Memory, Optional[MemorySummary]]:
+        _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
+        return self._measure_memory(_inference)
 
+    def _train_speed(self, model_name: str, batch_size: int, sequence_length: int) -> float:
+        _train = self._prepare_train_func(model_name, batch_size, sequence_length)
+        return self._measure_speed(_train)
+
+    def _train_memory(
+        self, model_name: str, batch_size: int, sequence_length: int
+    ) -> [Memory, Optional[MemorySummary]]:
+        _train = self._prepare_train_func(model_name, batch_size, sequence_length)
+        return self._measure_memory(_train)
+
+    def _prepare_inference_func(self, model_name: str, batch_size: int, sequence_length: int) -> Callable[[], None]:
+        config = self.config_dict[model_name]
+
+        if self.args.torchscript:
+            config.torchscript = True
+
+        has_model_class_in_config = hasattr(config, "architecture") and len(config.architectures) > 1
+        if not self.args.only_pretrain_model and has_model_class_in_config:
+            try:
+                model_class = config.architectures[0]
+                transformers_module = __import__("transformers", fromlist=[model_class])
+                model_cls = getattr(transformers_module, model_class)
+                model = model_cls(config)
+            except ImportError:
+                raise ImportError(
+                    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to set `--only_pretrain_model` or `args.only_pretrain_model=True`."
+                )
+        else:
+            model = MODEL_MAPPING[config.__class__](config)
+
+        model.eval()
+        model.to(self.args.device)
+
+        # encoder-decoder has vocab size saved differently
+        vocab_size = config.vocab_size if hasattr(config, "vocab_size") else config.encoder.vocab_size
+        input_ids = torch.randint(vocab_size, (batch_size, sequence_length), dtype=torch.long, device=self.args.device)
+
+        if self.args.fp16:
+            logger.info("Running training in Mixed Precision...")
+            assert self.args.is_gpu, "Mixed precision is possible only for GPU."
+            # amp seems to have memory leaks so that memory usage
+            # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
+            model.half()
+
+        if self.args.torchscript:
+            with torch.no_grad():
+                inference_model = torch.jit.trace(model, input_ids)
+        else:
+            inference_model = model
+
+        def encoder_decoder_forward():
+            with torch.no_grad():
+                outputs = inference_model(input_ids, decoder_input_ids=input_ids)
+            return outputs
+
+        def encoder_forward():
+            with torch.no_grad():
+                outputs = inference_model(input_ids)
+            return outputs
+
+        _forward = encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
+        return _forward
+
+    def _prepare_train_func(self, model_name: str, batch_size: int, sequence_length: int) -> Callable[[], None]:
+        config = self.config_dict[model_name]
+
+        has_model_class_in_config = hasattr(config, "architecture") and len(config.architectures) > 1
+        if not self.args.only_pretrain_model and has_model_class_in_config:
+            try:
+                model_class = config.architectures[0]
+                transformers_module = __import__("transformers", fromlist=[model_class])
+                model_cls = getattr(transformers_module, model_class)
+                model = model_cls(config)
+            except ImportError:
+                raise ImportError(
+                    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to set `--only_pretrain_model` or `args.only_pretrain_model=True`."
+                )
+        else:
             model = MODEL_WITH_LM_HEAD_MAPPING[config.__class__](config)
-            model.to(self.args.device)
-            model.train()
 
-            # encoder-decoder has vocab size saved differently
-            vocab_size = config.vocab_size if hasattr(config, "vocab_size") else config.encoder.vocab_size
-            input_ids = torch.randint(
-                vocab_size, (batch_size, sequence_length), dtype=torch.long, device=self.args.device
-            )
+        if self.args.torchscript:
+            raise NotImplementedError("Training for torchscript is currently not implemented")
+        else:
+            train_model = model
 
-            if self.args.torchscript:
-                raise NotImplementedError("Training for torchscript is currently not implemented")
-            else:
-                train_model = model
+        model.eval()
+        model.to(self.args.device)
 
-            def compute_loss_and_backprob_encoder():
-                loss = train_model(input_ids, labels=input_ids)[0]
-                loss.backward()
-                train_model.zero_grad()
+        # encoder-decoder has vocab size saved differently
+        vocab_size = config.vocab_size if hasattr(config, "vocab_size") else config.encoder.vocab_size
+        input_ids = torch.randint(vocab_size, (batch_size, sequence_length), dtype=torch.long, device=self.args.device)
 
-            def compute_loss_and_backprob_encoder_decoder():
-                loss = train_model(input_ids, decoder_input_ids=input_ids, labels=input_ids)[0]
-                loss.backward()
-                train_model.zero_grad()
+        if self.args.fp16:
+            logger.info("Running training in Mixed Precision...")
+            assert self.args.is_gpu, "Mixed precision is possible only for GPU."
 
-            _train = (
-                compute_loss_and_backprob_encoder_decoder
-                if config.is_encoder_decoder
-                else compute_loss_and_backprob_encoder
-            )
+            # amp seems to have memory leaks so that memory usage
+            # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
+            model.half()
 
-            if trace_memory is True:
-                if self.args.trace_memory_line_by_line:
-                    trace = start_memory_tracing("transformers")
+        def compute_loss_and_backprob_encoder():
+            loss = train_model(input_ids, labels=input_ids)[0]
+            loss.backward()
+            train_model.zero_grad()
 
-                if self.args.n_gpu > 0:
-                    # gpu
-                    # clear gpu cache
-                    torch.cuda.empty_cache()
-                    if hasattr(torch.cuda, "max_memory_reserved"):
-                        torch.cuda.reset_peak_memory_stats()
-                    else:
-                        logger.info(
-                            "Please consider updating PyTorch to version 1.4 to get more accuracy on GPU memory usage"
-                        )
-                        torch.cuda.reset_max_memory_cached()
+        def compute_loss_and_backprob_encoder_decoder():
+            loss = train_model(input_ids, decoder_input_ids=input_ids, labels=input_ids)[0]
+            loss.backward()
+            train_model.zero_grad()
 
-                    # calculate loss and do backpropagation
-                    _train()
-                elif not self.args.no_tpu and is_torch_tpu_available():
-                    # tpu
-                    raise NotImplementedError(
-                        "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with `args.no_memory=True`"
-                    )
-                else:
-                    # cpu
-                    memory_bytes = measure_peak_memory_cpu(_train)
-                    memory = Memory(memory_bytes) if isinstance(memory_bytes, int) else memory_bytes
+        _train = (
+            compute_loss_and_backprob_encoder_decoder
+            if config.is_encoder_decoder
+            else compute_loss_and_backprob_encoder
+        )
+        return _train
 
-                if self.args.trace_memory_line_by_line:
-                    summary = stop_memory_tracing(trace)
-                else:
-                    summary = None
-
-                if self.args.n_gpu > 0:
-                    # gpu
-                    if hasattr(torch.cuda, "max_memory_reserved"):
-                        memory = Memory(torch.cuda.max_memory_reserved())
-                    else:
-                        logger.info(
-                            "Please consider updating PyTorch to version 1.4 to get more accuracy on GPU memory usage"
-                        )
-                        memory = Memory(torch.cuda.max_memory_reserved())
-
-                return memory, summary
-            else:
-                if (not self.args.no_tpu and is_torch_tpu_available()) or self.args.torchscript:
-                    # run additional 10 times to stabilize compilation for tpu and torchscript
-                    logger.info("Do inference on TPU or torchscript. Running model 5 times to stabilize compilation")
-                    timeit.repeat(
-                        _train, repeat=1, number=5,
-                    )
-
-                # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
-                runtimes = timeit.repeat(_train, repeat=self.args.repeat, number=10,)
-
-                if not self.args.no_tpu and is_torch_tpu_available() and self.args.tpu_print_metrics:
-                    import torch_xla.debug.metrics as met
-
-                    self.print_fn(met.metrics_report())
-
-                return min(runtimes) / 10.0
-        except RuntimeError as e:
-            self.print_fn("Doesn't fit on GPU. {}".format(e))
-            if trace_memory:
-                return "N/A", None
-            else:
-                return "N/A"
-
-    def inference(self, model_name, batch_size, sequence_length, trace_memory=False):
+    def _measure_speed(self, func) -> float:
         try:
-            config = self.config_dict[model_name]
-            model = None
+            if self.args.is_tpu or self.args.torchscript:
+                # run additional 10 times to stabilize compilation for tpu and torchscript
+                logger.info("Do inference on TPU or torchscript. Running model 5 times to stabilize compilation")
+                timeit.repeat(
+                    func, repeat=1, number=5,
+                )
 
-            if self.args.torchscript:
-                config.torchscript = True
+            # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
+            runtimes = timeit.repeat(func, repeat=self.args.repeat, number=10,)
 
-            if self.args.with_lm_head:
-                model = MODEL_WITH_LM_HEAD_MAPPING[config.__class__](config)
-            else:
-                model = MODEL_MAPPING[config.__class__](config)
+            if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
+                import torch_xla.debug.metrics as met
 
-            model.eval()
-            model.to(self.args.device)
-
-            # encoder-decoder has vocab size saved differently
-            vocab_size = config.vocab_size if hasattr(config, "vocab_size") else config.encoder.vocab_size
-
-            input_ids = torch.randint(
-                vocab_size, (batch_size, sequence_length), dtype=torch.long, device=self.args.device
-            )
-
-            if self.args.torchscript:
-                with torch.no_grad():
-                    if config.is_encoder_decoder:
-                        raise NotImplementedError("Torchscript is currently not supported for EncoderDecoder models")
-                    else:
-                        inference_model = torch.jit.trace(model, input_ids)
-            else:
-                inference_model = model
-
-            def encoder_decoder_forward():
-                with torch.no_grad():
-                    inference_model(input_ids, decoder_input_ids=input_ids)
-
-            def encoder_forward():
-                with torch.no_grad():
-                    inference_model(input_ids)
-
-            _forward = encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
-
-            if trace_memory is True:
-                if self.args.trace_memory_line_by_line:
-                    trace = start_memory_tracing("transformers")
-
-                if self.args.n_gpu > 0:
-                    # gpu
-                    # clear gpu cache
-                    torch.cuda.empty_cache()
-                    if hasattr(torch.cuda, "max_memory_reserved"):
-                        torch.cuda.reset_peak_memory_stats()
-                    else:
-                        logger.info(
-                            "Please consider updating PyTorch to version 1.4 to get more accuracy on GPU memory usage"
-                        )
-                        torch.cuda.reset_max_memory_cached()
-
-                    # run forward
-                    _forward()
-                elif not self.args.no_tpu and is_torch_tpu_available():
-                    # tpu
-                    raise NotImplementedError(
-                        "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with `args.no_memory=True`"
-                    )
-                else:
-                    # cpu
-                    memory_bytes = measure_peak_memory_cpu(_forward)
-                    memory = Memory(memory_bytes) if isinstance(memory_bytes, int) else memory_bytes
-
-                if self.args.trace_memory_line_by_line:
-                    summary = stop_memory_tracing(trace)
-                else:
-                    summary = None
-
-                if self.args.n_gpu > 0:
-                    # gpu
-                    if hasattr(torch.cuda, "max_memory_reserved"):
-                        memory = Memory(torch.cuda.max_memory_reserved())
-                    else:
-                        logger.info(
-                            "Please consider updating PyTorch to version 1.4 to get more accuracy on GPU memory usage"
-                        )
-                        memory = Memory(torch.cuda.max_memory_cached())
-
-                return memory, summary
-            else:
-
-                if (not self.args.no_tpu and is_torch_tpu_available()) or self.args.torchscript:
-                    # run additional 10 times to stabilize compilation for tpu and torchscript
-                    logger.info("Do inference on TPU or torchscript. Running model 5 times to stabilize compilation")
-                    timeit.repeat(
-                        _forward, repeat=1, number=5,
-                    )
-
-                # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
-                runtimes = timeit.repeat(_forward, repeat=self.args.repeat, number=10,)
-
-                if not self.args.no_tpu and is_torch_tpu_available() and self.args.tpu_print_metrics:
-                    import torch_xla.debug.metrics as met
-
-                    self.print_fn(met.metrics_report())
-
-                return min(runtimes) / 10.0
+                self.print_fn(met.metrics_report())
 
+            return min(runtimes) / 10.0
         except RuntimeError as e:
             self.print_fn("Doesn't fit on GPU. {}".format(e))
-            if trace_memory:
-                return "N/A", None
+            return "N/A"
+
+    def _measure_memory(self, func: Callable[[], None]) -> [Memory, MemorySummary]:
+        try:
+            if self.args.trace_memory_line_by_line:
+                trace = start_memory_tracing("transformers")
+
+            if self.args.is_tpu:
+                # tpu
+                raise NotImplementedError(
+                    "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with `--no_memory` or `args.no_memory=True`"
+                )
+            elif self.args.is_gpu:
+                if not is_py3nvml_available():
+                    logger.warning(
+                        "py3nvml not installed, we won't log GPU memory usage. "
+                        "Install py3nvml (pip install py3nvml) to log information about GPU."
+                    )
+                    memory = "N/A"
+                else:
+                    logger.info(
+                        "Measuring total GPU usage on GPU device. Make sure to not have additional processes running on the same GPU."
+                    )
+                    # init nvml
+                    nvml.nvmlInit()
+                    func()
+                    handle = nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx)
+                    meminfo = nvml.nvmlDeviceGetMemoryInfo(handle)
+                    max_bytes_in_use = meminfo.used
+                    memory = Memory(max_bytes_in_use)
+                    # shutdown nvml
+                    nvml.nvmlShutdown()
             else:
-                return "N/A"
+                # cpu
+                memory_bytes = measure_peak_memory_cpu(func)
+                memory = Memory(memory_bytes) if isinstance(memory_bytes, int) else memory_bytes
+
+            if self.args.trace_memory_line_by_line:
+                summary = stop_memory_tracing(trace)
+            else:
+                summary = None
+
+            return memory, summary
+        except RuntimeError as e:
+            self.print_fn("Doesn't fit on GPU. {}".format(e))
+            return "N/A", None

src/transformers/benchmark/benchmark_args.py

@@ -34,11 +34,17 @@ logger = logging.getLogger(__name__)
 
 @dataclass
 class PyTorchBenchmarkArguments(BenchmarkArguments):
-    no_cuda: bool = field(default=False, metadata={"help": "Whether to run on available cuda devices"})
     torchscript: bool = field(default=False, metadata={"help": "Trace the models using torchscript"})
-    no_tpu: bool = field(default=False, metadata={"help": "Whether to run on available tpu devices"})
-    fp16: bool = field(default=False, metadata={"help": "Use FP16 to accelerate inference."})
-    tpu_print_metrics: bool = field(default=False, metadata={"help": "Use FP16 to accelerate inference."})
+    torch_xla_tpu_print_metrics: bool = field(default=False, metadata={"help": "Print Xla/PyTorch tpu metrics"})
+    fp16_opt_level: str = field(
+        default="O1",
+        metadata={
+            "help": (
+                "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+                "See details at https://nvidia.github.io/apex/amp.html"
+            )
+        },
+    )
 
     @cached_property
     @torch_required
@@ -55,9 +61,14 @@ class PyTorchBenchmarkArguments(BenchmarkArguments):
             n_gpu = torch.cuda.device_count()
         return device, n_gpu
 
+    @property
+    def is_tpu(self):
+        return is_torch_tpu_available() and not self.no_tpu
+
     @property
     @torch_required
     def device_idx(self) -> int:
+        # TODO(PVP): currently only single GPU is supported
         return torch.cuda.current_device()
 
     @property
@@ -69,3 +80,7 @@ class PyTorchBenchmarkArguments(BenchmarkArguments):
     @torch_required
     def n_gpu(self):
         return self._setup_devices[1]
+
+    @property
+    def is_gpu(self):
+        return self.n_gpu > 0

src/transformers/benchmark/benchmark_args_tf.py

@@ -0,0 +1,105 @@
+# coding=utf-8
+# Copyright 2018 The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from dataclasses import dataclass, field
+from typing import Tuple
+
+from ..file_utils import cached_property, is_tf_available, tf_required
+from .benchmark_args_utils import BenchmarkArguments
+
+
+if is_tf_available():
+    import tensorflow as tf
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class TensorflowBenchmarkArguments(BenchmarkArguments):
+    tpu_name: str = field(
+        default=None, metadata={"help": "Name of TPU"},
+    )
+    device_idx: int = field(
+        default=0, metadata={"help": "CPU / GPU device index. Defaults to 0."},
+    )
+    eager_mode: bool = field(default=False, metadata={"help": "Benchmark models in eager model."})
+    use_xla: bool = field(
+        default=False,
+        metadata={
+            "help": "Benchmark models using XLA JIT compilation. Note that `eager_model` has to be set to `False`."
+        },
+    )
+
+    @cached_property
+    @tf_required
+    def _setup_tpu(self) -> Tuple["tf.distribute.cluster_resolver.TPUClusterResolver"]:
+        if not self.no_tpu:
+            try:
+                if self.tpu_name:
+                    tpu = tf.distribute.cluster_resolver.TPUClusterResolver(self.tpu_name)
+                else:
+                    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
+            except ValueError:
+                tpu = None
+        return tpu
+
+    @cached_property
+    @tf_required
+    def _setup_strategy(self) -> Tuple["tf.distribute.Strategy", "tf.distribute.cluster_resolver.TPUClusterResolver"]:
+        if self.is_tpu:
+            tf.config.experimental_connect_to_cluster(self._setup_tpu)
+            tf.tpu.experimental.initialize_tpu_system(self._setup_tpu)
+
+            strategy = tf.distribute.experimental.TPUStrategy(self._setup_tpu)
+        else:
+            # currently no multi gpu is allowed
+            if self.is_gpu:
+                # TODO: Currently only single GPU is supported
+                tf.config.experimental.set_visible_devices(self.gpu_list[self.device_idx], "GPU")
+                strategy = tf.distribute.OneDeviceStrategy(device=f"/gpu:{self.device_idx}")
+            else:
+                tf.config.experimental.set_visible_devices([], "GPU")  # disable GPU
+                strategy = tf.distribute.OneDeviceStrategy(device=f"/cpu:{self.device_idx}")
+
+        return strategy
+
+    @property
+    @tf_required
+    def is_tpu(self) -> bool:
+        return self._setup_tpu is not None
+
+    @property
+    @tf_required
+    def strategy(self) -> "tf.distribute.Strategy":
+        return self._setup_strategy
+
+    @property
+    @tf_required
+    def gpu_list(self):
+        return tf.config.list_physical_devices("GPU")
+
+    @property
+    @tf_required
+    def n_gpu(self) -> int:
+        if not self.no_cuda:
+            return len(self.gpu_list)
+        return 0
+
+    @property
+    def is_gpu(self) -> bool:
+        return self.n_gpu > 0

src/transformers/benchmark/benchmark_args_utils.py

@@ -16,11 +16,15 @@
 
 import dataclasses
 import json
+import logging
 from dataclasses import dataclass, field
 from time import time
 from typing import List
 
 
+logger = logging.getLogger(__name__)
+
+
 def list_field(default=None, metadata=None):
     return field(default_factory=lambda: default, metadata=metadata)
 
@@ -53,6 +57,9 @@ class BenchmarkArguments:
     )
 
     no_inference: bool = field(default=False, metadata={"help": "Don't benchmark inference of model"})
+    no_cuda: bool = field(default=False, metadata={"help": "Whether to run on available cuda devices"})
+    no_tpu: bool = field(default=False, metadata={"help": "Whether to run on available tpu devices"})
+    fp16: bool = field(default=False, metadata={"help": "Use FP16 to accelerate inference."})
     training: bool = field(default=False, metadata={"help": "Benchmark training of model"})
     verbose: bool = field(default=False, metadata={"help": "Verbose memory tracing"})
     no_speed: bool = field(default=False, metadata={"help": "Don't perform speed measurments"})
@@ -61,10 +68,10 @@ class BenchmarkArguments:
     save_to_csv: bool = field(default=False, metadata={"help": "Save result to a CSV file"})
     log_print: bool = field(default=False, metadata={"help": "Save all print statements in a log file"})
     no_env_print: bool = field(default=False, metadata={"help": "Don't print environment information"})
-    with_lm_head: bool = field(
+    no_multi_process: bool = field(
         default=False,
         metadata={
-            "help": "Use model with its language model head (MODEL_WITH_LM_HEAD_MAPPING instead of MODEL_MAPPING)"
+            "help": "Don't use multiprocessing for memory and speed measurement. It is highly recommended to use multiprocessing for accurate CPU and GPU memory measurements. This option should only be used for debugging / testing and on TPU."
         },
     )
     inference_time_csv_file: str = field(
@@ -92,6 +99,12 @@ class BenchmarkArguments:
         metadata={"help": "Log filename used if print statements are saved in log."},
     )
     repeat: int = field(default=3, metadata={"help": "Times an experiment will be run."})
+    only_pretrain_model: bool = field(
+        default=False,
+        metadata={
+            "help": "Instead of loading the model as defined in `config.architectures` if exists, just load the pretrain model weights."
+        },
+    )
 
     def to_json_string(self):
         """
@@ -101,4 +114,17 @@ class BenchmarkArguments:
 
     @property
     def model_names(self):
+        assert (
+            len(self.models) > 0
+        ), "Please make sure you provide at least one model name / model identifier, *e.g.* `--models bert-base-cased` or `args.models = ['bert-base-cased']."
         return self.models
+
+    @property
+    def do_multi_processing(self):
+        if self.no_multi_process:
+            return False
+        elif self.is_tpu:
+            logger.info("Multiprocessing is currently not possible on TPU.")
+            return False
+        else:
+            return True

src/transformers/benchmark/benchmark_tf.py

@@ -0,0 +1,229 @@
+# coding=utf-8
+# Copyright 2018 The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+    Benchmarking the library on inference and training in PyTorch.
+"""
+
+
+import logging
+import random
+import timeit
+from functools import wraps
+from typing import Callable, Optional
+
+from transformers import TF_MODEL_MAPPING, PretrainedConfig, is_py3nvml_available, is_tf_available
+
+from .benchmark_utils import (
+    Benchmark,
+    Memory,
+    MemorySummary,
+    measure_peak_memory_cpu,
+    start_memory_tracing,
+    stop_memory_tracing,
+)
+
+
+if is_tf_available():
+    import tensorflow as tf
+    from .benchmark_args_tf import TensorflowBenchmarkArguments
+    from tensorflow.python.framework.errors_impl import ResourceExhaustedError
+
+if is_py3nvml_available():
+    import py3nvml.py3nvml as nvml
+
+logger = logging.getLogger(__name__)
+
+
+def run_with_tf_optimizations(do_eager_mode: bool, use_xla: bool):
+    def run_func(func):
+        @wraps(func)
+        def run_in_eager_mode(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        @wraps(func)
+        @tf.function(experimental_compile=use_xla)
+        def run_in_graph_mode(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        if do_eager_mode is True:
+            assert (
+                use_xla is False
+            ), "Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."
+            return run_in_eager_mode
+        else:
+            return run_in_graph_mode
+
+    return run_func
+
+
+def random_input_ids(batch_size: int, sequence_length: int, vocab_size: int) -> ["tf.Tensor"]:
+    rng = random.Random()
+    values = [rng.randint(0, vocab_size - 1) for i in range(batch_size * sequence_length)]
+    return tf.constant(values, shape=(batch_size, sequence_length), dtype=tf.int32)
+
+
+class TensorflowBenchmark(Benchmark):
+
+    args: TensorflowBenchmarkArguments
+    configs: PretrainedConfig
+    framework: str = "Tensorflow"
+
+    @property
+    def framework_version(self):
+        return tf.__version__
+
+    def _inference_speed(self, model_name: str, batch_size: int, sequence_length: int) -> float:
+        # initialize GPU on separate process
+        strategy = self.args.strategy
+        assert strategy is not None, "A device strategy has to be initialized before using Tensorflow."
+        _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
+        return self._measure_speed(_inference)
+
+    def _train_speed(self, model_name, batch_size, sequence_length):
+        raise NotImplementedError(
+            "Training is currently not really implemented." "Wait for TFTrainer to support CLM and MLM."
+        )
+
+    def _inference_memory(
+        self, model_name: str, batch_size: int, sequence_length: int
+    ) -> [Memory, Optional[MemorySummary]]:
+        # initialize GPU on separate process
+        if self.args.is_gpu:
+            tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx], True)
+        strategy = self.args.strategy
+        assert strategy is not None, "A device strategy has to be initialized before using Tensorflow."
+        _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
+        return self._measure_memory(_inference)
+
+    def _train_memory(self, model_name, batch_size, sequence_length):
+        raise NotImplementedError(
+            "Training is currently not really implemented. Wait for TFTrainer to support CLM and MLM."
+        )
+
+    def _prepare_inference_func(self, model_name: str, batch_size: int, sequence_length: int) -> Callable[[], None]:
+        config = self.config_dict[model_name]
+
+        if self.args.fp16:
+            raise NotImplementedError("Mixed precision is currently not supported.")
+
+        has_model_class_in_config = hasattr(config, "architecture") and len(config.architectures) > 1
+        if not self.args.only_pretrain_model and has_model_class_in_config:
+            try:
+                model_class = "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
+                transformers_module = __import__("transformers", fromlist=[model_class])
+                model_cls = getattr(transformers_module, model_class)
+                model = model_cls(config)
+            except ImportError:
+                raise ImportError(
+                    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to set `--only_pretrain_model` or `args.only_pretrain_model=True`."
+                )
+        else:
+            model = TF_MODEL_MAPPING[config.__class__](config)
+
+        # encoder-decoder has vocab size saved differently
+        vocab_size = config.vocab_size if hasattr(config, "vocab_size") else config.encoder.vocab_size
+        input_ids = random_input_ids(batch_size, sequence_length, vocab_size)
+
+        @run_with_tf_optimizations(self.args.eager_mode, self.args.use_xla)
+        def encoder_decoder_forward():
+            return model(input_ids, decoder_input_ids=input_ids, training=False)
+
+        @run_with_tf_optimizations(self.args.eager_mode, self.args.use_xla)
+        def encoder_forward():
+            return model(input_ids, training=False)
+
+        _inference = encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
+
+        return _inference
+
+    def _measure_speed(self, func) -> float:
+        with self.args.strategy.scope():
+            try:
+                if self.args.is_tpu or self.args.use_xla:
+                    # run additional 10 times to stabilize compilation for tpu
+                    logger.info("Do inference on TPU. Running model 5 times to stabilize compilation")
+                    timeit.repeat(func, repeat=1, number=5)
+
+                # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
+                runtimes = timeit.repeat(func, repeat=self.args.repeat, number=10,)
+
+                return min(runtimes) / 10.0
+            except ResourceExhaustedError as e:
+                self.print_fn("Doesn't fit on GPU. {}".format(e))
+
+    def _measure_memory(self, func: Callable[[], None]) -> [Memory, MemorySummary]:
+        logger.info(
+            "Note that Tensorflow allocates more memory than"
+            "it might need to speed up computation."
+            "The memory reported here corresponds to the memory"
+            "reported by `nvidia-smi`, which can vary depending"
+            "on total available memory on the GPU that is used."
+        )
+        with self.args.strategy.scope():
+            try:
+                if self.args.trace_memory_line_by_line:
+                    assert (
+                        self.args.eager_mode
+                    ), "`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory consumption line by line."
+                    trace = start_memory_tracing("transformers")
+
+                if self.args.is_tpu:
+                    # tpu
+                    raise NotImplementedError(
+                        "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with `args.no_memory=True`"
+                    )
+                elif self.args.is_gpu:
+                    # gpu
+                    if not is_py3nvml_available():
+                        logger.warning(
+                            "py3nvml not installed, we won't log GPU memory usage. "
+                            "Install py3nvml (pip install py3nvml) to log information about GPU."
+                        )
+                        memory = "N/A"
+                    else:
+                        logger.info(
+                            "Measuring total GPU usage on GPU device. Make sure to not have additional processes running on the same GPU."
+                        )
+                        # init nvml
+                        nvml.nvmlInit()
+                        func()
+                        handle = nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx)
+                        meminfo = nvml.nvmlDeviceGetMemoryInfo(handle)
+                        max_bytes_in_use = meminfo.used
+                        memory = Memory(max_bytes_in_use)
+                        # shutdown nvml
+                        nvml.nvmlShutdown()
+                else:
+                    # cpu
+                    if self.args.trace_memory_line_by_line:
+                        logger.info(
+                            "When enabling line by line tracing, the max peak memory for CPU is inaccurate in Tensorflow."
+                        )
+                        memory = None
+                    else:
+                        memory_bytes = measure_peak_memory_cpu(func)
+                        memory = Memory(memory_bytes) if isinstance(memory_bytes, int) else memory_bytes
+                if self.args.trace_memory_line_by_line:
+                    summary = stop_memory_tracing(trace)
+                    if memory is None:
+                        memory = summary.total
+                else:
+                    summary = None
+
+                return memory, summary
+            except ResourceExhaustedError as e:
+                self.print_fn("Doesn't fit on GPU. {}".format(e))
+                return "N/A", None

src/transformers/benchmark/benchmark_utils.py

@@ -14,14 +14,14 @@ import sys
 from abc import ABC, abstractmethod
 from collections import defaultdict, namedtuple
 from datetime import datetime
-from multiprocessing import Pipe, Process
+from multiprocessing import Pipe, Process, Queue
 from multiprocessing.connection import Connection
 from typing import Callable, Iterable, List, NamedTuple, Optional, Union
 
 from transformers import AutoConfig, PretrainedConfig
 from transformers import __version__ as version
 
-from ..file_utils import is_tf_available, is_torch_available, is_torch_tpu_available
+from ..file_utils import is_psutil_available, is_py3nvml_available, is_tf_available, is_torch_available
 from .benchmark_args_utils import BenchmarkArguments
 
 
@@ -31,6 +31,11 @@ if is_torch_available():
 if is_tf_available():
     from tensorflow.python.eager import context as tf_context
 
+if is_psutil_available():
+    import psutil
+
+if is_py3nvml_available():
+    import py3nvml.py3nvml as nvml
 
 if platform.system() == "Windows":
     from signal import CTRL_C_EVENT as SIGKILL
@@ -56,6 +61,45 @@ BenchmarkOutput = namedtuple(
 )
 
 
+def separate_process_wrapper_fn(func: Callable[[], None], do_multi_processing: bool) -> Callable[[], None]:
+    """
+        This function wraps another function into its own separated process.
+        In order to ensure accurate memory measurements it is important that the function
+        is executed in a separate process
+
+        Args:
+            - `func`: (`callable`): function() -> ...
+                generic function which will be executed in its own separate process
+            - `do_multi_processing`: (`bool`)
+                Whether to run function on separate process or not
+    """
+
+    def multi_process_func(*args, **kwargs):
+        # run function in an individual
+        # process to get correct memory
+        def wrapper_func(queue: Queue, *args):
+            try:
+                result = func(*args)
+            except Exception as e:
+                logger.error(e)
+                print(e)
+                result = "N/A"
+            queue.put(result)
+
+        queue = Queue()
+        p = Process(target=wrapper_func, args=[queue] + list(args))
+        p.start()
+        result = queue.get()
+        p.join()
+        return result
+
+    if do_multi_processing:
+        logging.info("fFunction {func} is executed in its own process...")
+        return multi_process_func
+    else:
+        return func
+
+
 def is_memory_tracing_enabled():
     global _is_memory_tracing_enabled
     return _is_memory_tracing_enabled
@@ -136,7 +180,7 @@ class MemorySummary(NamedTuple):
 MemoryTrace = List[UsedMemoryState]
 
 
-def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5) -> int:
+def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5, device_idx=None) -> int:
     """
         measures peak cpu memory consumption of a given `function`
         running the function for at least interval seconds
@@ -148,16 +192,38 @@ def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5) -> int:
             - `function`: (`callable`): function() -> ...
                 function without any arguments to measure for which to measure the peak memory
 
-            - `interval`: (`float`)
+            - `interval`: (`float`, `optional`, defaults to `0.5`)
                 interval in second for which to measure the memory usage
 
+            - `device_idx`: (`int`, `optional`, defaults to `None`)
+                device id for which to measure gpu usage
+
         Returns:
             - `max_memory`: (`int`)
                 cosumed memory peak in Bytes
     """
-    try:
-        import psutil
-    except (ImportError):
+
+    def get_cpu_memory(process_id: int) -> int:
+        """
+            measures current cpu memory usage of a given `process_id`
+
+            Args:
+                - `process_id`: (`int`)
+                    process_id for which to measure memory
+
+            Returns
+                - `memory`: (`int`)
+                    cosumed memory in Bytes
+        """
+        process = psutil.Process(process_id)
+        try:
+            meminfo_attr = "memory_info" if hasattr(process, "memory_info") else "get_memory_info"
+            memory = getattr(process, meminfo_attr)()[0]
+        except psutil.AccessDenied:
+            raise ValueError("Error with Psutil.")
+        return memory
+
+    if not is_psutil_available():
         logger.warning(
             "Psutil not installed, we won't log CPU memory usage. "
             "Install Psutil (pip install psutil) to use CPU memory tracing."
@@ -165,26 +231,6 @@ def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5) -> int:
         max_memory = "N/A"
     else:
 
-        def _get_memory(process_id: int) -> int:
-            """
-                measures current cpu memory usage of a given `process_id`
-
-                Args:
-                    - `process_id`: (`int`)
-                        process_id for which to measure memory
-
-                Returns
-                    - `memory`: (`int`)
-                        cosumed memory in Bytes
-            """
-            process = psutil.Process(process_id)
-            try:
-                meminfo_attr = "memory_info" if hasattr(process, "memory_info") else "get_memory_info"
-                memory = getattr(process, meminfo_attr)()[0]
-            except psutil.AccessDenied:
-                raise ValueError("Error with Psutil.")
-            return memory
-
         class MemoryMeasureProcess(Process):
 
             """
@@ -198,13 +244,13 @@ def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5) -> int:
                 self.interval = interval
                 self.connection = child_connection
                 self.num_measurements = 1
-                self.mem_usage = _get_memory(process_id)
+                self.mem_usage = get_cpu_memory(self.process_id)
 
             def run(self):
                 self.connection.send(0)
                 stop = False
                 while True:
-                    self.mem_usage = max(self.mem_usage, _get_memory(self.process_id))
+                    self.mem_usage = max(self.mem_usage, get_cpu_memory(self.process_id))
                     self.num_measurements += 1
 
                     if stop:
@@ -296,34 +342,31 @@ def start_memory_tracing(
             - 'line_text' (string): Text of the line in the python script
 
     """
-    try:
-        import psutil
-    except (ImportError):
+    if is_psutil_available():
+        process = psutil.Process(os.getpid())
+    else:
         logger.warning(
             "Psutil not installed, we won't log CPU memory usage. "
             "Install psutil (pip install psutil) to use CPU memory tracing."
         )
         process = None
+
+    if is_py3nvml_available():
+        try:
+            nvml.nvmlInit()
+            devices = list(range(nvml.nvmlDeviceGetCount())) if gpus_to_trace is None else gpus_to_trace
+            nvml.nvmlShutdown()
+        except (OSError, nvml.NVMLError):
+            logger.warning("Error while initializing comunication with GPU. " "We won't perform GPU memory tracing.")
+            log_gpu = False
+        else:
+            log_gpu = is_torch_available() or is_tf_available()
     else:
-        process = psutil.Process(os.getpid())
-
-    try:
-        from py3nvml import py3nvml
-
-        py3nvml.nvmlInit()
-        devices = list(range(py3nvml.nvmlDeviceGetCount())) if gpus_to_trace is None else gpus_to_trace
-        py3nvml.nvmlShutdown()
-    except ImportError:
         logger.warning(
             "py3nvml not installed, we won't log GPU memory usage. "
             "Install py3nvml (pip install py3nvml) to use GPU memory tracing."
         )
         log_gpu = False
-    except (OSError, py3nvml.NVMLError):
-        logger.warning("Error while initializing comunication with GPU. " "We won't perform GPU memory tracing.")
-        log_gpu = False
-    else:
-        log_gpu = is_torch_available() or is_tf_available()
 
     memory_trace = []
 
@@ -385,14 +428,14 @@ def start_memory_tracing(
                 tf_context.context()._clear_caches()  # See https://github.com/tensorflow/tensorflow/issues/20218#issuecomment-416771802
 
             # Sum used memory for all GPUs
-            py3nvml.nvmlInit()
+            nvml.nvmlInit()
 
             for i in devices:
-                handle = py3nvml.nvmlDeviceGetHandleByIndex(i)
-                meminfo = py3nvml.nvmlDeviceGetMemoryInfo(handle)
+                handle = nvml.nvmlDeviceGetHandleByIndex(i)
+                meminfo = nvml.nvmlDeviceGetMemoryInfo(handle)
                 gpu_mem += meminfo.used
 
-            py3nvml.nvmlShutdown()
+            nvml.nvmlShutdown()
 
         mem_state = UsedMemoryState(traced_state, cpu_mem, gpu_mem)
         memory_trace.append(mem_state)
@@ -522,7 +565,6 @@ class Benchmark(ABC):
 
     def __init__(self, args: BenchmarkArguments = None, configs: PretrainedConfig = None):
         self.args = args
-
         if configs is None:
             self.config_dict = {
                 model_name: AutoConfig.from_pretrained(model_name) for model_name in self.args.model_names
@@ -530,6 +572,11 @@ class Benchmark(ABC):
         else:
             self.config_dict = {model_name: config for model_name, config in zip(self.args.model_names, configs)}
 
+        if not self.args.no_memory and os.getenv("TRANSFORMERS_USE_MULTIPROCESSING") == 0:
+            logger.warning(
+                "Memory consumption will not be measured accurately if `args.no_multi_process` is set to `True.` The flag 'TRANSFORMERS_USE_MULTIPROCESSING' should only be disabled for debugging / testing."
+            )
+
         self._print_fn = None
         self._framework_version = None
         self._environment_info = None
@@ -541,7 +588,7 @@ class Benchmark(ABC):
 
                 def print_and_log(*args):
                     with open(self.args.log_filename, "a") as log_file:
-                        log_file.write(str(*args) + "\n")
+                        log_file.write("".join(args) + "\n")
                     print(*args)
 
                 self._print_fn = print_and_log
@@ -549,27 +596,43 @@ class Benchmark(ABC):
                 self._print_fn = print
         return self._print_fn
 
-    @property
-    def is_gpu(self):
-        return self.args.n_gpu > 0
-
-    @property
-    def is_tpu(self):
-        return is_torch_tpu_available() and not self.args.no_tpu
-
     @property
     @abstractmethod
     def framework_version(self):
         pass
 
     @abstractmethod
-    def train(self, model_name, batch_size, sequence_length):
+    def _inference_speed(self, model_name: str, batch_size: int, sequence_length: int) -> float:
         pass
 
     @abstractmethod
-    def inference(self, model_name, batch_size, sequence_length):
+    def _train_speed(self, model_name: str, batch_size: int, sequence_length: int) -> float:
         pass
 
+    @abstractmethod
+    def _inference_memory(
+        self, model_name: str, batch_size: int, sequence_length: int
+    ) -> [Memory, Optional[MemorySummary]]:
+        pass
+
+    @abstractmethod
+    def _train_memory(
+        self, model_name: str, batch_size: int, sequence_length: int
+    ) -> [Memory, Optional[MemorySummary]]:
+        pass
+
+    def inference_speed(self, *args, **kwargs) -> float:
+        return separate_process_wrapper_fn(self._inference_speed, self.args.do_multi_processing)(*args, **kwargs)
+
+    def train_speed(self, *args, **kwargs) -> float:
+        return separate_process_wrapper_fn(self._train_speed, self.args.do_multi_processing)(*args, **kwargs)
+
+    def inference_memory(self, *args, **kwargs) -> [Memory, Optional[MemorySummary]]:
+        return separate_process_wrapper_fn(self._inference_memory, self.args.do_multi_processing)(*args, **kwargs)
+
+    def train_memory(self, *args, **kwargs) -> [Memory, Optional[MemorySummary]]:
+        return separate_process_wrapper_fn(self._train_memory, self.args.do_multi_processing)(*args, **kwargs)
+
     def run(self):
         result_dict = {model_name: {} for model_name in self.args.model_names}
         inference_result_time = copy.deepcopy(result_dict)
@@ -596,64 +659,60 @@ class Benchmark(ABC):
                 for sequence_length in self.args.sequence_lengths:
                     if not self.args.no_inference:
                         if not self.args.no_memory:
-                            memory, inference_summary = self.inference(
-                                model_name, batch_size, sequence_length, trace_memory=True
-                            )
+                            memory, inference_summary = self.inference_memory(model_name, batch_size, sequence_length)
                             inference_result_memory[model_name]["result"][batch_size][sequence_length] = memory
                         if not self.args.no_speed:
-                            time = self.inference(model_name, batch_size, sequence_length, trace_memory=False)
+                            time = self.inference_speed(model_name, batch_size, sequence_length)
                             inference_result_time[model_name]["result"][batch_size][sequence_length] = time
 
                     if self.args.training:
                         if not self.args.no_memory:
-                            memory, train_summary = self.train(
-                                model_name, batch_size, sequence_length, trace_memory=True
-                            )
+                            memory, train_summary = self.train_memory(model_name, batch_size, sequence_length)
                             train_result_memory[model_name]["result"][batch_size][sequence_length] = memory
                         if not self.args.no_speed:
-                            time = self.inference(model_name, batch_size, sequence_length, trace_memory=False)
+                            time = self.train_speed(model_name, batch_size, sequence_length)
                             train_result_time[model_name]["result"][batch_size][sequence_length] = time
 
         if not self.args.no_inference:
             if not self.args.no_speed:
-                self.print_fn("======= INFERENCE - SPEED - RESULT =======")
-                self.print_results(inference_result_time)
+                self.print_fn("\n" + 20 * "=" + ("INFERENCE - SPEED - RESULT").center(40) + 20 * "=")
+                self.print_results(inference_result_time, type_label="Time in s")
                 self.save_to_csv(inference_result_time, self.args.inference_time_csv_file)
-                if self.is_tpu:
+                if self.args.is_tpu:
                     self.print_fn(
                         "TPU was used for inference. Note that the time after compilation stabilized (after ~10 inferences model.forward(..) calls) was measured."
                     )
 
             if not self.args.no_memory:
-                self.print_fn("======= INFERENCE - MEMORY - RESULT =======")
-                self.print_results(inference_result_memory)
+                self.print_fn("\n" + 20 * "=" + ("INFERENCE - MEMORY - RESULT").center(40) + 20 * "=")
+                self.print_results(inference_result_memory, type_label="Memory in MB")
                 self.save_to_csv(inference_result_memory, self.args.inference_memory_csv_file)
 
             if self.args.trace_memory_line_by_line:
-                self.print_fn("======= INFERENCE - MEMORY LINE BY LINE TRACE - SUMMARY =======")
+                self.print_fn("\n" + 20 * "=" + ("INFERENCE - MEMOMRY - LINE BY LINE - SUMMARY").center(40) + 20 * "=")
                 self.print_memory_trace_statistics(inference_summary)
 
         if self.args.training:
             if not self.args.no_speed:
-                self.print_fn("======= TRAIN - SPEED - RESULT =======")
-                self.print_results(train_result_time)
+                self.print_fn("\n" + 20 * "=" + ("TRAIN - SPEED - RESULTS").center(40) + 20 * "=")
+                self.print_results(train_result_time, "Time in s")
                 self.save_to_csv(train_result_time, self.args.train_time_csv_file)
-                if self.is_tpu:
+                if self.args.is_tpu:
                     self.print_fn(
                         "TPU was used for training. Note that the time after compilation stabilized (after ~10 train loss=model.forward(...) + loss.backward() calls) was measured."
                     )
 
             if not self.args.no_memory:
-                self.print_fn("======= TRAIN - MEMORY - RESULT =======")
-                self.print_results(train_result_memory)
+                self.print_fn("\n" + 20 * "=" + ("TRAIN - MEMORY - RESULTS").center(40) + 20 * "=")
+                self.print_results(train_result_memory, type_label="Memory in MB")
                 self.save_to_csv(train_result_memory, self.args.train_memory_csv_file)
 
             if self.args.trace_memory_line_by_line:
-                self.print_fn("======= TRAIN - MEMORY LINE BY LINE TRACE - SUMMARY =======")
+                self.print_fn("\n" + 20 * "=" + ("TRAIN - MEMOMRY - LINE BY LINE - SUMMARY").center(40) + 20 * "=")
                 self.print_memory_trace_statistics(train_summary)
 
         if not self.args.no_env_print:
-            self.print_fn("\n======== ENVIRONMENT - INFORMATION ========")
+            self.print_fn("\n" + 20 * "=" + ("ENVIRONMENT INFORMATION").center(40) + 20 * "=")
             self.print_fn(
                 "\n".join(["- {}: {}".format(prop, val) for prop, val in self.environment_info.items()]) + "\n"
             )
@@ -681,6 +740,9 @@ class Benchmark(ABC):
             info["framework"] = self.framework
             if self.framework == "PyTorch":
                 info["use_torchscript"] = self.args.torchscript
+            if self.framework == "Tensorflow":
+                info["eager_mode"] = self.args.eager_mode
+                info["use_xla"] = self.args.use_xla
             info["framework_version"] = self.framework_version
             info["python_version"] = platform.python_version()
             info["system"] = platform.system()
@@ -688,27 +750,31 @@ class Benchmark(ABC):
             info["architecture"] = platform.architecture()[0]
             info["date"] = datetime.date(datetime.now())
             info["time"] = datetime.time(datetime.now())
+            info["fp16"] = self.args.fp16
+            info["use_multiprocessing"] = self.args.do_multi_processing
+            info["only_pretrain_model"] = self.args.only_pretrain_model
 
-            try:
-                import psutil
-            except (ImportError):
+            if is_psutil_available():
+                info["cpu_ram_mb"] = bytes_to_mega_bytes(psutil.virtual_memory().total)
+            else:
                 logger.warning(
                     "Psutil not installed, we won't log available CPU memory."
                     "Install psutil (pip install psutil) to log available CPU memory."
                 )
                 info["cpu_ram_mb"] = "N/A"
-            else:
-                info["cpu_ram_mb"] = bytes_to_mega_bytes(psutil.virtual_memory().total)
 
-            info["use_gpu"] = self.is_gpu
-            if self.is_gpu:
-                info["num_gpus"] = self.args.n_gpu
-                try:
-                    from py3nvml import py3nvml
-
-                    py3nvml.nvmlInit()
-                    handle = py3nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx)
-                except ImportError:
+            info["use_gpu"] = self.args.is_gpu
+            if self.args.is_gpu:
+                info["num_gpus"] = 1  # TODO(PVP) Currently only single GPU is supported
+                if is_py3nvml_available():
+                    nvml.nvmlInit()
+                    handle = nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx)
+                    info["gpu"] = nvml.nvmlDeviceGetName(handle)
+                    info["gpu_ram_mb"] = bytes_to_mega_bytes(nvml.nvmlDeviceGetMemoryInfo(handle).total)
+                    info["gpu_power_watts"] = nvml.nvmlDeviceGetPowerManagementLimit(handle) / 1000
+                    info["gpu_performance_state"] = nvml.nvmlDeviceGetPerformanceState(handle)
+                    nvml.nvmlShutdown()
+                else:
                     logger.warning(
                         "py3nvml not installed, we won't log GPU memory usage. "
                         "Install py3nvml (pip install py3nvml) to log information about GPU."
@@ -717,41 +783,35 @@ class Benchmark(ABC):
                     info["gpu_ram_mb"] = "N/A"
                     info["gpu_power_watts"] = "N/A"
                     info["gpu_performance_state"] = "N/A"
-                except (OSError, py3nvml.NVMLError):
-                    logger.warning(
-                        "Error while initializing comunication with GPU. " "We won't log information about GPU."
-                    )
-                    info["gpu"] = "N/A"
-                    info["gpu_ram_mb"] = "N/A"
-                    info["gpu_power_watts"] = "N/A"
-                    info["gpu_performance_state"] = "N/A"
-                    py3nvml.nvmlShutdown()
-                else:
-                    info["gpu"] = py3nvml.nvmlDeviceGetName(handle)
-                    info["gpu_ram_mb"] = bytes_to_mega_bytes(py3nvml.nvmlDeviceGetMemoryInfo(handle).total)
-                    info["gpu_power_watts"] = py3nvml.nvmlDeviceGetPowerManagementLimit(handle) / 1000
-                    info["gpu_performance_state"] = py3nvml.nvmlDeviceGetPerformanceState(handle)
-                    py3nvml.nvmlShutdown()
 
-            info["use_tpu"] = self.is_tpu
+            info["use_tpu"] = self.args.is_tpu
             # TODO(PVP): See if we can add more information about TPU
             # see: https://github.com/pytorch/xla/issues/2180
 
             self._environment_info = info
         return self._environment_info
 
-    def print_results(self, result_dict):
+    def print_results(self, result_dict, type_label):
+        self.print_fn(80 * "-")
+        self.print_fn(
+            "Model Name".center(30) + "Batch Size".center(15) + "Seq Length".center(15) + type_label.center(15)
+        )
+        self.print_fn(80 * "-")
         for model_name in self.args.model_names:
-            self.print_fn("\t" + f"======= MODEL CHECKPOINT: {model_name} =======")
             for batch_size in result_dict[model_name]["bs"]:
                 for sequence_length in result_dict[model_name]["ss"]:
                     result = result_dict[model_name]["result"][batch_size][sequence_length]
                     if isinstance(result, float):
-                        self.print_fn(
-                            f"\t\t{model_name}/{batch_size}/{sequence_length}: " f"{(round(1000 * result) / 1000)}s"
-                        )
+                        result = round(1000 * result) / 1000
+                        result = "< 0.001" if result == 0.0 else str(result)
                     else:
-                        self.print_fn(f"\t\t{model_name}/{batch_size}/{sequence_length}: " f"{result} MB")
+                        result = str(result)
+                    self.print_fn(
+                        model_name.center(30) + str(batch_size).center(15),
+                        str(sequence_length).center(15),
+                        result.center(15),
+                    )
+        self.print_fn(80 * "-")
 
     def print_memory_trace_statistics(self, summary: MemorySummary):
         self.print_fn(

src/transformers/configuration_auto.py

@@ -19,7 +19,7 @@ import logging
 from collections import OrderedDict
 
 from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
-from .configuration_bart import BART_PRETRAINED_CONFIG_ARCHIVE_MAP, BartConfig
+from .configuration_bart import BART_PRETRAINED_CONFIG_ARCHIVE_MAP, BartConfig, MBartConfig
 from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig
 from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
 from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
@@ -30,6 +30,7 @@ from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, Flau
 from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config
 from .configuration_longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig
 from .configuration_marian import MarianConfig
+from .configuration_mobilebert import MobileBertConfig
 from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
 from .configuration_reformer import ReformerConfig
 from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
@@ -75,11 +76,13 @@ CONFIG_MAPPING = OrderedDict(
     [
         ("retribert", RetriBertConfig,),
         ("t5", T5Config,),
+        ("mobilebert", MobileBertConfig,),
         ("distilbert", DistilBertConfig,),
         ("albert", AlbertConfig,),
         ("camembert", CamembertConfig,),
         ("xlm-roberta", XLMRobertaConfig,),
         ("marian", MarianConfig,),
+        ("mbart", MBartConfig,),
         ("bart", BartConfig,),
         ("reformer", ReformerConfig,),
         ("longformer", LongformerConfig,),

src/transformers/configuration_bart.py

@@ -41,6 +41,7 @@ class BartConfig(PretrainedConfig):
     def __init__(
         self,
         activation_dropout=0.0,
+        extra_pos_embeddings=2,
         activation_function="gelu",
         vocab_size=50265,
         d_model=1024,
@@ -118,6 +119,9 @@ class BartConfig(PretrainedConfig):
         # Classifier stuff
         self.classif_dropout = classifier_dropout
 
+        # pos embedding offset
+        self.extra_pos_embeddings = self.pad_token_id + 1
+
     @property
     def num_attention_heads(self) -> int:
         return self.encoder_attention_heads
@@ -133,3 +137,7 @@ class BartConfig(PretrainedConfig):
         if self.normalize_before or self.add_final_layer_norm or self.scale_embedding:
             logger.info("This configuration is a mixture of MBART and BART settings")
         return False
+
+
+class MBartConfig(BartConfig):
+    model_type = "mbart"

src/transformers/configuration_bert.py

@@ -90,6 +90,8 @@ class BertConfig(PretrainedConfig):
                 The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
             layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
                 The epsilon used by the layer normalization layers.
+            gradient_checkpointing (:obj:`bool`, optional, defaults to False):
+                If True, use gradient checkpointing to save memory at the expense of slower backward pass.
 
         Example::
 
@@ -121,6 +123,7 @@ class BertConfig(PretrainedConfig):
         initializer_range=0.02,
         layer_norm_eps=1e-12,
         pad_token_id=0,
+        gradient_checkpointing=False,
         **kwargs
     ):
         super().__init__(pad_token_id=pad_token_id, **kwargs)
@@ -137,3 +140,4 @@ class BertConfig(PretrainedConfig):
         self.type_vocab_size = type_vocab_size
         self.initializer_range = initializer_range
         self.layer_norm_eps = layer_norm_eps
+        self.gradient_checkpointing = gradient_checkpointing

src/transformers/configuration_electra.py

@@ -76,6 +76,28 @@ class ElectraConfig(PretrainedConfig):
                 The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
             layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
                 The epsilon used by the layer normalization layers.
+            summary_type (:obj:`string`, optional, defaults to "first"):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.ElectraForMultipleChoice`.
+                Is one of the following options:
+
+                    - 'last' => take the last token hidden state (like XLNet)
+                    - 'first' => take the first token hidden state (like Bert)
+                    - 'mean' => take the mean of all tokens hidden states
+                    - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2)
+                    - 'attn' => Not implemented now, use multi-head attention
+            summary_use_proj (:obj:`boolean`, optional, defaults to :obj:`True`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.ElectraForMultipleChoice`.
+                Add a projection after the vector extraction
+            summary_activation (:obj:`string` or :obj:`None`, optional, defaults to :obj:`None`):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.ElectraForMultipleChoice`.
+                'gelu' => add a gelu activation to the output, Other => no activation.
+            summary_last_dropout (:obj:`float`, optional, defaults to 0.0):
+                Argument used when doing sequence summary. Used in for the multiple choice head in
+                :class:`~transformers.ElectraForMultipleChoice`.
+                Add a dropout after the projection and activation
 
         Example::
 
@@ -107,6 +129,10 @@ class ElectraConfig(PretrainedConfig):
         type_vocab_size=2,
         initializer_range=0.02,
         layer_norm_eps=1e-12,
+        summary_type="first",
+        summary_use_proj=True,
+        summary_activation="gelu",
+        summary_last_dropout=0.1,
         pad_token_id=0,
         **kwargs
     ):
@@ -125,3 +151,8 @@ class ElectraConfig(PretrainedConfig):
         self.type_vocab_size = type_vocab_size
         self.initializer_range = initializer_range
         self.layer_norm_eps = layer_norm_eps
+
+        self.summary_type = summary_type
+        self.summary_use_proj = summary_use_proj
+        self.summary_activation = summary_activation
+        self.summary_last_dropout = summary_last_dropout

src/transformers/configuration_encoder_decoder.py

@@ -37,7 +37,7 @@ class EncoderDecoderConfig(PretrainedConfig):
                 Remaining dictionary of keyword arguments. Notably:
                     encoder (:class:`PretrainedConfig`, optional, defaults to `None`):
                         An instance of a configuration object that defines the encoder config.
-                    encoder (:class:`PretrainedConfig`, optional, defaults to `None`):
+                    decoder (:class:`PretrainedConfig`, optional, defaults to `None`):
                         An instance of a configuration object that defines the decoder config.
 
         Example::

src/transformers/configuration_mobilebert.py

@@ -0,0 +1,159 @@
+# coding=utf-8
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" MobileBERT model configuration """
+
+import logging
+
+from .configuration_utils import PretrainedConfig
+
+
+logger = logging.getLogger(__name__)
+
+MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "mobilebert-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/google/mobilebert-uncased/config.json"
+}
+
+
+class MobileBertConfig(PretrainedConfig):
+    r"""
+        This is the configuration class to store the configuration of a :class:`~transformers.MobileBertModel`.
+        It is used to instantiate a MobileBERT model according to the specified arguments, defining the model
+        architecture.
+
+        Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+        to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+        for more information.
+
+
+        Args:
+            vocab_size (:obj:`int`, optional, defaults to 30522):
+                Vocabulary size of the MobileBERT model. Defines the different tokens that
+                can be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.MobileBertModel`.
+            hidden_size (:obj:`int`, optional, defaults to 512):
+                Dimensionality of the encoder layers and the pooler layer.
+            num_hidden_layers (:obj:`int`, optional, defaults to 24):
+                Number of hidden layers in the Transformer encoder.
+            num_attention_heads (:obj:`int`, optional, defaults to 4):
+                Number of attention heads for each attention layer in the Transformer encoder.
+            intermediate_size (:obj:`int`, optional, defaults to 512):
+                Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+            hidden_act (:obj:`str` or :obj:`function`, optional, defaults to "relu"):
+                The non-linear activation function (function or string) in the encoder and pooler.
+                If string, "gelu", "relu", "swish" and "gelu_new" are supported.
+            hidden_dropout_prob (:obj:`float`, optional, defaults to 0.0):
+                The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0.1):
+                The dropout ratio for the attention probabilities.
+            max_position_embeddings (:obj:`int`, optional, defaults to 512):
+                The maximum sequence length that this model might ever be used with.
+                Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+            type_vocab_size (:obj:`int`, optional, defaults to 2):
+                The vocabulary size of the `token_type_ids` passed into :class:`~transformers.MobileBertModel`.
+            initializer_range (:obj:`float`, optional, defaults to 0.02):
+                The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+            layer_norm_eps (:obj:`float`, optional, defaults to 1e-12):
+                The epsilon used by the layer normalization layers.
+
+            pad_token_id (:obj:`int`, optional, defaults to 0):
+                The ID of the token in the word embedding to use as padding.
+            embedding_size (:obj:`int`, optional, defaults to 128):
+                The dimension of the word embedding vectors.
+            trigram_input (:obj:`bool`, optional, defaults to True):
+                Use a convolution of trigram as input.
+            use_bottleneck (:obj:`bool`, optional, defaults to True):
+                Whether to use bottleneck in BERT.
+            intra_bottleneck_size (:obj:`int`, optional, defaults to 128):
+                Size of bottleneck layer output.
+            use_bottleneck_attention (:obj:`bool`, optional, defaults to False):
+                Whether to use attention inputs from the bottleneck transformation.
+            key_query_shared_bottleneck (:obj:`bool`, optional, defaults to True):
+                Whether to use the same linear transformation for query&key in the bottleneck.
+            num_feedforward_networks (:obj:`int`, optional, defaults to 4):
+                Number of FFNs in a block.
+            normalization_type (:obj:`str`, optional, defaults to "no_norm"):
+                The normalization type in BERT.
+
+        Example:
+
+            from transformers import MobileBertModel, MobileBertConfig
+
+            # Initializing a MobileBERT configuration
+            configuration = MobileBertConfig()
+
+            # Initializing a model from the configuration above
+            model = MobileBertModel(configuration)
+
+            # Accessing the model configuration
+            configuration = model.config
+
+        Attributes:
+            pretrained_config_archive_map (Dict[str, str]):
+                A dictionary containing all the available pre-trained checkpoints.
+    """
+    pretrained_config_archive_map = MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
+    model_type = "mobilebert"
+
+    def __init__(
+        self,
+        vocab_size=30522,
+        hidden_size=512,
+        num_hidden_layers=24,
+        num_attention_heads=4,
+        intermediate_size=512,
+        hidden_act="relu",
+        hidden_dropout_prob=0.0,
+        attention_probs_dropout_prob=0.1,
+        max_position_embeddings=512,
+        type_vocab_size=2,
+        initializer_range=0.02,
+        layer_norm_eps=1e-12,
+        pad_token_id=0,
+        embedding_size=128,
+        trigram_input=True,
+        use_bottleneck=True,
+        intra_bottleneck_size=128,
+        use_bottleneck_attention=False,
+        key_query_shared_bottleneck=True,
+        num_feedforward_networks=4,
+        normalization_type="no_norm",
+        classifier_activation=True,
+        **kwargs
+    ):
+        super().__init__(pad_token_id=pad_token_id, **kwargs)
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.type_vocab_size = type_vocab_size
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.embedding_size = embedding_size
+        self.trigram_input = trigram_input
+        self.use_bottleneck = use_bottleneck
+        self.intra_bottleneck_size = intra_bottleneck_size
+        self.use_bottleneck_attention = use_bottleneck_attention
+        self.key_query_shared_bottleneck = key_query_shared_bottleneck
+        self.num_feedforward_networks = num_feedforward_networks
+        self.normalization_type = normalization_type
+        self.classifier_activation = classifier_activation
+
+        if self.use_bottleneck:
+            self.true_hidden_size = intra_bottleneck_size
+        else:
+            self.true_hidden_size = hidden_size

src/transformers/configuration_reformer.py

@@ -97,7 +97,7 @@ class ReformerConfig(PretrainedConfig):
                 Number of following neighbouring chunks to attend to in LocalSelfAttention layer in addition to itself.
             local_attention_probs_dropout_prob (:obj:`float`, optional, defaults to 0.1):
                 The dropout ratio for the attention probabilities in LocalSelfAttention.
-            lsh_chunk_length (:obj:`int`, optional, defaults to 64):
+            lsh_attn_chunk_length (:obj:`int`, optional, defaults to 64):
                 Length of chunk which attends to itself in LSHSelfAttention. Chunking reduces memory complexity from sequence length x sequence length (self attention) to chunk length x chunk length x sequence length / chunk length (chunked self attention).
             lsh_num_chunks_before (:obj:`int`, optional, defaults to 1):
                 Number of previous neighbouring chunks to attend to in LSHSelfAttention layer to itself.

src/transformers/configuration_t5.py

@@ -41,6 +41,8 @@ class T5Config(PretrainedConfig):
             vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `T5Model`.
             d_model: Size of the encoder layers and the pooler layer. `d_model` can also accesed via the property `hidden_size`.
             num_layers: Number of hidden layers in the Transformer encoder. `num_layers` can also be accessed via the property `num_hidden_layers`.
+            d_kv: Size of the key, query, value projections per attention head. `d_kv` has to be equal to `d_model // num_heads`.
+            d_ff: Size of the intermediate feed forward layer in each `T5Block`.
             num_heads: Number of attention heads for each attention layer in
                 the Transformer encoder. `num_heads` can also be accessed via the property `num_attention_heads`.
             intermediate_size: The size of the "intermediate" (i.e., feed-forward)

src/transformers/convert_mobilebert_original_tf_checkpoint_to_pytorch.py

@@ -0,0 +1,42 @@
+import argparse
+import logging
+
+import torch
+
+from transformers import MobileBertConfig, MobileBertForPreTraining, load_tf_weights_in_mobilebert
+
+
+logging.basicConfig(level=logging.INFO)
+
+
+def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, mobilebert_config_file, pytorch_dump_path):
+    # Initialise PyTorch model
+    config = MobileBertConfig.from_json_file(mobilebert_config_file)
+    print("Building PyTorch model from configuration: {}".format(str(config)))
+    model = MobileBertForPreTraining(config)
+    # Load weights from tf checkpoint
+    model = load_tf_weights_in_mobilebert(model, config, tf_checkpoint_path)
+    # Save pytorch-model
+    print("Save PyTorch model to {}".format(pytorch_dump_path))
+    torch.save(model.state_dict(), pytorch_dump_path)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    # Required parameters
+    parser.add_argument(
+        "--tf_checkpoint_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path."
+    )
+    parser.add_argument(
+        "--mobilebert_config_file",
+        default=None,
+        type=str,
+        required=True,
+        help="The config json file corresponding to the pre-trained MobileBERT model. \n"
+        "This specifies the model architecture.",
+    )
+    parser.add_argument(
+        "--pytorch_dump_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
+    )
+    args = parser.parse_args()
+    convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.mobilebert_config_file, args.pytorch_dump_path)

src/transformers/data/data_collator.py

@@ -42,10 +42,10 @@ def default_data_collator(features: List[InputDataClass]) -> Dict[str, torch.Ten
     # Special handling for labels.
     # Ensure that tensor is created with the correct type
     # (it should be automatically the case, but let's make sure of it.)
-    if "label" in first:
+    if "label" in first and first["label"] is not None:
         dtype = torch.long if type(first["label"]) is int else torch.float
         batch["labels"] = torch.tensor([f["label"] for f in features], dtype=dtype)
-    elif "label_ids" in first:
+    elif "label_ids" in first and first["label_ids"] is not None:
         if isinstance(first["label_ids"], torch.Tensor):
             batch["labels"] = torch.stack([f["label_ids"] for f in features])
         else:

src/transformers/data/datasets/glue.py

@@ -9,6 +9,7 @@ import torch
 from filelock import FileLock
 from torch.utils.data.dataset import Dataset
 
+from ...tokenization_bart import BartTokenizer, BartTokenizerFast
 from ...tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
 from ...tokenization_utils import PreTrainedTokenizer
 from ...tokenization_xlm_roberta import XLMRobertaTokenizer
@@ -92,6 +93,8 @@ class GlueDataset(Dataset):
             RobertaTokenizer,
             RobertaTokenizerFast,
             XLMRobertaTokenizer,
+            BartTokenizer,
+            BartTokenizerFast,
         ):
             # HACK(label indices are swapped in RoBERTa pretrained model)
             label_list[1], label_list[2] = label_list[2], label_list[1]

src/transformers/file_utils.py

@@ -16,7 +16,7 @@ from contextlib import contextmanager
 from functools import partial, wraps
 from hashlib import sha256
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional, Union
 from urllib.parse import urlparse
 from zipfile import ZipFile, is_zipfile
 
@@ -81,6 +81,31 @@ except ImportError:
     _torch_tpu_available = False
 
 
+try:
+    import psutil  # noqa: F401
+
+    _psutil_available = True
+
+except ImportError:
+    _psutil_available = False
+
+
+try:
+    import py3nvml  # noqa: F401
+
+    _py3nvml_available = True
+
+except ImportError:
+    _py3nvml_available = False
+
+
+try:
+    from apex import amp  # noqa: F401
+
+    _has_apex = True
+except ImportError:
+    _has_apex = False
+
 default_cache_path = os.path.join(torch_cache_home, "transformers")
 
 
@@ -115,6 +140,18 @@ def is_torch_tpu_available():
     return _torch_tpu_available
 
 
+def is_psutil_available():
+    return _psutil_available
+
+
+def is_py3nvml_available():
+    return _py3nvml_available
+
+
+def is_apex_available():
+    return _has_apex
+
+
 def add_start_docstrings(*docstr):
     def docstring_decorator(fn):
         fn.__doc__ = "".join(docstr) + (fn.__doc__ if fn.__doc__ is not None else "")
@@ -234,7 +271,7 @@ def cached_path(
     force_download=False,
     proxies=None,
     resume_download=False,
-    user_agent=None,
+    user_agent: Union[Dict, str, None] = None,
     extract_compressed_file=False,
     force_extract=False,
     local_files_only=False,
@@ -320,7 +357,7 @@ def cached_path(
     return output_path
 
 
-def http_get(url, temp_file, proxies=None, resume_size=0, user_agent=None):
+def http_get(url, temp_file, proxies=None, resume_size=0, user_agent: Union[Dict, str, None] = None):
     ua = "transformers/{}; python/{}".format(__version__, sys.version.split()[0])
     if is_torch_available():
         ua += "; torch/{}".format(torch.__version__)
@@ -360,7 +397,7 @@ def get_from_cache(
     proxies=None,
     etag_timeout=10,
     resume_download=False,
-    user_agent=None,
+    user_agent: Union[Dict, str, None] = None,
     local_files_only=False,
 ) -> Optional[str]:
     """

src/transformers/modeling_albert.py

@@ -269,7 +269,9 @@ class AlbertLayer(nn.Module):
         self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
         self.activation = ACT2FN[config.hidden_act]
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, output_attentions=False):
+    def forward(
+        self, hidden_states, attention_mask=None, head_mask=None, output_attentions=False, output_hidden_states=False
+    ):
         attention_output = self.attention(hidden_states, attention_mask, head_mask, output_attentions)
         ffn_output = self.ffn(attention_output[0])
         ffn_output = self.activation(ffn_output)
@@ -283,10 +285,11 @@ class AlbertLayerGroup(nn.Module):
     def __init__(self, config):
         super().__init__()
 
-        self.output_hidden_states = config.output_hidden_states
         self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)])
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, output_attentions=False):
+    def forward(
+        self, hidden_states, attention_mask=None, head_mask=None, output_attentions=False, output_hidden_states=False
+    ):
         layer_hidden_states = ()
         layer_attentions = ()
 
@@ -297,11 +300,11 @@ class AlbertLayerGroup(nn.Module):
             if output_attentions:
                 layer_attentions = layer_attentions + (layer_output[1],)
 
-            if self.output_hidden_states:
+            if output_hidden_states:
                 layer_hidden_states = layer_hidden_states + (hidden_states,)
 
         outputs = (hidden_states,)
-        if self.output_hidden_states:
+        if output_hidden_states:
             outputs = outputs + (layer_hidden_states,)
         if output_attentions:
             outputs = outputs + (layer_attentions,)
@@ -313,16 +316,17 @@ class AlbertTransformer(nn.Module):
         super().__init__()
 
         self.config = config
-        self.output_hidden_states = config.output_hidden_states
         self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size)
         self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)])
 
-    def forward(self, hidden_states, attention_mask=None, head_mask=None, output_attentions=False):
+    def forward(
+        self, hidden_states, attention_mask=None, head_mask=None, output_attentions=False, output_hidden_states=False
+    ):
         hidden_states = self.embedding_hidden_mapping_in(hidden_states)
 
         all_attentions = ()
 
-        if self.output_hidden_states:
+        if output_hidden_states:
             all_hidden_states = (hidden_states,)
 
         for i in range(self.config.num_hidden_layers):
@@ -337,17 +341,18 @@ class AlbertTransformer(nn.Module):
                 attention_mask,
                 head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group],
                 output_attentions,
+                output_hidden_states,
             )
             hidden_states = layer_group_output[0]
 
             if output_attentions:
                 all_attentions = all_attentions + layer_group_output[-1]
 
-            if self.output_hidden_states:
+            if output_hidden_states:
                 all_hidden_states = all_hidden_states + (hidden_states,)
 
         outputs = (hidden_states,)
-        if self.output_hidden_states:
+        if output_hidden_states:
             outputs = outputs + (all_hidden_states,)
         if output_attentions:
             outputs = outputs + (all_attentions,)
@@ -489,6 +494,7 @@ class AlbertModel(AlbertPreTrainedModel):
         head_mask=None,
         inputs_embeds=None,
         output_attentions=None,
+        output_hidden_states=None,
     ):
         r"""
     Return:
@@ -504,12 +510,12 @@ class AlbertModel(AlbertPreTrainedModel):
             This output is usually *not* a good summary
             of the semantic content of the input, you're often better with averaging or pooling
             the sequence of hidden-states for the whole input sequence.
-        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
             Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
             of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
             Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or ``config.output_attentions=True``):
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
             Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
             :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
@@ -530,6 +536,9 @@ class AlbertModel(AlbertPreTrainedModel):
         """
 
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
 
         if input_ids is not None and inputs_embeds is not None:
             raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
@@ -556,7 +565,11 @@ class AlbertModel(AlbertPreTrainedModel):
             input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds
         )
         encoder_outputs = self.encoder(
-            embedding_output, extended_attention_mask, head_mask=head_mask, output_attentions=output_attentions,
+            embedding_output,
+            extended_attention_mask,
+            head_mask=head_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
         )
 
         sequence_output = encoder_outputs[0]
@@ -603,6 +616,7 @@ class AlbertForPreTraining(AlbertPreTrainedModel):
         labels=None,
         sentence_order_label=None,
         output_attentions=None,
+        output_hidden_states=None,
         **kwargs,
     ):
         r"""
@@ -628,12 +642,12 @@ class AlbertForPreTraining(AlbertPreTrainedModel):
         sop_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, 2)`):
             Prediction scores of the next sequence prediction (classification) head (scores of True/False
             continuation before SoftMax).
-        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when :obj:`config.output_hidden_states=True`):
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
             Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
             of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
             Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or ``config.output_attentions=True``):
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
             Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
             :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
@@ -672,6 +686,7 @@ class AlbertForPreTraining(AlbertPreTrainedModel):
             head_mask=head_mask,
             inputs_embeds=inputs_embeds,
             output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
         )
 
         sequence_output, pooled_output = outputs[:2]
@@ -758,6 +773,7 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
         inputs_embeds=None,
         labels=None,
         output_attentions=None,
+        output_hidden_states=None,
         **kwargs
     ):
         r"""
@@ -775,12 +791,12 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
             Masked language modeling loss.
         prediction_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, config.vocab_size)`)
             Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
-        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
             Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
             of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
             Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or ``config.output_attentions=True``):
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
             Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
             :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
@@ -815,6 +831,7 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
             head_mask=head_mask,
             inputs_embeds=inputs_embeds,
             output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
         )
         sequence_outputs = outputs[0]
 
@@ -856,6 +873,7 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):
         inputs_embeds=None,
         labels=None,
         output_attentions=None,
+        output_hidden_states=None,
     ):
         r"""
         labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
@@ -870,12 +888,12 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):
             Classification (or regression if config.num_labels==1) loss.
         logits ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
             Classification (or regression if config.num_labels==1) scores (before SoftMax).
-        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
             Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
             of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
             Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or ``config.output_attentions=True``):
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
             Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
             :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
@@ -904,6 +922,7 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):
             head_mask=head_mask,
             inputs_embeds=inputs_embeds,
             output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
         )
 
         pooled_output = outputs[1]
@@ -953,6 +972,7 @@ class AlbertForTokenClassification(AlbertPreTrainedModel):
         inputs_embeds=None,
         labels=None,
         output_attentions=None,
+        output_hidden_states=None,
     ):
         r"""
         labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
@@ -965,12 +985,12 @@ class AlbertForTokenClassification(AlbertPreTrainedModel):
             Classification loss.
         scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, config.num_labels)`)
             Classification scores (before SoftMax).
-        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
             Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
             of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
             Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or ``config.output_attentions=True``):
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
             Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
             :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
@@ -1001,6 +1021,7 @@ class AlbertForTokenClassification(AlbertPreTrainedModel):
             head_mask=head_mask,
             inputs_embeds=inputs_embeds,
             output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
         )
 
         sequence_output = outputs[0]
@@ -1052,6 +1073,7 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):
         start_positions=None,
         end_positions=None,
         output_attentions=None,
+        output_hidden_states=None,
     ):
         r"""
         start_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
@@ -1071,12 +1093,12 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):
             Span-start scores (before SoftMax).
         end_scores: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)``
             Span-end scores (before SoftMax).
-        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
             Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
             of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
             Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or ``config.output_attentions=True``):
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
             Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
             :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
@@ -1107,6 +1129,7 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):
             head_mask=head_mask,
             inputs_embeds=inputs_embeds,
             output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
         )
 
         sequence_output = outputs[0]
@@ -1163,6 +1186,7 @@ class AlbertForMultipleChoice(AlbertPreTrainedModel):
         inputs_embeds=None,
         labels=None,
         output_attentions=None,
+        output_hidden_states=None,
     ):
         r"""
         labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
@@ -1178,12 +1202,12 @@ class AlbertForMultipleChoice(AlbertPreTrainedModel):
             `num_choices` is the second dimension of the input tensors. (see `input_ids` above).
 
             Classification scores (before SoftMax).
-        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
             Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
             of shape :obj:`(batch_size, sequence_length, hidden_size)`.
 
             Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
             Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
             :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
 
@@ -1228,6 +1252,7 @@ class AlbertForMultipleChoice(AlbertPreTrainedModel):
             head_mask=head_mask,
             inputs_embeds=inputs_embeds,
             output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
         )
 
         pooled_output = outputs[1]