add run_nlp_glue

2020-06-24 14:46:16 +02:00
167 changed files with 6402 additions and 9127 deletions
@@ -5,26 +5,15 @@ function deploy_doc(){
 	git checkout $1
 	if [ ! -z "$2" ]
 	then
-		if [ "$2" == "master" ]; then
-		    echo "Pushing master"
-			make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir/$2/
-			cp -r _build/html/_static .
-		elif ssh -oStrictHostKeyChecking=no $doc "[ -d $dir/$2 ]"; then
+		if [ -d "$dir/$2" ]; then
 			echo "Directory" $2 "already exists"
-			scp -r -oStrictHostKeyChecking=no _static/* $doc:$dir/$2/_static/
 		else
 			echo "Pushing version" $2
-			make clean && make html
-			rm -rf _build/html/_static
-			cp -r _static _build/html
-			scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2
+			make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2
 		fi
 	else
-		echo "Pushing stable"
-		make clean && make html
-		rm -rf _build/html/_static
-		cp -r _static _build/html
-		scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir
+		echo "Pushing master"
+		make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir
 	fi
 }

@@ -59,7 +59,7 @@ Choose the right framework for every part of a model's lifetime
 | [Quick tour: Share your models ](#Quick-tour-of-model-sharing) | Upload and share your fine-tuned models with the community |
 | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers |
 | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers |
-| [Documentation](https://huggingface.co/transformers/) | Full API documentation and more |
+| Documentation [(master)](https://huggingface.co/transformers/master) [(stable)](https://huggingface.co/transformers/) [(v2.10.0)](https://huggingface.co/transformers/v2.10.0) [(v2.9.0/v2.9.1)](https://huggingface.co/transformers/v2.9.1) [(v2.8.0)](https://huggingface.co/transformers/v2.8.0) [(v2.7.0)](https://huggingface.co/transformers/v2.7.0) [(v2.6.0)](https://huggingface.co/transformers/v2.6.0) [(v2.5.0/v2.5.1)](https://huggingface.co/transformers/v2.5.1) [(v2.4.0/v2.4.1)](https://huggingface.co/transformers/v2.4.0)[(v2.3.0)](https://huggingface.co/transformers/v2.3.0)[(v2.2.0/v2.2.1/v2.2.2)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) | Full API documentation and more |

 ## Installation

@@ -9,8 +9,4 @@

 .highlight .kn, .highlight .nv, .highlight .s2, .highlight .ow {
    color: #6670FF;
-}
-
-.highlight .gp {
-    color: #FB8D68;
 }
@@ -44,7 +44,6 @@
    display: flex;
    flex-direction: row;
    justify-content: flex-end;
-    margin-right: 30px;
 }

 .framework-selector > button {
@@ -61,12 +60,6 @@
    padding: 5px;
 }

-/* Copy button */
-
-a.copybtn {
-    margin: 3px;
-}
-
 /* The literal code blocks */
 .rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal {
    color: #6670FF;
@@ -84,14 +84,10 @@ function addGithubButton() {
 function addVersionControl() {
    // To grab the version currently in view, we parse the url
    const parts = location.toString().split('/');
-    let versionIndex = parts.length - 2;
-    // Index page may not have a last part with filename.html so we need to go up
-    if (parts[parts.length - 1] != "" && ! parts[parts.length - 1].match(/\.html$/)) {
-        versionIndex = parts.length - 1;
-    }
+    let versionIndex = parts.length - 2
    // Main classes and models are nested so we need to go deeper
-    else if (parts[versionIndex] == "main_classes" || parts[versionIndex] == "model_doc") {
-        versionIndex = versionIndex - 1;
+    if (parts[versionIndex] == "main_classes" || parts[versionIndex] == "model_doc") {
+        versionIndex = parts.length - 3
    } 
    const version = parts[versionIndex];

@@ -100,12 +96,8 @@ function addVersionControl() {

    const htmlLines = [];
    for (const [key, value] of Object.entries(versionMapping)) {
-        let baseUrlIndex = (version == "transformers") ? versionIndex + 1: versionIndex;
-        var urlParts = parts.slice(0, baseUrlIndex);
-        if (key != "") {
-            urlParts = urlParts.concat([key]);
-        }
-        urlParts = urlParts.concat(parts.slice(versionIndex+1));
+        var urlParts = (key == "") ? [] : [key];
+        urlParts = urlParts.concat(parts.slice(versionIndex));
        htmlLines.push(`<a href="${urlParts.join('/')}">${value}</a>`);
    }

@@ -157,8 +149,6 @@ function platformToggle() {
    const codeBlocks = Array.from(document.getElementsByClassName("highlight"));
    const pytorchIdentifier = "## PYTORCH CODE";
    const tensorflowIdentifier = "## TENSORFLOW CODE";
-
-    const promptSpanIdentifier = `<span class="gp">&gt;&gt;&gt; </span>`
    const pytorchSpanIdentifier = `<span class="c1">${pytorchIdentifier}</span>`;
    const tensorflowSpanIdentifier = `<span class="c1">${tensorflowIdentifier}</span>`;

@@ -171,22 +161,10 @@ function platformToggle() {
        let tensorflowSpans;

        if(pytorchSpanPosition < tensorflowSpanPosition){
-            const isPrompt = spans.slice(
-                spans.indexOf(tensorflowSpanIdentifier) - promptSpanIdentifier.length,
-                spans.indexOf(tensorflowSpanIdentifier)
-            ) == promptSpanIdentifier;
-            const finalTensorflowSpanPosition = isPrompt ? tensorflowSpanPosition - promptSpanIdentifier.length : tensorflowSpanPosition;
-
-            pytorchSpans = spans.slice(pytorchSpanPosition + pytorchSpanIdentifier.length + 1, finalTensorflowSpanPosition);
+            pytorchSpans = spans.slice(pytorchSpanPosition + pytorchSpanIdentifier.length + 1, tensorflowSpanPosition);
            tensorflowSpans = spans.slice(tensorflowSpanPosition + tensorflowSpanIdentifier.length + 1, spans.length);
        }else{
-            const isPrompt = spans.slice(
-                spans.indexOf(pytorchSpanIdentifier) - promptSpanIdentifier.length,
-                spans.indexOf(pytorchSpanIdentifier)
-            ) == promptSpanIdentifier;
-            const finalPytorchSpanPosition = isPrompt ? pytorchSpanPosition - promptSpanIdentifier.length : pytorchSpanPosition;
-
-            tensorflowSpans = spans.slice(tensorflowSpanPosition + tensorflowSpanIdentifier.length + 1, finalPytorchSpanPosition);
+            tensorflowSpans = spans.slice(tensorflowSpanPosition + tensorflowSpanIdentifier.length + 1, pytorchSpanPosition);
            pytorchSpans = spans.slice(pytorchSpanPosition + pytorchSpanIdentifier.length + 1, spans.length);
        }

@@ -44,8 +44,7 @@ extensions = [
    'sphinx.ext.napoleon',
    'recommonmark',
    'sphinx.ext.viewcode',
-    'sphinx_markdown_tables',
-    'sphinx_copybutton'
+    'sphinx_markdown_tables'
 ]

 # Add any paths that contain templates here, relative to this directory.
@@ -75,8 +74,6 @@ exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = None

-# Remove the prompt when copying examples
-copybutton_prompt_text = ">>> "

 # -- Options for HTML output -------------------------------------------------

@@ -45,16 +45,17 @@ tokenizer, which is a `WordPiece <https://arxiv.org/pdf/1609.08144.pdf>`__ token

 ::

-    >>> from transformers import BertTokenizer
-    >>> tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
+    from transformers import BertTokenizer
+    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")

-    >>> sequence = "A Titan RTX has 24GB of VRAM"
+    sequence = "A Titan RTX has 24GB of VRAM"

 The tokenizer takes care of splitting the sequence into tokens available in the tokenizer vocabulary.

 ::

-    >>> tokenized_sequence = tokenizer.tokenize(sequence)
+    tokenized_sequence = tokenizer.tokenize(sequence)
+    print(tokenized_sequence)

 The tokens are either words or subwords. Here for instance, "VRAM" wasn't in the model vocabulary, so it's been split
 in "V", "RA" and "M". To indicate those tokens are not separate words but parts of the same word, a double-dash is
@@ -62,7 +63,6 @@ added for "RA" and "M":

 ::

-    >>> print(tokenized_sequence)
    ['A', 'Titan', 'R', '##T', '##X', 'has', '24', '##GB', 'of', 'V', '##RA', '##M']

 These tokens can then be converted into IDs which are understandable by the model. This can be done by directly feeding
@@ -71,14 +71,14 @@ the sentence to the tokenizer, which leverages the Rust implementation of

 ::

-    >>> encoded_sequence = tokenizer(sequence)["input_ids"]
+    encoded_sequence = tokenizer(sequence)["input_ids"]
+    print(encoded_sequence)

 The tokenizer returns a dictionary with all the arguments necessary for its corresponding model to work properly. The
 token indices are under the key "input_ids":

 ::

-    >>> print(encoded_sequence)
    [101, 138, 18696, 155, 1942, 3190, 1144, 1572, 13745, 1104, 159, 9664, 2107, 102]

 Note that the tokenizer automatically adds "special tokens" (if the associated model rely on them) which are special
@@ -86,14 +86,13 @@ IDs the model sometimes uses. If we decode the previous sequence of ids,

 ::

-    >>> decoded_sequence = tokenizer.decode(encoded_sequence)
+    tokenizer.decode(encoded_sequence)

 we will see 

 ::

-    >>> print(decoded_sequence)
-    [CLS] A Titan RTX has 24GB of VRAM [SEP]
+    '[CLS] A Titan RTX has 24GB of VRAM [SEP]'

 because this is the way a :class:`~transformers.BertModel` is going to expect its inputs.

@@ -109,20 +108,21 @@ For example, consider these two sequences:

 ::

-    >>> from transformers import BertTokenizer
-    >>> tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
+    from transformers import BertTokenizer
+    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")

-    >>> sequence_a = "This is a short sequence."
-    >>> sequence_b = "This is a rather long sequence. It is at least longer than the sequence A."
+    sequence_a = "This is a short sequence."
+    sequence_b = "This is a rather long sequence. It is at least longer than the sequence A."

-    >>> encoded_sequence_a = tokenizer(sequence_a)["input_ids"]
-    >>> encoded_sequence_b = tokenizer(sequence_b)["input_ids"]
+    encoded_sequence_a = tokenizer(sequence_a)["input_ids"]
+    encoded_sequence_b = tokenizer(sequence_b)["input_ids"]
+    
+    len(encoded_sequence_a), len(encoded_sequence_b)

 The encoded versions have different lengths:

 ::

-    >>> len(encoded_sequence_a), len(encoded_sequence_b)
    (8, 19)

 Therefore, we can't be put then together in a same tensor as-is. The first sequence needs to be padded up to the length
@@ -133,14 +133,15 @@ it to pad like this:

 ::

-    >>> padded_sequences = tokenizer([sequence_a, sequence_b], padding=True)
+    padded_sequences = tokenizer([sequence_a, sequence_b], padding=True)
+    padded_sequences["input_ids"]

 We can see that 0s have been added on the right of the first sentence to make it the same length as the second one:

 ::

-    >>> padded_sequences["input_ids"]
-    [[101, 1188, 1110, 170, 1603, 4954, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 1188, 1110, 170, 1897, 1263, 4954, 119, 1135, 1110, 1120, 1655, 2039, 1190, 1103, 4954, 138, 119, 102]]
+    [[101, 1188, 1110, 170, 1603, 4954, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+     [101, 1188, 1110, 170, 1897, 1263, 4954, 119, 1135, 1110, 1120, 1655, 2039, 1190, 1103, 4954, 138, 119, 102]]

 This can then be converted into a tensor in PyTorch or TensorFlow. The attention mask is a binary tensor indicating
 the position of the padded indices so that the model does not attend to them. For the
@@ -149,8 +150,14 @@ a padded value. This attention mask is in the dictionary returned by the tokeniz

 ::

-    >>> padded_sequences["attention_mask"]
-    [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
+    padded_sequences["attention_mask"]
+
+will give back
+
+::
+
+    [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]

 .. _token-type-ids:

@@ -163,27 +170,26 @@ tokens. For example, the BERT model builds its two sequence input as such:

 ::

-   >>> # [CLS] SEQUENCE_A [SEP] SEQUENCE_B [SEP]
+   # [CLS] SEQUENCE_A [SEP] SEQUENCE_B [SEP]

 We can use our tokenizer to automatically generate such a sentence by passing the two sequences as two arguments (and
 not a list like before) like this:

 ::

-    >>> from transformers import BertTokenizer
-    >>> tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
-    >>> sequence_a = "HuggingFace is based in NYC"
-    >>> sequence_b = "Where is HuggingFace based?"
+    from transformers import BertTokenizer
+    tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
+    sequence_a = "HuggingFace is based in NYC"
+    sequence_b = "Where is HuggingFace based?"

-    >>> encoded_dict = tokenizer(sequence_a, sequence_b)
-    >>> decoded = tokenizer.decode(encoded_dict["input_ids"])
+    encoded_dict = tokenizer(sequence_a, sequence_b)
+    tokenizer.decode(encoded_dict["input_ids"])

 which will return:

 ::

-    >>> print(decoded)
-    [CLS] HuggingFace is based in NYC [SEP] Where is HuggingFace based? [SEP]
+    "[CLS] HuggingFace is based in NYC [SEP] Where is HuggingFace based? [SEP]"

 This is enough for some models to understand where one sequence ends and where another begins. However, other models
 such as BERT have an additional mechanism, which are the token type IDs (also called segment IDs). They are a binary
@@ -193,7 +199,12 @@ The tokenizer returns in the dictionary under the key "token_type_ids":

 ::

-    >>> encoded_dict['token_type_ids']
+    encoded_dict['token_type_ids']
+
+will return
+
+::
+
    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]

 The first sequence, the "context" used for the question, has all its tokens represented by :obj:`0`, whereas the
@@ -139,8 +139,6 @@ conversion utilities for the following models:

    task_summary
    model_summary
-    training
-    preprocessing
    serialization
    model_sharing
    multilingual
@@ -36,11 +36,10 @@ Here is an example using the ``xlm-clm-enfr-1024`` checkpoint (Causal language m

 .. code-block::

-    >>> import torch
-    >>> from transformers import XLMTokenizer, XLMWithLMHeadModel
+    import torch
+    from transformers import XLMTokenizer, XLMWithLMHeadModel

-    >>> tokenizer = XLMTokenizer.from_pretrained("xlm-clm-enfr-1024")
-    >>> model = XLMWithLMHeadModel.from_pretrained("xlm-clm-enfr-1024")
+    tokenizer = XLMTokenizer.from_pretrained("xlm-clm-1024-enfr")


 The different languages this model/tokenizer handles, as well as the ids of these languages are visible using the
@@ -48,15 +47,16 @@ The different languages this model/tokenizer handles, as well as the ids of thes

 .. code-block::

-    >>> print(tokenizer.lang2id)
-    {'en': 0, 'fr': 1}
+    # Continuation of the previous script
+    print(tokenizer.lang2id)  # {'en': 0, 'fr': 1}


 These ids should be used when passing a language parameter during a model pass. Let's define our inputs:

 .. code-block::

-    >>> input_ids = torch.tensor([tokenizer.encode("Wikipedia was used to")]) # batch size of 1
+    # Continuation of the previous script
+    input_ids = torch.tensor([tokenizer.encode("Wikipedia was used to")]) # batch size of 1


 We should now define the language embedding by using the previously defined language id. We want to create a tensor
@@ -64,18 +64,20 @@ filled with the appropriate language ids, of the same size as input_ids. For eng

 .. code-block::

-    >>> language_id = tokenizer.lang2id['en']  # 0
-    >>> langs = torch.tensor([language_id] * input_ids.shape[1])  # torch.tensor([0, 0, 0, ..., 0])
+    # Continuation of the previous script
+    language_id = tokenizer.lang2id['en']  # 0
+    langs = torch.tensor([language_id] * input_ids.shape[1])  # torch.tensor([0, 0, 0, ..., 0])

-    >>> # We reshape it to be of size (batch_size, sequence_length)
-    >>> langs = langs.view(1, -1) # is now of shape [1, sequence_length] (we have a batch size of 1)
+    # We reshape it to be of size (batch_size, sequence_length)
+    langs = langs.view(1, -1) # is now of shape [1, sequence_length] (we have a batch size of 1)


 You can then feed it all as input to your model:

 .. code-block::

-    >>> outputs = model(input_ids, langs=langs)
+    # Continuation of the previous script
+    outputs = model(input_ids, langs=langs)


 The example `run_generation.py <https://github.com/huggingface/transformers/blob/master/examples/text-generation/run_generation.py>`__
@@ -1,373 +0,0 @@
-Preprocessing data
-==================
-
-In this tutorial, we'll explore how to preprocess your data using 🤗 Transformers. The main tool for this is what we
-
-call a :doc:`tokenizer <main_classes/tokenizer>`. You can build one using the tokenizer class associated to the model
-you would like to use, or directly with the :class:`~transformers.AutoTokenizer` class.
-
-As we saw in the :doc:`quicktour </quicktour>`, the tokenizer will first split a given text in words (or part of words,
-punctuation symbols, etc.) usually called `tokens`. Then it will convert those `tokens` into numbers, to be able to
-build a tensor out of them and feed them to the model. It will also add any additional inputs the model might expect to
-work properly.
-
-.. note::
-
-    If you plan on using a pretrained model, it's important to use the associated pretrained tokenizer: it will split
-    the text you give it in tokens the same way for the pretraining corpus, and it will use the same correspondence
-    token to index (that we usually call a `vocab`) as during pretraining.
-
-To automatically download the vocab used during pretraining or fine-tuning a given model, you can use the 
-:func:`~transformers.AutoTokenizer.from_pretrained` method:
-
-::
-
-    from transformers import AutoTokenizer
-    tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
-
-Base use
-~~~~~~~~
-
-A :class:`~transformers.PreTrainedTokenizer` has many methods, but the only one you need to remember for preprocessing
-is its ``__call__``: you just need to feed your sentence to your tokenizer object.
-
-::
-
-    encoded_input = tokenizer("Hello, I'm a single sentence!")
-    print(encoded_input)
-
-This will return a dictionary string to list of ints like this one:
-
-::
-
-    {'input_ids': [101, 138, 18696, 155, 1942, 3190, 1144, 1572, 13745, 1104, 159, 9664, 2107, 102], 
-     'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
-     'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
-
-The `input_ids <glossary.html#input-ids>`__ are the indices corresponding to each token in our sentence. We will see
-below what the `attention_mask <glossary.html#attention-mask>`__ is used for and in
-:ref:`the next section <sentence-pairs>` the goal of `token_type_ids <glossary.html#token-type-ids>`__.
-
-The tokenizer can decode a list of token ids in a proper sentence:
-
-::
-
-    tokenizer.decode(encoded_input["input_ids"])
-
-which should return
-
-::
-
-    "[CLS] Hello, I'm a single sentence! [SEP]"
-
-As you can see, the tokenizer automatically added some special tokens that the model expect. Not all model need special
-tokens; for instance, if we had used` gtp2-medium` instead of `bert-base-cased` to create our tokenizer, we would have
-
-seen the same sentence as the original one here. You can disable this behavior (which is only advised if you have added
-those special tokens yourself) by passing ``add_special_tokens=False``.
-
-If you have several sentences you want to process, you can do this efficiently by sending them as a list to the
-tokenizer:
-
-::
-
-    batch_sentences = ["Hello I'm a single sentence",
-                       "And another sentence",
-                       "And the very very last one"]
-    encoded_inputs = tokenizer(batch_sentences)
-    print(encoded_inputs)
-
-We get back a dictionary once again, this time with values being list of list of ints:
-
-::
-
-    {'input_ids': [[101, 8667, 146, 112, 182, 170, 1423, 5650, 102],
-                   [101, 1262, 1330, 5650, 102],
-                   [101, 1262, 1103, 1304, 1304, 1314, 1141, 102]],
-     'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0],
-                        [0, 0, 0, 0, 0],
-                        [0, 0, 0, 0, 0, 0, 0, 0]],
-     'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1],
-                        [1, 1, 1, 1, 1],
-                        [1, 1, 1, 1, 1, 1, 1, 1]]}
-
-If the purpose of sending several sentences at a time to the tokenizer is to build a batch to feed the model, you will
-probably want:
-
- To pad each sentence to the maximum length there is in your batch.
- To truncate each sentence to the maximum length the model can accept (if applicable).
- To return tensors.
-
-You can do all of this by using the following options when feeding your list of sentences to the tokenizer:
-
-::
-
-    ## PYTORCH CODE
-    batch = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="pt")
-    print(batch)
-    ## TENSORFLOW CODE
-    batch = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="tf")
-    print(batch)
-
-which should now return a dictionary string to tensor like this:
-
-::
-
-    {'input_ids': tensor([[ 101, 8667,  146,  112,  182,  170, 1423, 5650,  102],
-                          [ 101, 1262, 1330, 5650,  102,    0,    0,    0,    0],
-                          [ 101, 1262, 1103, 1304, 1304, 1314, 1141,  102,    0]]),
-     'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0],
-                               [0, 0, 0, 0, 0, 0, 0, 0, 0],
-                               [0, 0, 0, 0, 0, 0, 0, 0, 0]]), 
-     'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1],
-                               [1, 1, 1, 1, 1, 0, 0, 0, 0],
-                               [1, 1, 1, 1, 1, 1, 1, 1, 0]])}
-
-We can now see what the `attention_mask <glossary.html#attention-mask>`__ is all about: it points out which tokens the
-model should pay attention to and which ones it should not (because they represent padding in this case).
-
-
-Note that if your model does not have a maximum length associated to it, the command above will throw a warning. You
-can safely ignore it. You can also pass ``verbose=False`` to stop the tokenizer to throw those kinds of warnings.
-
-.. _sentence-pairs:
-
-Preprocessing pairs of sentences
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Sometimes you need to feed pair of sentences to your model. For instance, if you want to classify if two sentences in a
-pair are similar, or for question-answering models, which take a context and a question. For BERT models, the input is
-then represented like this:
-
-::
-
-    [CLS] Sequence A [SEP] Sequence B [SEP]
-
-You can encode a pair of sentences in the format expected by your model by supplying the two sentences as two arguments
-
-(not a list since a list of two sentences will be interpreted as a batch of two single sentences, as we saw before).
-
-
-::
-
-    encoded_input = tokenizer("How old are you?", "I'm 6 years old")
-    print(encoded_input)
-
-This will once again return a dict string to list of ints:
-
-::
-
-    {'input_ids': [101, 1731, 1385, 1132, 1128, 136, 102, 146, 112, 182, 127, 1201, 1385, 102], 
-     'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1], 
-     'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
-
-This shows us what the `token_type_ids <glossary.html#token-type-ids>`__ are for: they indicate to the model which part
-of the inputs correspond to the first sentence and which part corresponds to the second sentence. Note that
-`token_type_ids` are not required or handled by all models. By default, a tokenizer will only return the inputs that
-its associated model expects. You can force the return (or the non-return) of any of those special arguments by
-using ``return_input_ids`` or ``return_token_type_ids``.
-
-If we decode the token ids we obtained, we will see that the special tokens have been properly added.
-
-::
-
-    tokenizer.decode(encoded_input["input_ids"])
-
-will return:
-
-::
-
-    "[CLS] How old are you? [SEP] I'm 6 years old [SEP]"
-
-If you have a list of pairs of sequences you want to process, you should feed them as two lists to your tokenizer: the
-list of first sentences and the list of second sentences:
-
-::
-
-    batch_sentences = ["Hello I'm a single sentence",
-                       "And another sentence",
-                       "And the very very last one"]
-    batch_of_second_sentences = ["I'm a sentence that goes with the first sentence",
-                                 "And I should be encoded with the second sentence",
-                                 "And I go with the very last one"]
-    encoded_inputs = tokenizer(batch_sentences, batch_of_second_sentences)
-    print(encoded_inputs)
-
-will return a dict with the values being list of lists of ints:
-
-::
-
-    {'input_ids': [[101, 8667, 146, 112, 182, 170, 1423, 5650, 102, 146, 112, 182, 170, 5650, 1115, 2947, 1114, 1103, 1148, 5650, 102], 
-                   [101, 1262, 1330, 5650, 102, 1262, 146, 1431, 1129, 12544, 1114, 1103, 1248, 5650, 102], 
-                   [101, 1262, 1103, 1304, 1304, 1314, 1141, 102, 1262, 146, 1301, 1114, 1103, 1304, 1314, 1141, 102]], 
-    'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 
-                       [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 
-                       [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 
-    'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 
-                       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 
-                       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
-
-To double-check what is fed to the model, we can decode each list in `input_ids` one by one:
-
-::
-
-    for ids in encoded_inputs["input_ids"]:
-        print(tokenizer.decode(ids))
-
-which will return:
-
-::
-
-    [CLS] Hello I'm a single sentence [SEP] I'm a sentence that goes with the first sentence [SEP]
-    [CLS] And another sentence [SEP] And I should be encoded with the second sentence [SEP]
-    [CLS] And the very very last one [SEP] And I go with the very last one [SEP]
-
-Once again, you can automatically pad your inputs to the maximum sentence length in the batch, truncate to the maximum
-length the model can accept and return tensors directly with the following:
-
-::
-
-    ## PYTORCH CODE
-    batch = tokenizer(batch_sentences, batch_of_second_sentences, padding=True, truncation=True, return_tensors="pt")
-    ## TENSORFLOW CODE
-    batch = tokenizer(batch_sentences, batch_of_second_sentences, padding=True, truncation=True, return_tensors="tf")
-
-Everything you always wanted to know about padding and truncation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-We have seen the commands that will work for most cases (pad your batch to the length of the maximum sentence and
-
-truncate to the maximum length the mode can accept). However, the API supports more strategies if you need them. The
-three arguments you need to know for this are :obj:`padding`, :obj:`truncation` and :obj:`max_length`.
-
- :obj:`padding` controls the padding. It can be a boolean or a string which should be:
-
-    - :obj:`True` or :obj:`'longest'` to pad to the longest sequence in the batch (doing no padding if you only provide
-      a single sequence).
-    - :obj:`'max_length'` to pad to a length specified by the :obj:`max_length` argument or the maximum length accepted
-      by the model if no :obj:`max_length` is provided (``max_length=None``). If you only provide a single sequence,
-      padding will still be applied to it. 
-    - :obj:`False` or :obj:`'do_not_pad'` to not pad the sequences. As we have seen before, this is the default
-      behavior.
-
- :obj:`truncation` controls the truncation. It can be a boolean or a string which should be:
-
-    - :obj:`True` or :obj:`'only_first'` truncate to a maximum length specified by the :obj:`max_length` argument or
-      the maximum length accepted by the model if no :obj:`max_length` is provided (``max_length=None``). This will
-      only truncate the first sentence of a pair if a pair of sequence (or a batch of pairs of sequences) is provided.
-    - :obj:`'only_second'` truncate to a maximum length specified by the :obj:`max_length` argument or the maximum
-      length accepted by the model if no :obj:`max_length` is provided (``max_length=None``). This will only truncate
-      the second sentence of a pair if a pair of sequence (or a batch of pairs of sequences) is provided.
-    - :obj:`'longest_first'` truncate to a maximum length specified by the :obj:`max_length` argument or the maximum
-      length accepted by the model if no :obj:`max_length` is provided (``max_length=None``). This will truncate token
-      by token, removing a token from the longest sequence in the pair until the proper length is reached.
-    - :obj:`False` or :obj:`'do_not_truncate'` to not truncate the sequences. As we have seen before, this is the
-      default behavior.
-
- :obj:`max_length` to control the length of the padding/truncation. It can be an integer or :obj:`None`, in which case
-  it will default to the maximum length the model can accept. If the model has no specific maximum input length,
-  truncation/padding to :obj:`max_length` is deactivated.
-
-Here is a table summarizing the recommend way to setup padding and truncation. If you use pair of inputs sequence in
-any of the following examples, you can replace :obj:`truncation=True` by a :obj:`STRATEGY` selected in 
-:obj:`['only_first', 'only_second', 'longest_first']`, i.e. :obj:`truncation='only_second'` or
-:obj:`truncation= 'longest_first'` to control how both sequence in the pair are truncated as detailed before.
-
-+--------------------------------------+-----------------------------------+---------------------------------------------------------------------------------------------+
-| Truncation                           | Padding                           | Instruction                                                                                 |
-+======================================+===================================+=============================================================================================+
-| no truncation                        | no padding                        | :obj:`tokenizer(batch_sentences)`                                                           |
-|                                      +-----------------------------------+---------------------------------------------------------------------------------------------+
-|                                      | padding to max sequence in batch  | :obj:`tokenizer(batch_sentences, padding=True)` or                                          |
-|                                      |                                   | :obj:`tokenizer(batch_sentences, padding='longest')`                                        |
-|                                      +-----------------------------------+---------------------------------------------------------------------------------------------+
-|                                      | padding to max model input length | :obj:`tokenizer(batch_sentences, padding='max_length')`                                     |
-|                                      +-----------------------------------+---------------------------------------------------------------------------------------------+
-|                                      | padding to specific length        | :obj:`tokenizer(batch_sentences, padding='max_length', max_length=42)`                      |
-+--------------------------------------+-----------------------------------+---------------------------------------------------------------------------------------------+
-| truncation to max model input length | no padding                        | :obj:`tokenizer(batch_sentences, truncation=True)` or                                       |
-|                                      |                                   | :obj:`tokenizer(batch_sentences, truncation=STRATEGY)`                                      |
-|                                      +-----------------------------------+---------------------------------------------------------------------------------------------+
-|                                      | padding to max sequence in batch  | :obj:`tokenizer(batch_sentences, padding=True, truncation=True)` or                         |
-|                                      |                                   | :obj:`tokenizer(batch_sentences, padding=True, truncation=STRATEGY)`                        |
-|                                      +-----------------------------------+---------------------------------------------------------------------------------------------+
-|                                      | padding to max model input length | :obj:`tokenizer(batch_sentences, padding='max_length', truncation=True)` or                 |
-|                                      |                                   | :obj:`tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY)`                |
-|                                      +-----------------------------------+---------------------------------------------------------------------------------------------+
-|                                      | padding to specific length        | Not possible                                                                                |
-+--------------------------------------+-----------------------------------+---------------------------------------------------------------------------------------------+
-| truncation to specific length        | no padding                        | :obj:`tokenizer(batch_sentences, truncation=True, max_length=42)` or                        |
-|                                      |                                   | :obj:`tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)`                       |
-|                                      +-----------------------------------+---------------------------------------------------------------------------------------------+
-|                                      | padding to max sequence in batch  | :obj:`tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)` or          |
-|                                      |                                   | :obj:`tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)`         |
-|                                      +-----------------------------------+---------------------------------------------------------------------------------------------+
-|                                      | padding to max model input length | Not possible                                                                                |
-|                                      +-----------------------------------+---------------------------------------------------------------------------------------------+
-|                                      | padding to specific length        | :obj:`tokenizer(batch_sentences, padding='max_length', truncation=True, max_length=42)` or  |
-|                                      |                                   | :obj:`tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY, max_length=42)` |
-+--------------------------------------+-----------------------------------+---------------------------------------------------------------------------------------------+
-
-Pre-tokenized inputs
-~~~~~~~~~~~~~~~~~~~~
-
-The tokenizer also accept pre-tokenized inputs. This is particularly useful when you want to compute labels and extract
-predictions in `named entity recognition (NER) <https://en.wikipedia.org/wiki/Named-entity_recognition>`__ or
-`part-of-speech tagging (POS tagging) <https://en.wikipedia.org/wiki/Part-of-speech_tagging>`__.
-
-If you want to use pre-tokenized inputs, just set :obj:`is_pretokenized=True` when passing your inputs to the
-tokenizer. For instance:
-
-::
-
-    encoded_input = tokenizer(["Hello", "I'm", "a", "single", "sentence"], is_pretokenized=True)
-    print(encoded_input)
-
-will return:
-
-::
-
-    {'input_ids': [101, 8667, 146, 112, 182, 170, 1423, 5650, 102],
-     'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0], 
-     'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}
-
-Note that the tokenizer still adds the ids of special tokens (if applicable) unless you pass
-``add_special_tokens=False``.
-
-This works exactly as before for batch of sentences or batch of pairs of sentences. You can encode a batch of sentences
-like this:
-
-::
-
-    batch_sentences = [["Hello", "I'm", "a", "single", "sentence"],
-                       ["And", "another", "sentence"],
-                       ["And", "the", "very", "very", "last", "one"]]
-    encoded_inputs = tokenizer(batch_sentences, is_pretokenized=True)
-
-or a batch of pair sentences like this:
-
-::
-
-    batch_of_second_sentences = [["I'm", "a", "sentence", "that", "goes", "with", "the", "first", "sentence"],
-                                 ["And", "I", "should", "be", "encoded", "with", "the", "second", "sentence"],
-                                 ["And", "I", "go", "with", "the", "very", "last", "one"]]
-    encoded_inputs = tokenizer(batch_sentences, batch_of_second_sentences, is_pretokenized=True)
-
-And you can add padding, truncation as well as directly return tensors like before:
-
-::
-
-    ## PYTORCH CODE
-    batch = tokenizer(batch_sentences,
-                      batch_of_second_sentences,
-                      is_pretokenized=True,
-                      padding=True,
-                      truncation=True,
-                      return_tensors="pt")
-    ## TENSORFLOW CODE
-    batch = tokenizer(batch_sentences,
-                      batch_of_second_sentences,
-                      is_pretokenized=True,
-                      padding=True,
-                      truncation=True,
-                      return_tensors="tf")
@@ -1,393 +1,378 @@
-Quick tour
-==========
-
-Let's have a quick look at the 🤗 Transformers library features. The library downloads pretrained models for
-Natural Language Understanding (NLU) tasks, such as analyzing the sentiment of a text, and Natural Language Generation (NLG),
-such as completing a prompt with new text or translating in another language.
-
-First we will see how to easily leverage the pipeline API to quickly use those pretrained models at inference. Then, we
-will dig a little bit more and see how the library gives you access to those models and helps you preprocess your data.
-
-.. note::
-
-    All code examples presented in the documentation have a switch on the top left for Pytorch versus TensorFlow. If
-    not, the code is expected to work for both backends without any change needed.
-
-Getting started on a task with a pipeline
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The easiest way to use a pretrained model on a given task is to use :func:`~transformers.pipeline`. 🤗 Transformers
-provides the following tasks out of the box:
-
- Sentiment analysis: is a text positive or negative?
- Text generation (in English): provide a prompt and the model will generate what follows.
- Name entity recognition (NER): in an input sentence, label each word with the entity it represents (person, place,
-  etc.)
- Question answering: provide the model with some context and a question, extract the answer from the context.
- Filling masked text: given a text with masked words (e.g., replaced by ``[MASK]``), fill the blanks.
- Summarization: generate a summary of a long text.
- Translation: translate a text in another language.
- Feature extraction: return a tensor representation of the text.
-
-Let's see how this work for sentiment analysis (the other tasks are all covered in the
-:doc:`task summary </task_summary>`):
-
-.. code-block::
-
-    >>> from transformers import pipeline
-    >>> classifier = pipeline('sentiment-analysis')
-
-When typing this command for the first time, a pretrained model and its tokenizer are downloaded and cached. We will
-look at both later on, but as an introduction the tokenizer's job is to preprocess the text for the model, which is
-then responsible for making predictions. The pipeline groups all of that together, and post-process the predictions to
-make them readable. For instance:
-
-
-.. code-block::
-
-    >>> classifier('We are very happy to show you the 🤗 Transformers library.')
-    [{'label': 'POSITIVE', 'score': 0.9997795224189758}]
-
-That's encouraging! You can use it on a list of sentences, which will be preprocessed then fed to the model as a
-`batch`, returning a list of dictionaries like this one:
-
-.. code-block::
-
-    >>> results = classifier(["We are very happy to show you the 🤗 Transformers library.",
-    ...            "We hope you don't hate it."])
-    >>> for result in results:
-    ...     print(f"label: {result['label']}, with score: {round(result['score'], 4)}")
-    label: POSITIVE, with score: 0.9998
-    label: NEGATIVE, with score: 0.5309
-
-You can see the second sentence has been classified as negative (it needs to be positive or negative) but its score is
-fairly neutral.
-
-By default, the model downloaded for this pipeline is called "distilbert-base-uncased-finetuned-sst-2-english". We can
-look at its `model page <https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english>`__ to get more
-information about it. It uses the :doc:`DistilBERT architecture </model_doc/distilbert>` and has been fine-tuned on a
-dataset called SST-2 for the sentiment analysis task.
-
-Let's say we want to use another model; for instance, one that has been trained on French data. We can search through
-the `model hub <https://huggingface.co/models>`__ that gathers models pretrained on a lot of data by research labs, but
-also community models (usually fine-tuned versions of those big models on a specific dataset). Applying the tags
-"French" and "text-classification" gives back a suggestion "nlptown/bert-base-multilingual-uncased-sentiment". Let's
-see how we can use it.
-
-You can directly pass the name of the model to use to :func:`~transformers.pipeline`:
-
-.. code-block::
-
-    >>> classifier = pipeline('sentiment-analysis', model="nlptown/bert-base-multilingual-uncased-sentiment")
-
-This classifier can now deal with texts in English, French, but also Dutch, German, Italian and Spanish! You can also
-replace that name by a local folder where you have saved a pretrained model (see below). You can also pass a model
-object and its associated tokenizer.
-
-We will need two classes for this. The first is :class:`~transformers.AutoTokenizer`, which we will use to download the
-tokenizer associated to the model we picked and instantiate it. The second is
-:class:`~transformers.AutoModelForSequenceClassification` (or
-:class:`~transformers.TFAutoModelForSequenceClassification` if you are using TensorFlow), which we will use to download
-the model itself. Note that if we were using the library on an other task, the class of the model would change. The
-:doc:`task summary </task_summary>` tutorial summarizes which class is used for which task.
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> from transformers import AutoTokenizer, AutoModelForSequenceClassification
-    >>> ## TENSORFLOW CODE
-    >>> from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
-
-Now, to download the models and tokenizer we found previously, we just have to use the
-:func:`~transformers.AutoModelForSequenceClassification.from_pretrained` method (feel free to replace ``model_name`` by
-any other model from the model hub):
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
-    >>> model = AutoModelForSequenceClassification.from_pretrained(model_name)
-    >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
-    >>> pipe = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
-    >>> ## TENSORFLOW CODE
-    >>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
-    >>> # This model only exists in PyTorch, so we use the `from_pt` flag to import that model in TensorFlow.
-    >>> model = TFAutoModelForSequenceClassification.from_pretrained(model_name, from_pt=True) 
-    >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
-    >>> classifier = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
-
-If you don't find a model that has been pretrained on some data similar to yours, you will need to fine-tune a
-pretrained model on your data. We provide :doc:`example scripts </examples>` to do so. Once you're done, don't forget
-to share your fine-tuned model on the hub with the community, using :doc:`this tutorial </model_sharing>`.
-
-.. _pretrained-model:
-
-Under the hood: pretrained models
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Let's now see what happens beneath the hood when using those pipelines. As we saw, the model and tokenizer are created
-using the :obj:`from_pretrained` method:
-
-::
-
-    >>> ## PYTORCH CODE
-    >>> from transformers import AutoTokenizer, AutoModelForSequenceClassification
-    >>> model_name = "distilbert-base-uncased-finetuned-sst-2-english"
-    >>> pt_model = AutoModelForSequenceClassification.from_pretrained(model_name)
-    >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
-    >>> ## TENSORFLOW CODE
-    >>> from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
-    >>> model_name = "distilbert-base-uncased-finetuned-sst-2-english"
-    >>> tf_model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
-    >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-Using the tokenizer
-^^^^^^^^^^^^^^^^^^^
-
-We mentioned the tokenizer is responsible for the preprocessing of your texts. First, it will split a given text in
-words (or part of words, punctuation symbols, etc.) usually called `tokens`. There are multiple rules that can govern
-that process, which is why we need to instantiate the tokenizer using the name of the model, to make sure we use the
-same rules as when the model was pretrained.
-
-The second step is to convert those `tokens` into numbers, to be able to build a tensor out of them and feed them to
-the model. To do this, the tokenizer has a `vocab`, which is the part we download when we instantiate it with the
-:obj:`from_pretrained` method, since we need to use the same `vocab` as when the model was pretrained.
-
-To apply these steps on a given text, we can just feed it to our tokenizer:
-
-.. code-block::
-
-    >>> inputs = tokenizer("We are very happy to show you the 🤗 Transformers library.")
-
-This returns a dictionary string to list of ints. It contains the `ids of the tokens <glossary.html#input-ids>`__,
-as mentioned before, but also additional arguments that will be useful to the model. Here for instance, we also have an
-`attention mask <glossary.html#attention-mask>`__ that the model will use to have a better understanding of the sequence:
-
-
-.. code-block::
-
-    >>> print(inputs)
-    {'input_ids': [101, 2057, 2024, 2200, 3407, 2000, 2265, 2017, 1996, 100, 19081, 3075, 1012, 102], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
-
-You can pass a list of sentences directly to your tokenizer. If your goal is to send them through your model as a
-batch, you probably want to pad them all to the same length, truncate them to the maximum length the model can accept
-and get tensors back. You can specify all of that to the tokenizer:
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> pt_batch = tokenizer(
-    ...     ["We are very happy to show you the 🤗 Transformers library.", "We hope you don't hate it."],
-    ...     padding=True,
-    ...     truncation=True,
-    ...     return_tensors="pt"
-    ... )
-    >>> ## TENSORFLOW CODE
-    >>> tf_batch = tokenizer(
-    ...     ["We are very happy to show you the 🤗 Transformers library.", "We hope you don't hate it."],
-    ...     padding=True,
-    ...     truncation=True,
-    ...     return_tensors="tf"
-    ... )
-
-The padding is automatically applied on the side the model expect it (in this case, on the right), with the
-padding token the model was pretrained with. The attention mask is also adapted to take the padding into account:
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> for key, value in pt_batch.items():
-    ...     print(f"{key}: {value.numpy().tolist()}")
-    input_ids: [[101, 2057, 2024, 2200, 3407, 2000, 2265, 2017, 1996, 100, 19081, 3075, 1012, 102], [101, 2057, 3246, 2017, 2123, 1005, 1056, 5223, 2009, 1012, 102, 0, 0, 0]]
-    attention_mask: [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]]
-    >>> ## TENSORFLOW CODE
-    >>> for key, value in tf_batch.items():
-    ...     print(f"{key}: {value.numpy().tolist()}")
-    input_ids: [[101, 2057, 2024, 2200, 3407, 2000, 2265, 2017, 1996, 100, 19081, 3075, 1012, 102], [101, 2057, 3246, 2017, 2123, 1005, 1056, 5223, 2009, 1012, 102, 0, 0, 0]]
-    attention_mask: [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]]
-
-You can learn more about tokenizers :doc:`here <preprocessing>`.
-
-Using the model
-^^^^^^^^^^^^^^^
-
-Once your input has been preprocessed by the tokenizer, you can directly send it to the model. As we mentioned, it will
-contain all the relevant information the model needs. If you're using a TensorFlow model, you can directly pass the
-dictionary keys to tensor, for a PyTorch model, you need to unpack the dictionary by adding :obj:`**`.
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> pt_outputs = pt_model(**pt_batch)
-    >>> ## TENSORFLOW CODE
-    >>> tf_outputs = tf_model(tf_batch)
-
-In 🤗 Transformers, all outputs are tuples (with only one element potentially). Here, we get a tuple with just the
-final activations of the model.
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> print(pt_outputs)
-    (tensor([[-4.0833,  4.3364],
-            [ 0.0818, -0.0418]], grad_fn=<AddmmBackward>),)
-    >>> ## TENSORFLOW CODE
-    >>> print(tf_outputs)
-    (<tf.Tensor: shape=(2, 2), dtype=float32, numpy=
-    array([[-4.0832963 ,  4.3364134 ],
-           [ 0.08181238, -0.04178794]], dtype=float32)>,)
-
-.. note::
-
-    All 🤗 Transformers models (PyTorch or TensorFlow) return the activations of the model *before* the final
-    activation function (like SoftMax) since this final activation function is often fused with the loss.
-
-Let's apply the SoftMax activation to get predictions.
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> import torch.nn.functional as F
-    >>> pt_predictions = F.softmax(pt_outputs[0], dim=-1)
-    >>> ## TENSORFLOW CODE
-    >>> import tensorflow as tf
-    >>> tf_predictions = tf.nn.softmax(tf_outputs[0], axis=-1)
-
-We can see we get the numbers from before:
-
-.. code-block::
-
-    >>> ## TENSORFLOW CODE
-    >>> print(tf_predictions)
-    tf.Tensor(
-    [[2.2042994e-04 9.9977952e-01]
-     [5.3086078e-01 4.6913919e-01]], shape=(2, 2), dtype=float32)
-    >>> ## PYTORCH CODE
-    >>> print(pt_predictions)
-    tensor([[2.2043e-04, 9.9978e-01],
-            [5.3086e-01, 4.6914e-01]], grad_fn=<SoftmaxBackward>)
-
-If you have labels, you can provide them to the model, it will return a tuple with the loss and the final activations.
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> import torch
-    >>> pt_outputs = pt_model(**pt_batch, labels = torch.tensor([1, 0]))
-    >>> ## TENSORFLOW CODE
-    >>> import tensorflow as tf
-    >>> tf_outputs = tf_model(tf_batch, labels = tf.constant([1, 0]))
-
-Models are standard `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__ or
-`tf.keras.Model <https://www.tensorflow.org/api_docs/python/tf/keras/Model>`__ so you can use them in your usual
-training loop. 🤗 Transformers also provides a :class:`~transformers.Trainer` (or :class:`~transformers.TFTrainer` if
-you are using TensorFlow) class to help with your training (taking care of things such as distributed training, mixed
-precision, etc.). See the training tutorial (coming soon) for more details.
-
-Once your model is fine-tuned, you can save it with its tokenizer the following way:
-
-::
-
-    tokenizer.save_pretrained(save_directory)
-    model.save_pretrained(save_directory)
-
-You can then load this model back using the :func:`~transformers.AutoModel.from_pretrained` method by passing the
-directory name instead of the model name. One cool feature of 🤗 Transformers is that you can easily switch between
-PyTorch and TensorFlow: any model saved as before can be loaded back either in PyTorch or TensorFlow. If you are
-loading a saved PyTorch model in a TensorFlow model, use :func:`~transformers.TFAutoModel.from_pretrained` like this:
-
-::
-
-    tokenizer = AutoTokenizer.from_pretrained(save_directory)
-    model = TFAutoModel.from_pretrained(save_directory, from_pt=True)
-
-and if you are loading a saved TensorFlow model in a PyTorch model, you should use the following code:
-
-::
-
-    tokenizer = AutoTokenizer.from_pretrained(save_directory)
-    model = AutoModel.from_pretrained(save_directory, from_tf=True)
-
-Lastly, you can also ask the model to return all hidden states and all attention weights if you need them:
-
-
-::
-
-    >>> ## PYTORCH CODE
-    >>> pt_outputs = pt_model(**pt_batch, output_hidden_states=True, output_attentions=True)
-    >>> all_hidden_states, all_attentions = pt_outputs[-2:]
-    >>> ## TENSORFLOW CODE
-    >>> tf_outputs = tf_model(tf_batch, output_hidden_states=True, output_attentions=True)
-    >>> all_hidden_states, all_attentions = tf_outputs[-2:]
-
-Accessing the code
-^^^^^^^^^^^^^^^^^^
-
-The :obj:`AutoModel` and :obj:`AutoTokenizer` classes are just shortcuts that will automatically work with any
-pretrained model. Behind the scenes, the library has one model class per combination of architecture plus class, so the
-code is easy to access and tweak if you need to.
-
-In our previous example, the model was called "distilbert-base-uncased-finetuned-sst-2-english", which means it's
-using the :doc:`DistilBERT </model_doc/distilbert>` architecture. The model automatically created is then a
-:class:`~transformers.DistilBertForSequenceClassification`. You can look at its documentation for all details relevant
-to that specific model, or browse the source code. This is how you would directly instantiate model and tokenizer
-without the auto magic:
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> from transformers import DistilBertTokenizer, DistilBertForSequenceClassification
-    >>> model_name = "distilbert-base-uncased-finetuned-sst-2-english"
-    >>> model = DistilBertForSequenceClassification.from_pretrained(model_name)
-    >>> tokenizer = DistilBertTokenizer.from_pretrained(model_name)
-    >>> ## TENSORFLOW CODE
-    >>> from transformers import DistilBertTokenizer, TFDistilBertForSequenceClassification
-    >>> model_name = "distilbert-base-uncased-finetuned-sst-2-english"
-    >>> model = TFDistilBertForSequenceClassification.from_pretrained(model_name)
-    >>> tokenizer = DistilBertTokenizer.from_pretrained(model_name)
-
-Customizing the model
-^^^^^^^^^^^^^^^^^^^^^
-
-If you want to change how the model itself is built, you can define your custom configuration class. Each architecture
-comes with its own relevant configuration (in the case of DistilBERT, :class:`~transformers.DistilBertConfig`) which
-allows you to specify any of the hidden dimension, dropout rate etc. If you do core modifications, like changing the
-hidden size, you won't be able to use a pretrained model anymore and will need to train from scratch. You would then
-instantiate the model directly from this configuration.
-
-Here we use the predefined vocabulary of DistilBERT (hence load the tokenizer with the
-:func:`~transformers.DistilBertTokenizer.from_pretrained` method) and initialize the model from scratch (hence
-instantiate the model from the configuration instead of using the
-:func:`~transformers.DistilBertForSequenceClassification.from_pretrained` method).
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
-    >>> config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
-    >>> tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
-    >>> model = DistilBertForSequenceClassification(config)
-    >>> ## TENSORFLOW CODE
-    >>> from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
-    >>> config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
-    >>> tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
-    >>> model = TFDistilBertForSequenceClassification(config)
-
-For something that only changes the head of the model (for instance, the number of labels), you can still use a
-pretrained model for the body. For instance, let's define a classifier for 10 different labels using a pretrained body.
-We could create a configuration with all the default values and just change the number of labels, but more easily, you
-can directly pass any argument a configuration would take to the :func:`from_pretrained` method and it will update the
-default configuration with it:
-
-.. code-block::
-
-    >>> ## PYTORCH CODE
-    >>> from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
-    >>> model_name = "distilbert-base-uncased"
-    >>> model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
-    >>> tokenizer = DistilBertTokenizer.from_pretrained(model_name)
-    >>> ## TENSORFLOW CODE
-    >>> from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
-    >>> model_name = "distilbert-base-uncased"
-    >>> model = TFDistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
-    >>> tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+Quick tour
+==========
+
+Let's have a quick look at the 🤗 Transformers library features. The library downloads pretrained models for
+Natural Language Understanding (NLU) tasks, such as analyzing the sentiment of a text, and Natural Language Generation (NLG),
+such as completing a prompt with new text or translating in another language.
+
+First we will see how to easily leverage the pipeline API to quickly use those pretrained models at inference. Then, we
+will dig a little bit more and see how the library gives you access to those models and helps you preprocess your data.
+
+.. note::
+
+    All code examples presented in the documentation have a switch on the top left for Pytorch versus TensorFlow. If
+    not, the code is expected to work for both backends without any change needed.
+
+Getting started on a task with a pipeline
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The easiest way to use a pretrained model on a given task is to use :func:`~transformers.pipeline`. 🤗 Transformers
+provides the following tasks out of the box:
+
+- Sentiment analysis: is a text positive or negative?
+- Text generation (in English): provide a prompt and the model will generate what follows.
+- Name entity recognition (NER): in an input sentence, label each word with the entity it represents (person, place,
+  etc.)
+- Question answering: provide the model with some context and a question, extract the answer from the context.
+- Filling masked text: given a text with masked words (e.g., replaced by ``[MASK]``), fill the blanks.
+- Summarization: generate a summary of a long text.
+- Translation: translate a text in another language.
+- Feature extraction: return a tensor representation of the text.
+
+Let's see how this work for sentiment analysis (the other tasks are all covered in the
+:doc:`task summary </task_summary>`):
+
+::
+
+    from transformers import pipeline
+    classifier = pipeline('sentiment-analysis')
+
+When typing this command for the first time, a pretrained model and its tokenizer are downloaded and cached. We will
+look at both later on, but as an introduction the tokenizer's job is to preprocess the text for the model, which is
+then responsible for making predictions. The pipeline groups all of that together, and post-process the predictions to
+make them readable. For instance
+
+::
+
+    classifier('We are very happy to show you the 🤗 Transformers library.')
+
+will return something like this:
+
+::
+
+    [{'label': 'POSITIVE', 'score': 0.9997795224189758}]
+
+That's encouraging! You can use it on a list of sentences, which will be preprocessed then fed to the model as a
+`batch`:
+
+::
+
+    classifier(["We are very happy to show you the 🤗 Transformers library.",
+                "We hope you don't hate it."])
+
+returning a list of dictionaries like this one:
+
+::
+
+    [{'label': 'POSITIVE', 'score': 0.9997795224189758},
+     {'label': 'NEGATIVE', 'score': 0.5308589935302734}]
+
+You can see the second sentence has been classified as negative (it needs to be positive or negative) but its score is
+fairly neutral.
+
+By default, the model downloaded for this pipeline is called "distilbert-base-uncased-finetuned-sst-2-english". We can
+look at its `model page <https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english>`__ to get more
+information about it. It uses the :doc:`DistilBERT architecture </model_doc/distilbert>` and has been fine-tuned on a
+dataset called SST-2 for the sentiment analysis task.
+
+Let's say we want to use another model; for instance, one that has been trained on French data. We can search through
+the `model hub <https://huggingface.co/models>`__ that gathers models pretrained on a lot of data by research labs, but
+also community models (usually fine-tuned versions of those big models on a specific dataset). Applying the tags
+"French" and "text-classification" gives back a suggestion "nlptown/bert-base-multilingual-uncased-sentiment". Let's
+see how we can use it. 
+
+You can directly pass the name of the model to use to :func:`~transformers.pipeline`:
+
+::
+
+    classifier = pipeline('sentiment-analysis', model="nlptown/bert-base-multilingual-uncased-sentiment")
+
+This classifier can now deal with texts in English, French, but also Dutch, German, Italian and Spanish! You can also
+replace that name by a local folder where you have saved a pretrained model (see below). You can also pass a model
+object and its associated tokenizer.
+
+We will need two classes for this. The first is :class:`~transformers.AutoTokenizer`, which we will use to download the
+tokenizer associated to the model we picked and instantiate it. The second is
+:class:`~transformers.AutoModelForSequenceClassification` (or
+:class:`~transformers.TFAutoModelForSequenceClassification` if you are using TensorFlow), which we will use to download
+the model itself. Note that if we were using the library on an other task, the class of the model would change. The
+:doc:`task summary </task_summary>` tutorial summarizes which class is used for which task.
+
+::
+
+    ## PYTORCH CODE
+    from transformers import AutoTokenizer, AutoModelForSequenceClassification
+    ## TENSORFLOW CODE
+    from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
+
+Now, to download the models and tokenizer we found previously, we just have to use the 
+:func:`~transformers.AutoModelForSequenceClassification.from_pretrained` method (feel free to replace ``model_name`` by
+any other model from the model hub):
+
+::
+
+    ## PYTORCH CODE
+    model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
+    model = AutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    pipe = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
+    ## TENSORFLOW CODE
+    model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
+    model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    classifier = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
+
+If you don't find a model that has been pretrained on some data similar to yours, you will need to fine-tune a
+pretrained model on your data. We provide :doc:`example scripts </examples>` to do so. Once you're done, don't forget
+to share your fine-tuned model on the hub with the community, using :doc:`this tutorial </model_sharing>`.
+
+.. _pretrained-model:
+
+Under the hood: pretrained models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Let's now see what happens beneath the hood when using those pipelines. As we saw, the model and tokenizer are created
+using the :obj:`from_pretrained` method:
+
+::
+
+    ## PYTORCH CODE
+    from transformers import AutoTokenizer, AutoModelForSequenceClassification
+    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    model = AutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    ## TENSORFLOW CODE
+    from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
+    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+Using the tokenizer
+^^^^^^^^^^^^^^^^^^^
+
+We mentioned the tokenizer is responsible for the preprocessing of your texts. First, it will split a given text in
+words (or part of words, punctuation symbols, etc.) usually called `tokens`. There are multiple rules that can govern
+that process, which is why we need to instantiate the tokenizer using the name of the model, to make sure we use the
+same rules as when the model was pretrained.
+
+The second step is to convert those `tokens` into numbers, to be able to build a tensor out of them and feed them to
+the model. To do this, the tokenizer has a `vocab`, which is the part we download when we instantiate it with the
+:obj:`from_pretrained` method, since we need to use the same `vocab` as when the model was pretrained.
+
+To apply these steps on a given text, we can just feed it to our tokenizer:
+
+::
+
+    input = tokenizer("We are very happy to show you the 🤗 Transformers library.")
+    print(input)
+
+This returns a dictionary string to list of ints. It contains the `ids of the tokens <glossary.html#input-ids>`__,
+as mentioned before, but also additional arguments that will be useful to the model. Here for instance, we also have an
+`attention mask <glossary.html#attention-mask>`__ that the model will use to have a better understanding of the sequence:
+
+
+::
+    {'input_ids': [101, 2057, 2024, 2200, 3407, 2000, 2265, 2017, 1996, 100, 19081, 3075, 1012, 102],
+     'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
+
+You can pass a list of sentences directly to your tokenizer. If your goal is to send them through your model as a
+batch, you probably want to pad them all to the same length, truncate them to the maximum length the model can accept
+and get tensors back. You can specify all of that to the tokenizer:
+
+::
+
+    ## PYTORCH CODE
+    batch = tokenizer(
+        ["We are very happy to show you the 🤗 Transformers library.",
+         "We hope you don't hate it."],
+        padding=True, truncation=True, return_tensors="pt")
+    print(batch)
+    ## TENSORFLOW CODE
+    batch = tokenizer(
+        ["We are very happy to show you the 🤗 Transformers library.",
+         "We hope you don't hate it."],
+        padding=True, truncation=True, return_tensors="tf")
+    print(batch)
+
+The padding is automatically applied on the side the model expect it (in this case, on the right), with the
+padding token the model was pretrained with. The attention mask is also adapted to take the padding into account:
+
+::
+
+    {'input_ids': tensor([[  101,  2057,  2024,  2200,  3407,  2000,  2265,  2017,  1996,   100, 19081,  3075,  1012,   102],
+                          [  101,  2057,  3246,  2017,  2123,  1005,  1056,  5223,  2009,  1012,   102,     0,     0,     0]]), 
+     'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+                               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]])}
+
+You can learn more about tokenizers on their :doc:`doc page <main_classes/tokenizer>` (tutorial coming soon).
+
+Using the model
+^^^^^^^^^^^^^^^
+
+Once your input has been preprocessed by the tokenizer, you can directly send it to the model. As we mentioned, it will
+contain all the relevant information the model needs. If you're using a TensorFlow model, you can directly pass the
+dictionary keys to tensor, for a PyTorch model, you need to unpack the dictionary by adding :obj:`**`.
+
+::
+
+    ## PYTORCH CODE
+    outputs = model(**batch)
+    ## TENSORFLOW CODE
+    outputs = model(batch)
+
+In 🤗 Transformers, all outputs are tuples (with only one element potentially). Here, we get a tuple with just the
+final activations of the model.
+
+::
+
+    (tensor([[-4.1329,  4.3811],
+             [ 0.0818, -0.0418]]),)
+
+.. note::
+
+    All 🤗 Transformers models (PyTorch or TensorFlow) return the activations of the model *before* the final
+    activation function (like SoftMax) since this final activation function is often fused with the loss.
+
+Let's apply the SoftMax activation to get predictions.
+
+::
+
+    ## PYTORCH CODE
+    import torch.nn.functional as F
+    predictions = F.softmax(outputs[0], dim=-1)
+    print(predictions)
+    ## TENSORFLOW CODE
+    predictions = tf.nn.softmax(outputs[0], axis=-1)
+    print(predictions)
+
+We can see we get the numbers from before:
+
+::
+
+    tensor([[2.0060e-04, 9.9980e-01],
+            [5.3086e-01, 4.6914e-01]])
+
+If you have labels, you can provide them to the model, it will return a tuple with the loss and the final activations.
+
+::
+
+    ## PYTORCH CODE
+    import torch
+    outputs = model(**batch, labels = torch.tensor([1, 0])
+    ## TENSORFLOW CODE
+    import tensorflow as tf
+    outputs = model(batch, labels = tf.constant([1, 0])
+
+Models are standard `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__ or
+`tf.keras.Model <https://www.tensorflow.org/api_docs/python/tf/keras/Model>`__ so you can use them in your usual
+training loop. 🤗 Transformers also provides a :class:`~transformers.Trainer` (or :class:`~transformers.TFTrainer` if
+you are using TensorFlow) class to help with your training (taking care of things such as distributed training, mixed
+precision, etc.). See the training tutorial (coming soon) for more details.
+
+Once your model is fine-tuned, you can save it with its tokenizer the following way:
+
+::
+
+    tokenizer.save_pretrained(save_directory)
+    model.save_pretrained(save_directory)
+
+You can then load this model back using the :func:`~transformers.AutoModel.from_pretrained` method by passing the
+directory name instead of the model name. One cool feature of 🤗 Transformers is that you can easily switch between
+PyTorch and TensorFlow: any model saved as before can be loaded back either in PyTorch or TensorFlow. If you are
+loading a saved PyTorch model in a TensorFlow model, use :func:`~transformers.TFAutoModel.from_pretrained` like this:
+
+::
+
+    tokenizer = AutoTokenizer.from_pretrained(save_directory)
+    model = TFAutoModel.from_pretrained(save_directory, from_pt=True)
+
+and if you are loading a saved TensorFlow model in a PyTorch model, you should use the following code:
+
+::
+
+    tokenizer = AutoTokenizer.from_pretrained(save_directory)
+    model = AutoModel.from_pretrained(save_directory, from_tf=True)
+
+Lastly, you can also ask the model to return all hidden states and all attention weights if you need them:
+
+
+::
+
+    ## PYTORCH CODE
+    outputs = model(**batch, output_hidden_states=True, output_attentions=True)
+    all_hidden_states, all_attentions = outputs[-2:]
+    ## TENSORFLOW CODE
+    outputs = model(batch, output_hidden_states=True, output_attentions=True)
+    all_hidden_states, all_attentions = outputs[-2:]
+
+Accessing the code
+^^^^^^^^^^^^^^^^^^
+
+The :obj:`AutoModel` and :obj:`AutoTokenizer` classes are just shortcuts that will automatically work with any
+pretrained model. Behind the scenes, the library has one model class per combination of architecture plus class, so the
+code is easy to access and tweak if you need to.
+
+In our previous example, the model was called "distilbert-base-uncased-finetuned-sst-2-english", which means it's
+using the :doc:`DistilBERT </model_doc/distilbert>` architecture. The model automatically created is then a
+:class:`~transformers.DistilBertForSequenceClassification`. You can look at its documentation for all details relevant
+to that specific model, or browse the source code. This is how you would directly instantiate model and tokenizer
+without the auto magic:
+
+::
+
+    ## PYTORCH CODE
+    from transformers import DistilBertTokenizer, DistilBertForSequenceClassification
+    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    model = DistilBertForSequenceClassification.from_pretrained(model_name)
+    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+    ## TENSORFLOW CODE
+    from transformers import DistilBertTokenizer, TFDistilBertForSequenceClassification
+    model_name = "distilbert-base-uncased-finetuned-sst-2-english"
+    model = TFDistilBertForSequenceClassification.from_pretrained(model_name)
+    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+
+Customizing the model
+^^^^^^^^^^^^^^^^^^^^^
+
+If you want to change how the model itself is built, you can define your custom configuration class. Each architecture
+comes with its own relevant configuration (in the case of DistilBERT, :class:`~transformers.DistilBertConfig`) which
+allows you to specify any of the hidden dimension, dropout rate etc. If you do core modifications, like changing the
+hidden size, you won't be able to use a pretrained model anymore and will need to train from scratch. You would then
+instantiate the model directly from this configuration.
+
+Here we use the predefined vocabulary of DistilBERT (hence load the tokenizer with the
+:func:`~transformers.DistilBertTokenizer.from_pretrained` method) and initialize the model from scratch (hence
+instantiate the model from the configuration instead of using the
+:func:`~transformers.DistilBertForSequenceClassification.from_pretrained` method).
+
+::
+
+    ## PYTORCH CODE
+    from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
+    config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
+    tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+    model = DistilBertForSequenceClassification(config)
+    ## TENSORFLOW CODE
+    from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
+    config = DistilBertConfig(n_heads=8, dim=512, hidden_dim=4*512)
+    tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+    model = TFDistilBertForSequenceClassification(config)
+
+For something that only changes the head of the model (for instance, the number of labels), you can still use a
+pretrained model for the body. For instance, let's define a classifier for 10 different labels using a pretrained body.
+We could create a configuration with all the default values and just change the number of labels, but more easily, you
+can directly pass any argument a configuration would take to the :func:`from_pretrained` method and it will update the
+default configuration with it:
+
+::
+
+    ## PYTORCH CODE
+    from transformers import DistilBertConfig, DistilBertTokenizer, DistilBertForSequenceClassification
+    model_name = "distilbert-base-uncased"
+    model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
+    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
+    ## TENSORFLOW CODE
+    from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForSequenceClassification
+    model_name = "distilbert-base-uncased"
+    model = TFDistilBertForSequenceClassification.from_pretrained(model_name, num_labels=10)
+    tokenizer = DistilBertTokenizer.from_pretrained(model_name)
@@ -1,323 +0,0 @@
-Training and fine-tuning
-========================
-
-Model classes in 🤗 Transformers are designed to be compatible with native
-PyTorch and TensorFlow 2 and can be used seemlessly with either. In this
-quickstart, we will show how to fine-tune (or train from scratch) a model
-using the standard training tools available in either framework. We will also
-show how to use our included :func:`~transformers.Trainer` class which
-handles much of the complexity of training for you.
-
-This guide assume that you are already familiar with loading and use our
-models for inference; otherwise, see the :doc:`task summary <task_summary>`. We also assume
-that you are familiar with training deep neural networks in either PyTorch or
-TF2, and focus specifically on the nuances and tools for training models in
-🤗 Transformers.
-
-Sections:
-
-  * :ref:`pytorch`
-  * :ref:`tensorflow`
-  * :ref:`trainer`
-  * :ref:`additional-resources`
-
-.. _pytorch:
-
-Fine-tuning in native PyTorch
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Model classes in 🤗 Transformers that don't begin with ``TF`` are
-`PyTorch Modules <https://pytorch.org/docs/master/generated/torch.nn.Module.html>`_,
-meaning that you can use them just as you would any model in PyTorch for
-both inference and optimization.
-
-Let's consider the common task of fine-tuning a masked language model like
-BERT on a sequence classification dataset. When we instantiate a model with
-:func:`~transformers.PreTrainedModel.from_pretrained`, the model
-configuration and pre-trained weights
-of the specified model are used to initialize the model. The
-library also includes a number of task-specific final layers or 'heads' whose
-weights are instantiated randomly when not present in the specified
-pre-trained model. For example, instantiating a model with
-``BertForSequenceClassification.from_pretrained('bert-base-uncased', num_classes=2)``
-will create a BERT model instance with encoder weights copied from the
-``bert-base-uncased`` model and a randomly initialized sequence
-classification head on top of the encoder with an output size of 2. Models
-are initialized in ``eval`` mode by default. We can call ``model.train()`` to
-put it in train mode.
-
-.. code-block:: python
-
-    from transformers import BertForSequenceClassification
-    model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
-    model.train()
-
-This is useful because it allows us to make use of the pre-trained BERT
-encoder and easily train it on whatever sequence classification dataset we
-choose. We can use any PyTorch optimizer, but our library also provides the
-:func:`~transformers.AdamW` optimizer which implements gradient bias
-correction as well as weight decay.
-
-.. code-block:: python
-
-    from transformers import AdamW
-    optimizer = AdamW(model.parameters(), lr=1e-5)
-
-The optimizer allows us to apply different hyperpameters for specific
-parameter groups. For example, we can apply weight decay to all parameters
-other than bias and layer normalization terms:
-
-.. code-block:: python
-
-    no_decay = ['bias', 'LayerNorm.weight']
-    optimizer_grouped_parameters = [
-        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
-        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
-    ]
-    optimizer = AdamW(optimizer_grouped_parameters, lr=1e-5)
-    
-Now we can set up a simple dummy training batch using
-:func:`~transformers.PreTrainedTokenizer.batch_encode_plus`. This returns a
-:func:`~transformers.BatchEncoding` instance which
-prepares everything we might need to pass to the model.
-
-.. code-block:: python
-
-    from transformers import BertTokenizer
-    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-    text_batch = ["I love Pixar.", "I don't care for Pixar."]
-    encoding = tokenizer(text_batch, return_tensors='pt', padding=True, truncation=True)
-    input_ids = encoding['input_ids']
-    attention_mask = encoding['attention_mask']
-
-When we call a classification model with the ``labels`` argument, the first
-returned element is the Cross Entropy loss between the predictions and the
-passed labels. Having already set up our optimizer, we can then do a
-backwards pass and update the weights:
-
-.. code-block:: python
-
-    labels = torch.tensor([1,0]).unsqueeze(0)
-    outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
-    loss = outputs[0]
-    loss.backward()
-    optimizer.step()
-
-Alternatively, you can just get the logits and calculate the loss yourself.
-The following is equivalent to the previous example:
-
-.. code-block:: python
-
-    from torch.nn import functional as F
-    labels = torch.tensor([1,0]).unsqueeze(0)
-    outputs = model(input_ids, attention_mask=attention_mask)
-    loss = F.cross_entropy(labels, outputs[0])
-    loss.backward()
-    optimizer.step()
-
-Of course, you can train on GPU by calling ``to('cuda')`` on the model and
-inputs as usual.
-
-We also provide a few learning rate scheduling tools. With the following, we
-can set up a scheduler which warms up for ``num_warmup_steps`` and then
-linearly decays to 0 by the end of training.
-
-.. code-block:: python
-
-    from transformers import get_linear_schedule_with_warmup
-    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_train_steps)
-
-Then all we have to do is call ``scheduler.step()`` after ``optimizer.step()``.
-
-.. code-block:: python
-
-    ...
-    loss.backward()
-    optimizer.step()
-    scheduler.step()
-
-We highly recommend using :func:`~transformers.Trainer`, discussed below,
-which conveniently handles the moving parts of training 🤗 Transformers models
-with features like mixed precision and easy tensorboard logging.
-
-
-Freezing the encoder
--------------------
-
-In some cases, you might be interested in keeping the weights of the
-pre-trained encoder frozen and optimizing only the weights of the head
-layers. To do so, simply set the ``requires_grad`` attribute to ``False`` on
-the encoder parameters, which can be accessed with the ``base_model``
-submodule on any task-specific model in the library:
-
-.. code-block:: python
-   
-    for param in model.base_model.parameters():
-        param.requires_grad = False
-
-
-.. _tensorflow:
-
-Fine-tuning in native TensorFlow 2
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Models can also be trained natively in TensorFlow 2. Just as with PyTorch,
-TensorFlow models can be instantiated with
-:func:`~transformers.PreTrainedModel.from_pretrained` to load the weights of
-the encoder from a pretrained model.
-
-.. code-block:: python
-
-    from transformers import TFBertForSequenceClassification
-    model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased')
-
-Let's use ``tensorflow_datasets`` to load in the `MRPC dataset
-<https://www.tensorflow.org/datasets/catalog/glue#gluemrpc>`_ from GLUE. We
-can then use our built-in
-:func:`~transformers.data.processors.glue.glue_convert_examples_to_features`
-to tokenize MRPC and convert it to a TensorFlow ``Dataset`` object. Note that
-tokenizers are framework-agnostic, so there is no need to prepend ``TF`` to
-the pretrained tokenizer name.
-
-.. code-block:: python
-
-    from transformers import BertTokenizer, glue_convert_examples_to_features
-    import tensorflow_datasets as tfds
-    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-    data = tfds.load('glue/mrpc')
-    train_dataset = glue_convert_examples_to_features(data['train'], tokenizer, max_length=128, task='mrpc')
-    train_dataset = train_dataset.shuffle(100).batch(32).repeat(2)
-
-The model can then be compiled and trained as any Keras model:
-
-.. code-block:: python
-    
-    optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
-    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
-    model.compile(optimizer=optimizer, loss=loss)
-    model.fit(train_dataset, epochs=2, steps_per_epoch=115)
-
-With the tight interoperability between TensorFlow and PyTorch models, you
-can even save the model and then reload it as a PyTorch model (or vice-versa):
-
-.. code-block:: python
-
-    from transformers import BertForSequenceClassification
-    model.save_pretrained('./my_mrpc_model/')
-    pytorch_model = BertForSequenceClassification.from_pretrained('./my_mrpc_model/', from_tf=True)
-
-
-.. _trainer:
-
-Trainer
-^^^^^^^
-
-We also provide a simple but feature-complete training and evaluation
-interface through :func:`~transformers.Trainer` and
-:func:`~transformers.TFTrainer`. You can train, fine-tune,
-and evaluate any 🤗 Transformers model with a wide range of training options and
-with built-in features like logging, gradient accumulation, and mixed
-precision.
-
-.. code-block:: python
-
-    ## PYTORCH CODE
-    from transformers import BertForSequenceClassification, Trainer, TrainingArguments
-
-    model = BertForSequenceClassification.from_pretrained("bert-large-uncased")
-
-    training_args = TrainingArguments(
-        output_dir='./results',          # output directory
-        num_train_epochs=3,              # total # of training epochs
-        per_device_train_batch_size=16,  # batch size per device during training
-        per_device_eval_batch_size=64,   # batch size for evaluation
-        warmup_steps=500,                # number of warmup steps for learning rate scheduler
-        weight_decay=0.01,               # strength of weight decay
-        logging_dir='./logs',            # directory for storing logs
-    )
-
-    trainer = Trainer(
-        model=model,                         # the instantiated 🤗 Transformers model to be trained
-        args=training_args,                  # training arguments, defined above
-        train_dataset=train_dataset,         # training dataset
-        eval_dataset=test_dataset            # evaluation dataset
-    )
-    ## TENSORFLOW CODE
-    from transformers import TFBertForSequenceClassification, TFTrainer, TFTrainingArguments
-
-    model = TFBertForSequenceClassification.from_pretrained("bert-large-uncased")
-
-    training_args = TFTrainingArguments(
-        output_dir='./results',          # output directory
-        num_train_epochs=3,              # total # of training epochs
-        per_device_train_batch_size=16,  # batch size per device during training
-        per_device_eval_batch_size=64,   # batch size for evaluation
-        warmup_steps=500,                # number of warmup steps for learning rate scheduler
-        weight_decay=0.01,               # strength of weight decay
-        logging_dir='./logs',            # directory for storing logs
-    )
-
-    trainer = TFTrainer(
-        model=model,                         # the instantiated 🤗 Transformers model to be trained
-        args=training_args,                  # training arguments, defined above
-        train_dataset=tfds_train_dataset,    # tensorflow_datasets training dataset
-        eval_dataset=tfds_test_dataset       # tensorflow_datasets evaluation dataset
-    )
-
-Now simply call ``trainer.train()`` to train and ``trainer.evaluate()`` to
-evaluate. You can use your own module as well, but the first
-argument returned from ``forward`` must be the loss which you wish to
-optimize.
-
-:func:`~transformers.Trainer` uses a built-in default function to collate
-batches and prepare them to be fed into the model. If needed, you can also
-use the ``data_collator`` argument to pass your own collator function which
-takes in the data in the format provides by your dataset and returns a
-batch ready to be fed into the model. Note that
-:func:`~transformers.TFTrainer` expects the passed datasets to be dataset
-objects from ``tensorflow_datasets``.
-
-To calculate additional metrics in addition to the loss, you can also define
-your own ``compute_metrics`` function and pass it to the trainer.
-
-.. code-block:: python
-
-    from sklearn.metrics import precision_recall_fscore_support
-
-    def compute_metrics(pred):
-        labels = pred.label_ids
-        preds = pred.predictions.argmax(-1)
-        precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')
-        acc = accuracy_score(labels, preds)
-        return {
-            'accuracy': acc,
-            'f1': f1,
-            'precision': precision,
-            'recall': recall
-        }
-
-Finally, you can view the results, including any calculated metrics, by
-launching tensorboard in your specified ``logging_dir`` directory.
-
-
-.. _additional-resources:
-
-Additional resources
-^^^^^^^^^^^^^^^^^^^^
-
-    * `A lightweight colab demo
-      <https://colab.research.google.com/drive/1-JIJlao4dI-Ilww_NnTc0rxtp-ymgDgM?usp=sharing>`_
-      which uses ``Trainer`` for IMDb sentiment classification.
-
-    * `🤗 Transformers Examples <https://github.com/huggingface/transformers/tree/master/examples>`_
-      including scripts for training and fine-tuning on GLUE, SQuAD, and
-      several other tasks.
-
-    * `How to train a language model
-      <https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb>`_,
-      a detailed colab notebook which uses ``Trainer`` to train a masked
-      language model from scratch on Esperanto.
-
-    * `🤗 Transformers Notebooks <./notebooks.html>`_ which contain dozens
-      of example notebooks from the community for training and using
-      🤗 Transformers on a variety of tasks.
@@ -1,7 +1,7 @@
 ## Examples

 Version 2.9 of 🤗 Transformers introduces a new [`Trainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py) class for PyTorch, and its equivalent [`TFTrainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer_tf.py) for TF 2.
-Running the examples requires PyTorch 1.3.1+ or TensorFlow 2.1+.
+Running the examples requires PyTorch 1.3.1+ or TensorFlow 2.0+.

 Here is the list of all our examples:
 - **grouped by task** (all official examples work for multiple models)
@@ -33,7 +33,7 @@ from transformers import (
    default_data_collator,
    set_seed,
 )
-from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels
+from utils_hans import HansDataset, InputFeatures, hans_processors


 logger = logging.getLogger(__name__)
@@ -130,7 +130,9 @@ def main():
    set_seed(training_args.seed)

    try:
-        num_labels = hans_tasks_num_labels[data_args.task_name]
+        processor = hans_processors[data_args.task_name]()
+        label_list = processor.get_labels()
+        num_labels = len(label_list)
    except KeyError:
        raise ValueError("Task not found: %s" % (data_args.task_name))

@@ -212,7 +214,6 @@ def main():

        pair_ids = [ex.pairID for ex in eval_dataset]
        output_eval_file = os.path.join(training_args.output_dir, "hans_predictions.txt")
-        label_list = eval_dataset.get_labels()
        if trainer.is_world_master():
            with open(output_eval_file, "w") as writer:
                writer.write("pairID,gold_label\n")
@@ -22,17 +22,7 @@ from typing import List, Optional, Union
 import tqdm
 from filelock import FileLock

-from transformers import (
-    BartTokenizer,
-    BartTokenizerFast,
-    DataProcessor,
-    PreTrainedTokenizer,
-    RobertaTokenizer,
-    RobertaTokenizerFast,
-    XLMRobertaTokenizer,
-    is_tf_available,
-    is_torch_available,
-)
+from transformers import DataProcessor, PreTrainedTokenizer, is_tf_available, is_torch_available


 logger = logging.getLogger(__name__)
@@ -115,17 +105,6 @@ if is_torch_available():
                    "dev" if evaluate else "train", tokenizer.__class__.__name__, str(max_seq_length), task,
                ),
            )
-            label_list = processor.get_labels()
-            if tokenizer.__class__ in (
-                RobertaTokenizer,
-                RobertaTokenizerFast,
-                XLMRobertaTokenizer,
-                BartTokenizer,
-                BartTokenizerFast,
-            ):
-                # HACK(label indices are swapped in RoBERTa pretrained model)
-                label_list[1], label_list[2] = label_list[2], label_list[1]
-            self.label_list = label_list

            # Make sure only the first process in distributed training processes the dataset,
            # and the others will use the cache.
@@ -137,6 +116,7 @@ if is_torch_available():
                    self.features = torch.load(cached_features_file)
                else:
                    logger.info(f"Creating features from dataset file at {data_dir}")
+                    label_list = processor.get_labels()

                    examples = (
                        processor.get_dev_examples(data_dir) if evaluate else processor.get_train_examples(data_dir)
@@ -153,9 +133,6 @@ if is_torch_available():
        def __getitem__(self, i) -> InputFeatures:
            return self.features[i]

-        def get_labels(self):
-            return self.label_list
-

 if is_tf_available():
    import tensorflow as tf
@@ -179,16 +156,6 @@ if is_tf_available():
        ):
            processor = hans_processors[task]()
            label_list = processor.get_labels()
-            if tokenizer.__class__ in (
-                RobertaTokenizer,
-                RobertaTokenizerFast,
-                XLMRobertaTokenizer,
-                BartTokenizer,
-                BartTokenizerFast,
-            ):
-                # HACK(label indices are swapped in RoBERTa pretrained model)
-                label_list[1], label_list[2] = label_list[2], label_list[1]
-            self.label_list = label_list

            examples = processor.get_dev_examples(data_dir) if evaluate else processor.get_train_examples(data_dir)
            self.features = hans_convert_examples_to_features(examples, label_list, max_seq_length, tokenizer)
@@ -239,9 +206,6 @@ if is_tf_available():
        def __getitem__(self, i) -> InputFeatures:
            return self.features[i]

-        def get_labels(self):
-            return self.label_list
-

 class HansProcessor(DataProcessor):
    """Processor for the HANS data set."""
@@ -1,19 +1,14 @@
 import csv
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import List, Optional
+from typing import Optional

 import matplotlib.pyplot as plt
 import numpy as np
-from matplotlib.ticker import ScalarFormatter

 from transformers import HfArgumentParser


-def list_field(default=None, metadata=None):
-    return field(default_factory=lambda: default, metadata=metadata)
-
-
@dataclass
 class PlotArguments:
    """
@@ -29,9 +24,6 @@ class PlotArguments:
        default=False,
        metadata={"help": "Whether the csv file has time results or memory results. Defaults to memory results."},
    )
-    no_log_scale: bool = field(
-        default=False, metadata={"help": "Disable logarithmic scale when plotting"},
-    )
    is_train: bool = field(
        default=False,
        metadata={
@@ -41,25 +33,6 @@ class PlotArguments:
    figure_png_file: Optional[str] = field(
        default=None, metadata={"help": "Filename under which the plot will be saved. If unused no plot is saved."},
    )
-    short_model_names: Optional[List[str]] = list_field(
-        default=None, metadata={"help": "List of model names that are used instead of the ones in the csv file."}
-    )
-
-
-def can_convert_to_int(string):
-    try:
-        int(string)
-        return True
-    except ValueError:
-        return False
-
-
-def can_convert_to_float(string):
-    try:
-        float(string)
-        return True
-    except ValueError:
-        return False


 class Plot:
@@ -73,31 +46,16 @@ class Plot:
                model_name = row["model"]
                self.result_dict[model_name]["bsz"].append(int(row["batch_size"]))
                self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
-                if can_convert_to_int(row["result"]):
-                    # value is not None
-                    self.result_dict[model_name]["result"][
-                        (int(row["batch_size"]), int(row["sequence_length"]))
-                    ] = int(row["result"])
-                elif can_convert_to_float(row["result"]):
-                    # value is not None
-                    self.result_dict[model_name]["result"][
-                        (int(row["batch_size"]), int(row["sequence_length"]))
-                    ] = float(row["result"])
+                self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = row[
+                    "result"
+                ]

    def plot(self):
        fig, ax = plt.subplots()
        title_str = "Time usage" if self.args.is_time else "Memory usage"
        title_str = title_str + " for training" if self.args.is_train else title_str + " for inference"

-        if not self.args.no_log_scale:
-            # set logarithm scales
-            ax.set_xscale("log")
-            ax.set_yscale("log")
-
-        for axis in [ax.xaxis, ax.yaxis]:
-            axis.set_major_formatter(ScalarFormatter())
-
-        for model_name_idx, model_name in enumerate(self.result_dict.keys()):
+        for model_name in self.result_dict.keys():
            batch_sizes = sorted(list(set(self.result_dict[model_name]["bsz"])))
            sequence_lengths = sorted(list(set(self.result_dict[model_name]["seq_len"])))
            results = self.result_dict[model_name]["result"]
@@ -106,33 +64,28 @@ class Plot:
                (batch_sizes, sequence_lengths) if self.args.plot_along_batch else (sequence_lengths, batch_sizes)
            )

-            label_model_name = (
-                model_name if self.args.short_model_names is None else self.args.short_model_names[model_name_idx]
-            )
+            plt.xlim(min(x_axis_array), max(x_axis_array))

            for inner_loop_value in inner_loop_array:
                if self.args.plot_along_batch:
-                    y_axis_array = np.asarray(
-                        [results[(x, inner_loop_value)] for x in x_axis_array if (x, inner_loop_value) in results],
-                        dtype=np.int,
-                    )
+                    y_axis_array = np.asarray([results[(x, inner_loop_value)] for x in x_axis_array], dtype=np.int)
                else:
-                    y_axis_array = np.asarray(
-                        [results[(inner_loop_value, x)] for x in x_axis_array if (inner_loop_value, x) in results],
-                        dtype=np.float32,
-                    )
+                    y_axis_array = np.asarray([results[(inner_loop_value, x)] for x in x_axis_array], dtype=np.float32)
+
+                ax.set_xscale("log", basex=2)
+                ax.set_yscale("log", basey=10)

                (x_axis_label, inner_loop_label) = (
-                    ("batch_size", "len") if self.args.plot_along_batch else ("in #tokens", "bsz")
+                    ("batch_size", "sequence_length in #tokens")
+                    if self.args.plot_along_batch
+                    else ("sequence_length in #tokens", "batch_size")
                )

-                x_axis_array = np.asarray(x_axis_array, np.int)[: len(y_axis_array)]
-                plt.scatter(
-                    x_axis_array, y_axis_array, label=f"{label_model_name} - {inner_loop_label}: {inner_loop_value}"
-                )
+                x_axis_array = np.asarray(x_axis_array, np.int)
+                plt.scatter(x_axis_array, y_axis_array, label=f"{model_name} - {inner_loop_label}: {inner_loop_value}")
                plt.plot(x_axis_array, y_axis_array, "--")

-            title_str += f" {label_model_name} vs."
+            title_str += f" {model_name} vs."

        title_str = title_str[:-4]
        y_axis_label = "Time in s" if self.args.is_time else "Memory in MB"
@@ -1,4 +0,0 @@
-model,batch_size,sequence_length,result
-aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2,8,512,0.2032
-aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2,64,512,1.5279
-aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2,256,512,6.1837
@@ -1,176 +0,0 @@
-This directory contains examples for finetuning and evaluating transformers on summarization and translation tasks.
-Summarization support is more mature than translation support.
-Please tag @sshleifer with any issues/unexpected behaviors, or send a PR!
-For `bertabs` instructions, see `bertabs/README.md`. 
-
-### Data
-
-CNN/DailyMail data
-```bash
-cd examples/seq2seq
-wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/cnn_dm.tgz
-tar -xzvf cnn_dm.tgz
-
-export CNN_DIR=${PWD}/cnn_dm
-```
-
-this should make a directory called cnn_dm/ with files like `test.source`.
-To use your own data, copy that files format. Each article to be summarized is on its own line.
-
-XSUM Data:
-```bash
-cd examples/seq2seq
-wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/xsum.tar.gz
-tar -xzvf xsum.tar.gz
-export XSUM_DIR=${PWD}/xsum
-```
-
-
-WMT16 English-Romanian Translation Data:
-```bash
-cd examples/seq2seq
-wget https://s3.amazonaws.com/datasets.huggingface.co/translation/wmt_en_ro.tar.gz
-tar -xzvf wmt_en_ro.tar.gz
-export ENRO_DIR=${PWD}/wmt_en_ro
-```
-
-If you are using your own data, it must be formatted as one directory with 6 files: train.source, train.target, val.source, val.target, test.source, test.target.  
-The `.source` files are the input, the `.target` files are the desired output.
-
-### Evaluation
-
-To create summaries for each article in dataset, run:
-```bash
-python run_eval.py <path_to_test.source> test_generations.txt <model-name>  --score_path rouge_scores.txt
-```
-The default batch size, 4, fits in 16GB GPU memory, but may need to be adjusted to fit your system.
-
-
-### Summarization Finetuning
-Run/modify `finetune.sh`
-
-The following command should work on a 16GB GPU:
-```bash
-./finetune.sh \
-    --data_dir $XSUM_DIR \
-    --train_batch_size=1 \
-    --eval_batch_size=1 \
-    --output_dir=xsum_results \
-    --num_train_epochs 1 \
-    --model_name_or_path facebook/bart-large
-```
-
-*Note*: The following tips mostly apply to summarization finetuning.
-
-Tips:
- 1 epoch at batch size 1 for bart-large takes 24 hours and requires 13GB GPU RAM with fp16 on an NVIDIA-V100. 
- try `bart-base`, `--freeze_encoder` or `--freeze_embeds` for faster training/larger batch size.  (3hr/epoch with bs=8, see the "xsum_shared_task" command below)
- `fp16_opt_level=O1` (the default works best).
- If you are finetuning on your own dataset, start from `distilbart-cnn-12-6` if you want long summaries and `distilbart-xsum-12-6` if you want short summaries.
-(It rarely makes sense to start from `bart-large` unless you are a researching finetuning methods).
- In addition to the pytorch-lightning .ckpt checkpoint, a transformers checkpoint will be saved.
-Load it with `BartForConditionalGeneration.from_pretrained(f'{output_dir}/best_tfmr)`.
- At the moment, `--do_predict` does not work in a multi-gpu setting. You need to use `evaluate_checkpoint` or the `run_eval.py` code.
- If you want to run experiments on improving the summarization finetuning process, try the XSUM Shared Task (below). It's faster to train than CNNDM because the summaries are shorter.
- For CNN/DailyMail, the default `val_max_target_length` and `test_max_target_length` will truncate the ground truth labels, resulting in slightly higher rouge scores. To get accurate rouge scores, you should rerun calculate_rouge on the `{output_dir}/test_generations.txt` file saved by `trainer.test()`
- `--max_target_length=60 --val_max_target_length=60 --test_max_target_length=100 ` is a reasonable setting for XSUM.
- `wandb` can be used by specifying `--logger wandb_shared` or `--logger wandb`. It is useful for reproducibility. 
- This warning can be safely ignored: 
-    > "Some weights of BartForConditionalGeneration were not initialized from the model checkpoint at facebook/bart-large-xsum and are newly initialized: ['final_logits_bias']"
- Both finetuning and eval are 30% faster with `--fp16`. For that you need to [install apex](https://github.com/NVIDIA/apex#quick-start).
-
-#### Finetuning Outputs 
-As you train, `output_dir` will be filled with files, that look kind of like this (comments are mine). 
-Some of them are metrics, some of them are checkpoints, some of them are metadata. Here is a quick tour:
-
-```bash
-output_dir
-├── best_tfmr  # this is a huggingface checkpoint generated by save_pretrained. It is the same model as the PL .ckpt file below
-│   ├── config.json
-│   ├── merges.txt
-│   ├── pytorch_model.bin
-│   ├── special_tokens_map.json
-│   ├── tokenizer_config.json
-│   └── vocab.json
-├── git_log.json   # repo, branch, and commit hash
-├── val_avg_rouge2=0.1984-step_count=11.ckpt  # this is a pytorch lightning checkpoint associated with the best val score.
-├── metrics.json  # new validation metrics will continually be appended to this
-├── student  # this is a huggingface checkpoint generated by SummarizationDistiller. It is the student before it gets finetuned.
-│   ├── config.json
-│   └── pytorch_model.bin
-├── test_generations.txt 
-# ^^ are the summaries or translations produced by your best checkpoint on the test data. Populated when training is done 
-├── test_results.txt  # a convenience file with the test set metrics. This data is also in metrics.json['test']
-├── hparams.pkl  # the command line args passed after some light preprocessing. Should be saved fairly quickly.
-```
-After training, you can recover the best checkpoint by running
-```python
-from transformers import AutoModelForSeq2SeqLM
-model = AutoModelForSeq2SeqLM.from_pretrained(f'{output_dir}/best_tfmr')
-```
-
-
-### XSUM Shared Task
-Compare XSUM results with others by using `--logger wandb_shared`. This requires `wandb` registration.
-
-Here is an example command, but you can do whatever you want. Hopefully this will make debugging and collaboration easier!
-```bash
-./finetune.sh \
-    --data_dir $XSUM_DIR \
-    --output_dir xsum_frozen_embs \
-    --model_name_or_path facebook/bart-large \
-    --logger wandb_shared \
-    --train_batch_size 16 --eval_batch_size 16 --freeze_embeds --freeze_encoder \
-    --num_train_epochs 6 \
-    --max_target_length=60 --val_max_target_length=60 --test_max_target_length=100
-```
-
-You can see your wandb logs [here](https://app.wandb.ai/sshleifer/hf_xsum?workspace=user-)
-
-### DistilBART
-
-For the CNN/DailyMail dataset, (relatively longer, more extractive summaries), we found a simple technique that works:
-you just copy alternating layers from `bart-large-cnn` and finetune more on the same data. 
-
-For the XSUM dataset, that didn’t work as well so we used that same initialization strategy followed by a combination of Distillbert’s ce_loss and the hidden states MSE loss used in the tinybert paper.
-
-You can see the performance tradeoffs of model sizes [here](https://docs.google.com/spreadsheets/d/1EkhDMwVO02m8jCD1cG3RoFPLicpcL1GQHTQjfvDYgIM/edit#gid=0).
-and more granular timing results [here](https://docs.google.com/spreadsheets/d/1EkhDMwVO02m8jCD1cG3RoFPLicpcL1GQHTQjfvDYgIM/edit#gid=1753259047&range=B2:I23).
-
-#### No Teacher Distillation
-To run the simpler distilbart-cnn style distillation all you need is data, a GPU, and a properly initialized student.
-You don't even need `distillation.py`.
-
-Some [un-finetuned students](https://huggingface.co/models?search=sshleifer%2Fstudent) are available for replication purposes.
-They are initialized by copying layers from the associated `bart-large-{cnn|xsum}` teacher using `--init_strategy alternate`. (You can read about that in `initialization_utils.py`)
-The command that produced `sshleifer/distilbart-cnn-12-6` is
-```bash
-./train_distilbart_cnn.sh
-```  
-runtime: 6H on NVIDIA RTX 24GB GPU
-
-*Note*: You can get the same simple distillation logic by using `./run_distiller.sh --no_teacher` followed by identical arguments as the ones in `train_distilbart_cnn.sh`.
-If you are using `wandb` and comparing the two distillation methods, using this entry point will make your logs consistent,
-because you will have the same hyperparameters logged in every run.
-
-#### With a teacher
-*Note* only BART variants are supported
-
-In this method, we use try to enforce that the student and teacher produce similar encoder_outputs, logits, and hidden_states using `BartSummarizationDistiller`.
-This is how `sshleifer/distilbart-xsum*` checkpoints were produced.
-
-The command that produced `sshleifer/distilbart-xsum-12-6` is:
-
-```bash
-./train_distilbart_xsum.sh  
-```
-
-runtime: 13H on V-100 16GB GPU. 
-
-### Contributing
- follow the standard contributing guidelines and code of conduct.
- add tests to `test_seq2seq_examples.py`
- To run only the seq2seq tests, you must be in the root of the repository and run:
-```bash
-pytest examples/seq2seq/  
-```
@@ -1,252 +0,0 @@
-import argparse
-import logging
-import os
-import sys
-import tempfile
-import unittest
-from pathlib import Path
-from unittest.mock import patch
-
-import pytest
-import torch
-from torch.utils.data import DataLoader
-
-from transformers import AutoTokenizer
-
-from .distillation import distill_main, evaluate_checkpoint
-from .finetune import main
-from .run_eval import generate_summaries_or_translations, run_generate
-from .utils import SummarizationDataset, lmap, load_json
-
-
-logging.basicConfig(level=logging.DEBUG)
-
-logger = logging.getLogger()
-CUDA_AVAILABLE = torch.cuda.is_available()
-CHEAP_ARGS = {
-    "logger": "default",
-    "length_penalty": 0.5,
-    "cache_dir": "",
-    "task": "summarization",
-    "num_workers": 2,
-    "alpha_hid": 0,
-    "freeze_embeds": True,
-    "enc_only": False,
-    "tgt_suffix": "",
-    "resume_from_checkpoint": None,
-    "sortish_sampler": True,
-    "student_decoder_layers": 1,
-    "val_check_interval": 1.0,
-    "output_dir": "",
-    "fp16": CUDA_AVAILABLE,
-    "no_teacher": False,
-    "fp16_opt_level": "O1",
-    "gpus": 1 if CUDA_AVAILABLE else 0,
-    "n_tpu_cores": 0,
-    "max_grad_norm": 1.0,
-    "do_train": True,
-    "do_predict": True,
-    "gradient_accumulation_steps": 1,
-    "server_ip": "",
-    "server_port": "",
-    "seed": 42,
-    "model_name_or_path": "sshleifer/bart-tiny-random",
-    "config_name": "",
-    "tokenizer_name": "facebook/bart-large",
-    "do_lower_case": False,
-    "learning_rate": 0.3,
-    "weight_decay": 0.0,
-    "adam_epsilon": 1e-08,
-    "warmup_steps": 0,
-    "num_train_epochs": 1,
-    "train_batch_size": 2,
-    "eval_batch_size": 2,
-    "max_source_length": 12,
-    "max_target_length": 12,
-    "val_max_target_length": 12,
-    "test_max_target_length": 12,
-    "fast_dev_run": False,
-    "no_cache": False,
-    "n_train": -1,
-    "n_val": -1,
-    "n_test": -1,
-    "student_encoder_layers": 1,
-    "alpha_loss_encoder": 0.0,
-    "freeze_encoder": False,
-    "auto_scale_batch_size": False,
-}
-
-
-def _dump_articles(path: Path, articles: list):
-    with path.open("w") as f:
-        f.write("\n".join(articles))
-
-
-ARTICLES = [" Sam ate lunch today", "Sams lunch ingredients"]
-SUMMARIES = ["A very interesting story about what I ate for lunch.", "Avocado, celery, turkey, coffee"]
-T5_TINY = "patrickvonplaten/t5-tiny-random"
-BART_TINY = "sshleifer/bart-tiny-random"
-MBART_TINY = "sshleifer/tiny-mbart"
-MARIAN_TINY = "sshleifer/tiny-marian-en-de"
-stream_handler = logging.StreamHandler(sys.stdout)
-logger.addHandler(stream_handler)
-logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
-
-
-def make_test_data_dir(**kwargs):
-    tmp_dir = Path(tempfile.mkdtemp(**kwargs))
-    for split in ["train", "val", "test"]:
-        _dump_articles((tmp_dir / f"{split}.source"), ARTICLES)
-        _dump_articles((tmp_dir / f"{split}.target"), SUMMARIES)
-    return tmp_dir
-
-
-class TestSummarizationDistiller(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
-        return cls
-
-    @unittest.skipUnless(torch.cuda.device_count() > 1, "skipping multiGPU test")
-    def test_multigpu(self):
-        updates = dict(no_teacher=True, freeze_encoder=True, gpus=2, sortish_sampler=False,)
-        self._test_distiller_cli(updates)
-
-    def test_distill_no_teacher(self):
-        updates = dict(student_encoder_layers=2, student_decoder_layers=1, no_teacher=True)
-        self._test_distiller_cli(updates)
-
-    def test_distill_checkpointing_with_teacher(self):
-        updates = dict(
-            student_encoder_layers=2,
-            student_decoder_layers=1,
-            num_train_epochs=4,
-            val_check_interval=0.25,
-            alpha_hid=2.0,
-            model_name_or_path="IGNORE_THIS_IT_DOESNT_GET_USED",
-        )
-        model = self._test_distiller_cli(updates, check_contents=False)
-
-        ckpts = list(Path(model.output_dir).glob("*.ckpt"))
-        self.assertEqual(1, len(ckpts))
-        transformer_ckpts = list(Path(model.output_dir).glob("**/*.bin"))
-        self.assertEqual(len(transformer_ckpts), 2)
-        examples = lmap(str.strip, model.hparams.data_dir.joinpath("test.source").open().readlines())
-        out_path = tempfile.mktemp()
-        generate_summaries_or_translations(examples, out_path, str(model.output_dir / "best_tfmr"))
-        self.assertTrue(Path(out_path).exists())
-
-        evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
-
-    @unittest.skip("T5 distillation is broken at the moment")
-    def test_distill_t5(self):
-        updates = dict(
-            student_encoder_layers=1,
-            student_decoder_layers=1,
-            alpha_hid=2.0,
-            teacher=T5_TINY,
-            model_name_or_path=T5_TINY,
-            tokenizer_name=T5_TINY,
-        )
-        self._test_distiller_cli(updates)
-
-    def _test_distiller_cli(self, updates, check_contents=True):
-        default_updates = dict(
-            train_batch_size=1,
-            eval_batch_size=2,
-            num_train_epochs=2,
-            alpha_mlm=0.2,
-            alpha_ce=0.8,
-            do_predict=True,
-            model_name_or_path="sshleifer/tinier_bart",
-            teacher=CHEAP_ARGS["model_name_or_path"],
-            val_check_interval=0.5,
-            alpha_encoder_loss=0.4,
-        )
-        default_updates.update(updates)
-        args_d: dict = CHEAP_ARGS.copy()
-        tmp_dir = make_test_data_dir()
-        output_dir = tempfile.mkdtemp(prefix="output_")
-
-        args_d.update(data_dir=tmp_dir, output_dir=output_dir, **default_updates)
-        model = distill_main(argparse.Namespace(**args_d))
-        if not check_contents:
-            return model
-        contents = os.listdir(output_dir)
-        ckpt_name = "val_avg_rouge2=0.0000-step_count=2.ckpt"  # "val_avg_rouge2=0.0000-epoch=1.ckpt"  # "epoch=1-val_avg_rouge2=0.0000.ckpt"
-        contents = {os.path.basename(p) for p in contents}
-        self.assertIn(ckpt_name, contents)
-
-        self.assertIn("test_generations.txt", contents)
-        self.assertIn("test_results.txt", contents)
-
-        metrics = load_json(model.metrics_save_path)
-        last_step_stats = metrics["val"][-1]
-        self.assertGreaterEqual(last_step_stats["val_avg_gen_time"], 0.01)
-        self.assertGreaterEqual(1.0, last_step_stats["val_avg_gen_time"])
-        self.assertIsInstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
-        desired_n_evals = int(args_d["num_train_epochs"] * (1 / args_d["val_check_interval"]) + 1)
-        self.assertEqual(len(metrics["val"]), desired_n_evals)
-        self.assertEqual(len(metrics["test"]), 1)
-        return model
-
-
-@pytest.mark.parametrize(["model"], [pytest.param(T5_TINY), pytest.param(BART_TINY), pytest.param(MBART_TINY)])
-def test_run_eval_bart(model):
-    tmp = Path(tempfile.gettempdir()) / "utest_generations_bart_sum.hypo"
-
-    output_file_name = Path(tempfile.gettempdir()) / "utest_output_bart_sum.hypo"
-    assert not output_file_name.exists()
-    articles = [" New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County."]
-    _dump_articles(tmp, articles)
-    testargs = ["run_eval.py", str(tmp), str(output_file_name), model]  # TODO: test score_path
-    with patch.object(sys, "argv", testargs):
-        run_generate()
-        assert Path(output_file_name).exists()
-        os.remove(Path(output_file_name))
-
-
-@pytest.mark.parametrize(
-    ["model"], [pytest.param(T5_TINY), pytest.param(BART_TINY), pytest.param(MBART_TINY), pytest.param(MARIAN_TINY)]
-)
-def test_finetune(model):
-    args_d: dict = CHEAP_ARGS.copy()
-    task = "translation" if model in [MBART_TINY, MARIAN_TINY] else "summarization"
-    tmp_dir = make_test_data_dir()
-    output_dir = tempfile.mkdtemp(prefix="output_")
-    args_d.update(
-        data_dir=tmp_dir,
-        model_name_or_path=model,
-        tokenizer_name=None,
-        train_batch_size=2,
-        eval_batch_size=2,
-        output_dir=output_dir,
-        do_predict=True,
-        task=task,
-    )
-    assert "n_train" in args_d
-    args = argparse.Namespace(**args_d)
-    main(args)
-
-
-@pytest.mark.parametrize(
-    ["tok"], [pytest.param(T5_TINY), pytest.param(BART_TINY), pytest.param(MBART_TINY), pytest.param(MARIAN_TINY)]
-)
-def test_dataset(tok):
-    tokenizer = AutoTokenizer.from_pretrained(tok)
-    tmp_dir = make_test_data_dir()
-    max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
-    max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
-    trunc_target = 4
-    train_dataset = SummarizationDataset(
-        tokenizer, data_dir=tmp_dir, type_path="train", max_source_length=20, max_target_length=trunc_target,
-    )
-    dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
-    for batch in dataloader:
-        assert batch["attention_mask"].shape == batch["input_ids"].shape
-        # show that articles were trimmed.
-        assert batch["input_ids"].shape[1] == max_len_source
-        assert 20 >= batch["input_ids"].shape[1]  # trimmed significantly
-        # show that targets were truncated
-        assert batch["decoder_input_ids"].shape[1] == trunc_target  # Truncated
-        assert max_len_target > trunc_target  # Truncated
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-export PYTHONPATH="../":"${PYTHONPATH}"
-
-export BS=32
-export GAS=1
-
-python finetune.py \
-    --learning_rate=3e-5 \
-    --fp16 \
-    --gpus 1 \
-    --do_train \
-    --do_predict \
-    --val_check_interval 0.25 \
-    --n_val 500 \
-    --num_train_epochs 2 \
-    --freeze_encoder --freeze_embeds --data_dir $CNN_DIR \
-    --max_target_length 142 --val_max_target_length=142 \
-    --train_batch_size=$BS --eval_batch_size=$BS --gradient_accumulation_steps=$GAS \
-    --data_dir $CNN_DIR \
-    --model_name_or_path sshleifer/student_cnn_12_6 \
-    --tokenizer_name facebook/bart-large \
-    --output_dir distilbart-cnn-12-6 \
-    $@
-
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-export PYTHONPATH="../":"${PYTHONPATH}"
-export BS=16
-export GAS=2
-python distillation.py \
-  --learning_rate=3e-4 \
-  --do_train \
-  --do_predict \
-  --fp16 \
-  --val_check_interval 0.1 --n_val 1000 \
-  --teacher facebook/bart-large-xsum --data_dir $XSUM_DIR \
-  --max_target_length=60 --val_max_target_length=60 --test_max_target_length=100 \
-  --student_decoder_layers 6 --student_encoder_layers 12 \
-  --freeze_encoder --freeze_embeds \
-  --model_name_or_path IGNORED \
-  --alpha_hid=3. --length_penalty=0.5 \
-  --train_batch_size=$BS --eval_batch_size=$BS --gradient_accumulation_steps=$GAS --num_train_epochs=6 \
-  --tokenizer_name facebook/bart-large \
-  --output_dir distilbart_xsum_12_6 \
-  $@
@@ -0,0 +1,70 @@
+### Data
+
+CNN/DailyMail data
+```bash
+cd examples/summarization
+wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/cnn_dm.tgz
+tar -xzvf cnn_dm.tgz
+export CNN_DIR=${PWD}/cnn_dm
+```
+
+this should make a directory called cnn_dm/ with files like `test.source`.
+To use your own data, copy that files format. Each article to be summarized is on its own line.
+
+XSUM Data:
+```bash
+cd examples/summarization
+wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/xsum.tar.gz
+tar -xzvf xsum.tar.gz
+export XSUM_DIR=${PWD}/xsum
+```
+
+
+### Evaluation
+
+To create summaries for each article in dataset, run:
+```bash
+python run_eval.py <path_to_test.source> test_generations.txt <model-name>  --score_path rouge_scores.txt
+```
+The default batch size, 4, fits in 16GB GPU memory, but may need to be adjusted to fit your system.
+
+
+### Training
+Run/modify `finetune.sh`
+
+The following command should work on a 16GB GPU:
+```bash
+export me=`git config user.name`
+./finetune.sh \
+    --data_dir $XSUM_DIR \
+    --train_batch_size=1 \
+    --eval_batch_size=1 \
+    --output_dir="$me"_xsum_results \
+    --num_train_epochs 1
+```
+
+Tips:
+- 1 epoch at batch size 1 for bart-large takes 24 hours, requires 13GB GPU RAM with fp16 on an NVIDIA-V100. 
+- try `bart-base`, `--freeze_encoder` or `--freeze_embeds` for faster training/larger batch size.  (3hr/epoch with bs=8, see below)
+- `fp16_opt_level=O1` (the default works best).
+- If you are finetuning on your own dataset, start from `bart-large-cnn` if you want long summaries and `bart-large-xsum` if you want short summaries.
+(It rarely makes sense to start from `bart-large` unless you are a researching finetuning methods).
+- In addition to the pytorch-lightning .ckpt checkpoint, a transformers checkpoint will be saved.
+Load it with `BartForConditionalGeneration.from_pretrained(f'{output_dir}/best_tfmr)`.
+- At the moment, `--do_predict` does not work in a multi-gpu setting. You need to use `evaluate_checkpoint` or the `run_eval.py` code.
+- If you want to run experiments on improving the summarization finetuning process, try the XSUM Shared Task (below). It's faster to train than CNNDM because the summaries are shorter.    
+
+### XSUM Shared Task
+Compare XSUM results with others by using `--logger wandb_shared`. This requires `wandb` registration.
+Here is an example command
+```bash
+export me=`git config user.name`
+./finetune.sh \
+    --data_dir $XSUM_DIR \
+    --output_dir "$me"_xsum_frozen_embs \
+    --logger wandb_shared \
+    --train_batch_size 16 --eval_batch_size 16 --freeze_embeds --freeze_encoder \
+    --num_train_epochs 6
+```
+
+Results can be viewed [here](https://app.wandb.ai/sshleifer/hf_summarization/table?workspace=user-)
@@ -12,7 +12,7 @@ The model is loaded with the pre-trained weights for the abstractive summarizati
 git clone https://github.com/huggingface/transformers && cd transformers
 pip install .
 pip install nltk py-rouge
-cd examples/seq2seq/bertabs
+cd examples/summarization
 ```

 ## Reproduce the authors'  ROUGE score
@@ -32,12 +32,9 @@ class Seq2SeqLoggingCallback(pl.Callback):
            results_file = od / "test_results.txt"
            generations_file = od / "test_generations.txt"
        else:
-            # this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
-            # If people want this it will be easy enough to add back.
-            results_file = od / f"{type_path}_results/{trainer.global_step:05d}.txt"
-            generations_file = od / f"{type_path}_generations/{trainer.global_step:05d}.txt"
-            results_file.parent.mkdir(exist_ok=True)
-            generations_file.parent.mkdir(exist_ok=True)
+            results_file = od / f"{type_path}_results_{trainer.global_step:05d}.txt"
+            generations_file = od / f"{type_path}_generations_{trainer.global_step:05d}.txt"
+
        with open(results_file, "a+") as writer:
            for key in sorted(metrics):
                if key in ["log", "progress_bar", "preds"]:
@@ -66,25 +63,20 @@ class Seq2SeqLoggingCallback(pl.Callback):
        # mp stands for million parameters
        trainer.logger.log_metrics({"n_params": npars, "mp": npars / 1e6, "grad_mp": n_trainable_pars / 1e6})

+    @rank_zero_only
+    def on_validation_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule):
+        return self._write_logs(trainer, pl_module, "val")
+
    @rank_zero_only
    def on_test_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule):
        return self._write_logs(trainer, pl_module, "test")


-def get_checkpoint_callback(output_dir, metric):
+def get_rouge2_checkpoint_callback(output_dir):
    """Saves the best model by validation ROUGE2 score."""
-    if metric == "rouge2":
-        exp = "{val_avg_rouge2:.4f}-{step_count}"
-    elif metric == "bleu":
-        exp = "{val_avg_bleu:.4f}-{step_count}"
-    else:
-        raise NotImplementedError(
-            f"seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this function."
-        )
-
    checkpoint_callback = ModelCheckpoint(
-        filepath=os.path.join(output_dir, exp),
-        monitor=f"val_{metric}",
+        filepath=os.path.join(output_dir, "{val_avg_rouge2:.4f}-{step_count}"),
+        monitor="val_rouge",
        mode="max",
        save_top_k=1,
        period=0,  # maybe save a checkpoint every time val is run, not just end of epoch.
@@ -39,12 +39,13 @@ except ImportError:
    )


-class BartSummarizationDistiller(SummarizationModule):
+class SummarizationDistiller(SummarizationModule):
    loss_names = ["loss", "ce_loss", "mlm_loss", "enc_mse_loss", "hid_loss_enc", "hid_loss_dec"]

    def __init__(self, hparams):
        assert Path(hparams.data_dir).exists()
-        student, student_cfg, teacher = self.pre_init(hparams)
+
+        d_layers_to_copy, student, student_cfg, teacher = self.pre_init(hparams)

        super().__init__(hparams, model=student, config=student_cfg)
        self.teacher = teacher
@@ -72,15 +73,12 @@ class BartSummarizationDistiller(SummarizationModule):
            del self.teacher.model.encoder

    def pre_init(self, hparams):
-        self.output_dir = Path(hparams.output_dir)
-        self.output_dir.mkdir(exist_ok=True)
+        # Dump empty student model at a path, then call from_pretrained on it
        teacher = BartForConditionalGeneration.from_pretrained(hparams.teacher).eval()
        student_updates = {
            "decoder_layers": hparams.student_decoder_layers,
            "encoder_layers": hparams.student_encoder_layers,
        }
-        if hparams.length_penalty != -1:
-            student_updates["length_penalty"] = hparams.length_penalty
        d_layers_to_copy = get_layers_to_copy(student_updates["decoder_layers"], teacher.config.decoder_layers)
        e_layers_to_copy: List = get_layers_to_copy(student_updates["encoder_layers"], teacher.config.encoder_layers)
        hparams.d_layer_to_copy = d_layers_to_copy
@@ -91,13 +89,9 @@ class BartSummarizationDistiller(SummarizationModule):
        student_cfg = BartConfig(**kw)
        student = BartForConditionalGeneration(student_cfg)
        student, _ = init_student(student, teacher)
-        save_dir = self.output_dir.joinpath("student")
-        save_dir.mkdir(exist_ok=True)
-
        self.copy_to_student(d_layers_to_copy, e_layers_to_copy, hparams, student, teacher)
-        student.save_pretrained(save_dir)
-        hparams.model_name_or_path = str(save_dir)
-        return student, student_cfg, teacher
+        Path(hparams.output_dir).mkdir(exist_ok=True)
+        return d_layers_to_copy, student, student_cfg, teacher

    def copy_to_student(self, d_layers_to_copy, e_layers_to_copy, hparams, student, teacher):
        if teacher.config.model_type == "t5":
@@ -160,6 +154,7 @@ class BartSummarizationDistiller(SummarizationModule):

    def configure_optimizers(self):
        "Prepare optimizer and schedule (linear warmup and decay)"
+
        model = self.model
        no_decay = ["bias", "LayerNorm.weight"]
        optimizer_grouped_parameters = [
@@ -185,11 +180,18 @@ class BartSummarizationDistiller(SummarizationModule):
        # parser.add_argument("--alpha_cos", default=0.0, type=float)
        parser.add_argument("--alpha_encoder_loss", default=0.0, type=float)
        parser.add_argument("--alpha_hid", default=0.0, type=float, required=False)
-        parser.add_argument("--student_decoder_layers", default=12, type=int, required=False)
-        parser.add_argument("--student_encoder_layers", default=12, type=int, required=False)
-        parser.add_argument("--no_teacher", action="store_true", default=False)
-        parser.add_argument("--length_penalty", type=float, default=-1)
-
+        parser.add_argument(
+            "--student_decoder_layers", default=12, type=int, required=False,
+        )
+        parser.add_argument(
+            "--student_encoder_layers", default=12, type=int, required=False,
+        )
+        parser.add_argument(
+            "--no_teacher", action="store_true", default=False,
+        )
+        parser.add_argument(  # TODO: remove
+            "--enc_only", action="store_true", default=False,
+        )
        return parser

    def _step(self, batch):
@@ -267,14 +269,12 @@ class BartSummarizationDistiller(SummarizationModule):
        return sum(hidden_losses)


-class T5SummarizationDistiller(BartSummarizationDistiller):
+class T5SummarizationDistiller(SummarizationDistiller):
    def pre_init(self, hparams):
        raise NotImplementedError("T5 Distillation does not work yet")
-        self.output_dir = Path(hparams.output_dir)
-        self.output_dir.mkdir(exist_ok=True)
        teacher = T5ForConditionalGeneration.from_pretrained(hparams.teacher)
        n_layer = hparams.student_decoder_layers
-        assert n_layer == hparams.student_encoder_layers  # TODO(SS): relax this constraint so that we can do 12-6.
+        assert n_layer == hparams.student_encoder_layers  # TODO(SS): relax this
        d_layers_to_copy = get_layers_to_copy(n_layer, len(teacher.decoder.block))
        e_layers_to_copy: List = get_layers_to_copy(n_layer, len(teacher.encoder.block))
        student_updates = {"num_layers": n_layer}
@@ -291,13 +291,8 @@ class T5SummarizationDistiller(BartSummarizationDistiller):
        Path(hparams.output_dir).mkdir(exist_ok=True)
        task_specific_params = student.config.task_specific_params
        if task_specific_params is not None:
-            student.config.update(task_specific_params.get("summarization", {}))  # TODO: dont hardcode
-        save_dir = self.output_dir.joinpath("student")
-        save_dir.mkdir(exist_ok=True)
-
-        student.save_pretrained(save_dir)
-        hparams.model_name_or_path = str(save_dir)
-        return student, student_cfg, teacher
+            student.config.update(task_specific_params.get("summarization", {}))
+        return d_layers_to_copy, student, student_cfg, teacher

    def freeze_embeds(self):
        freeze_params(self.model.shared)
@@ -391,7 +386,7 @@ def create_module(args):
    elif args.enc_only:
        raise ValueError("Deleted that")
    else:
-        module_cls = BartSummarizationDistiller
+        module_cls = SummarizationDistiller
    args.setup_cls: str = module_cls.__name__
    model = module_cls(args)
    return model
@@ -423,18 +418,18 @@ def evaluate_checkpoint(ckpt_path: Path, dest_dir=None):
 def get_layers_to_copy(n_to_get, tot):
    all_layers = list(range(tot))
    if tot == 12:  # Alternating for special cases
-        layers_to_copy = {  # maps  num layers in student -> which teacher layers to copy
-            1: [0],
-            2: [0, 6],
-            3: [0, 6, 11],
-            4: [0, 4, 8, 11],
+        layers_to_copy = {  # maps # layers in student -> which teacher layers to copy
            6: [0, 2, 4, 7, 9, 11],
+            1: [11],
+            3: [0, 6, 11],
+            2: [0, 11],
+            4: [0, 4, 8, 11],
            9: [0, 1, 2, 4, 5, 7, 9, 10, 11],
            12: all_layers,
        }
        return layers_to_copy[n_to_get]
    else:
-        return all_layers[:n_to_get]  # TODO: better version on theseus-bart branch
+        return all_layers[:n_to_get]


 def distill_main(args):
@@ -448,7 +443,7 @@ def distill_main(args):

 if __name__ == "__main__":
    parser = argparse.ArgumentParser()
-    parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+    parser = SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
    args = parser.parse_args()

    distill_main(args)
@@ -3,7 +3,6 @@ import glob
 import logging
 import os
 import time
-from collections import defaultdict
 from pathlib import Path
 from typing import Dict, List, Tuple

@@ -24,14 +23,12 @@ try:
        flatten_list,
        pickle_save,
        save_git_info,
-        save_json,
        freeze_params,
        calculate_rouge,
        get_git_info,
        ROUGE_KEYS,
-        calculate_bleu_score,
    )
-    from .callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback
+    from .callbacks import Seq2SeqLoggingCallback, get_rouge2_checkpoint_callback
 except ImportError:
    from utils import (
        use_task_specific_params,
@@ -40,14 +37,12 @@ except ImportError:
        flatten_list,
        pickle_save,
        save_git_info,
-        save_json,
        freeze_params,
        calculate_rouge,
        get_git_info,
        ROUGE_KEYS,
-        calculate_bleu_score,
    )
-    from callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback
+    from callbacks import Seq2SeqLoggingCallback, get_rouge2_checkpoint_callback

 logger = logging.getLogger(__name__)

@@ -55,18 +50,15 @@ logger = logging.getLogger(__name__)
 class SummarizationModule(BaseTransformer):
    mode = "summarization"
    loss_names = ["loss"]
-    metric_names = ROUGE_KEYS
-    val_metric = "rouge2"

    def __init__(self, hparams, **kwargs):
        super().__init__(hparams, num_labels=None, mode=self.mode, **kwargs)
        use_task_specific_params(self.model, "summarization")
        save_git_info(self.hparams.output_dir)
-        self.metrics_save_path = Path(self.output_dir) / "metrics.json"
+        self.metrics_save_path = Path(self.output_dir) / "metrics.pkl"
        self.hparams_save_path = Path(self.output_dir) / "hparams.pkl"
-        pickle_save(self.hparams, self.hparams_save_path)
        self.step_count = 0
-        self.metrics = defaultdict(list)
+        self.metrics = {"train": [], "val": [], "test": []}

        self.dataset_kwargs: dict = dict(
            data_dir=self.hparams.data_dir,
@@ -97,12 +89,12 @@ class SummarizationModule(BaseTransformer):

    def freeze_embeds(self):
        """Freeze token embeddings and positional embeddings for bart, just token embeddings for t5."""
-        try:
+        if self.model.config.model_type == "bart":
            freeze_params(self.model.model.shared)
            for d in [self.model.model.encoder, self.model.model.decoder]:
                freeze_params(d.embed_positions)
                freeze_params(d.embed_tokens)
-        except AttributeError:
+        else:
            freeze_params(self.model.shared)
            for d in [self.model.encoder, self.model.decoder]:
                freeze_params(d.embed_tokens)
@@ -138,22 +130,19 @@ class SummarizationModule(BaseTransformer):
        self.step_count += 1
        losses = {k: torch.stack([x[k] for x in outputs]).mean() for k in self.loss_names}
        loss = losses["loss"]
-        rouges = {k: np.array([x[k] for x in outputs]).mean() for k in self.metric_names + ["gen_time", "summ_len"]}
-        rouge_tensor: torch.FloatTensor = torch.tensor(rouges[self.val_metric]).type_as(loss)
+        rouges = {k: np.array([x[k] for x in outputs]).mean() for k in ROUGE_KEYS + ["gen_time", "summ_len"]}
+        rouge_tensor: torch.FloatTensor = torch.tensor(rouges["rouge2"]).type_as(loss)
        rouges.update({k: v.item() for k, v in losses.items()})
        losses.update(rouges)
        metrics = {f"{prefix}_avg_{k}": x for k, x in losses.items()}
        metrics["step_count"] = self.step_count
        self.save_metrics(metrics, prefix)  # writes to self.metrics_save_path
        preds = flatten_list([x["preds"] for x in outputs])
-        return {"log": metrics, "preds": preds, f"{prefix}_loss": loss, f"{prefix}_{self.val_metric}": rouge_tensor}
+        return {"log": metrics, "preds": preds, f"{prefix}_loss": loss, f"{prefix}_rouge": rouge_tensor}

-    def save_metrics(self, latest_metrics, type_path) -> None:
-        self.metrics[type_path].append(latest_metrics)
-        save_json(self.metrics, self.metrics_save_path)
-
-    def calc_generative_metrics(self, preds, target) -> Dict:
-        return calculate_rouge(preds, target)
+    def save_metrics(self, metrics, prefix) -> None:
+        self.metrics[prefix].append(metrics)
+        pickle_save(self.metrics, self.metrics_save_path)

    def _generative_step(self, batch: dict) -> dict:
        pad_token_id = self.tokenizer.pad_token_id
@@ -165,7 +154,7 @@ class SummarizationModule(BaseTransformer):
        target = self.ids_to_clean_text(y)
        loss_tensors = self._step(batch)
        base_metrics = {name: loss for name, loss in zip(self.loss_names, loss_tensors)}
-        rouge: Dict = self.calc_generative_metrics(preds, target)
+        rouge: Dict = calculate_rouge(preds, target)
        summ_len = np.mean(lmap(len, generated_ids))
        base_metrics.update(gen_time=gen_time, summ_len=summ_len, preds=preds, target=target, **rouge)
        return base_metrics
@@ -270,33 +259,15 @@ class SummarizationModule(BaseTransformer):
        parser.add_argument("--n_train", type=int, default=-1, required=False, help="# examples. -1 means use all.")
        parser.add_argument("--n_val", type=int, default=500, required=False, help="# examples. -1 means use all.")
        parser.add_argument("--n_test", type=int, default=-1, required=False, help="# examples. -1 means use all.")
-        parser.add_argument(
-            "--task", type=str, default="summarization", required=False, help="# examples. -1 means use all."
-        )
        return parser


-class TranslationModule(SummarizationModule):
-    mode = "translation"
-    loss_names = ["loss"]
-    metric_names = ["bleu"]
-    val_metric = "bleu"
-
-    def calc_generative_metrics(self, preds, target) -> dict:
-        return calculate_bleu_score(preds, target)
-
-
 def main(args, model=None) -> SummarizationModule:
    Path(args.output_dir).mkdir(exist_ok=True)
    if len(os.listdir(args.output_dir)) > 3 and args.do_train:
        raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
    if model is None:
-        if args.task == "summarization":
-            model: SummarizationModule = SummarizationModule(args)
-        else:
-            model: SummarizationModule = TranslationModule(args)
-
-    dataset = Path(args.data_dir).name
+        model: BaseTransformer = SummarizationModule(args)
    if (
        args.logger == "default"
        or args.fast_dev_run
@@ -307,17 +278,17 @@ def main(args, model=None) -> SummarizationModule:
    elif args.logger == "wandb":
        from pytorch_lightning.loggers import WandbLogger

-        logger = WandbLogger(name=model.output_dir.name, project=dataset)
-
+        logger = WandbLogger(name=model.output_dir.name)
    elif args.logger == "wandb_shared":
        from pytorch_lightning.loggers import WandbLogger

-        logger = WandbLogger(name=model.output_dir.name, project=f"hf_{dataset}")
+        # TODO: separate LB for CNN, we should use Path(args.data_dir).name to determine the correct LB.
+        logger = WandbLogger(name=model.output_dir.name, project="hf_summarization")
    trainer: pl.Trainer = generic_train(
        model,
        args,
        logging_callback=Seq2SeqLoggingCallback(),
-        checkpoint_callback=get_checkpoint_callback(args.output_dir, model.val_metric),
+        checkpoint_callback=get_rouge2_checkpoint_callback(args.output_dir),
        logger=logger,
        # TODO: early stopping callback seems messed up
    )
@@ -1,8 +1,13 @@
+
 # Add parent directory to python path to access lightning_base.py
 export PYTHONPATH="../":"${PYTHONPATH}"

-# the proper usage is documented in the README, you need to specify data_dir, output_dir and model_name_or_path
+
+# --model_name_or_path=t5-base for t5
+
+# the proper usage is documented in the README
 python finetune.py \
+    --model_name_or_path=facebook/bart-large \
    --learning_rate=3e-5 \
    --fp16 \
    --gpus 1 \
@@ -11,4 +16,5 @@ python finetune.py \
    --n_val 1000 \
    --val_check_interval 0.1 \
    --sortish_sampler \
+    --max_target_length=56 \
    $@
@@ -1,3 +1,5 @@
+#CNN_DIR = /home/shleifer/transformers_fork/examples/summarization/bart/cnn_dm
+
 # Add parent directory to python path to access lightning_base.py
 export PYTHONPATH="../":"${PYTHONPATH}"

@@ -9,9 +9,9 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer


 try:
-    from .utils import calculate_rouge, use_task_specific_params, calculate_bleu_score
+    from .finetune import calculate_rouge, use_task_specific_params
 except ImportError:
-    from utils import calculate_rouge, use_task_specific_params, calculate_bleu_score
+    from finetune import calculate_rouge, use_task_specific_params

 DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

@@ -22,14 +22,8 @@ def chunks(lst, n):
        yield lst[i : i + n]


-def generate_summaries_or_translations(
-    examples: list,
-    out_file: str,
-    model_name: str,
-    batch_size: int = 8,
-    device: str = DEFAULT_DEVICE,
-    fp16=False,
-    **gen_kwargs,
+def generate_summaries(
+    examples: list, out_file: str, model_name: str, batch_size: int = 8, device: str = DEFAULT_DEVICE, fp16=False,
 ) -> None:
    fout = Path(out_file).open("w", encoding="utf-8")
    model_name = str(model_name)
@@ -45,10 +39,11 @@ def generate_summaries_or_translations(
    for batch in tqdm(list(chunks(examples, batch_size))):
        if "t5" in model_name:
            batch = [model.config.prefix + text for text in batch]
-        batch = tokenizer.batch_encode_plus(
-            batch, max_length=1024, return_tensors="pt", truncation=True, pad_to_max_length=True
-        ).to(device)
-        summaries = model.generate(**batch, **gen_kwargs)
+        dct = tokenizer.batch_encode_plus(batch, max_length=1024, return_tensors="pt", pad_to_max_length=True).to(
+            device
+        )
+        summaries = model.generate(**dct)
+
        dec = tokenizer.batch_decode(summaries, skip_special_tokens=True, clean_up_tokenization_spaces=False)
        for hypothesis in dec:
            fout.write(hypothesis + "\n")
@@ -62,26 +57,22 @@ def run_generate():
    parser.add_argument("model_name", type=str, help="like facebook/bart-large-cnn,t5-base, etc.")
    parser.add_argument("--reference_path", type=str, required=False, help="like cnn_dm/test_reference_summaries.txt")
    parser.add_argument("--score_path", type=str, required=False, help="where to save the rouge score in json format")
-    parser.add_argument("--metric", type=str, choices=["bleu", "rouge"], default="rouge")
    parser.add_argument("--device", type=str, required=False, default=DEFAULT_DEVICE, help="cuda, cuda:1, cpu etc.")
    parser.add_argument("--bs", type=int, default=8, required=False, help="batch size")
    parser.add_argument("--fp16", action="store_true")
    args = parser.parse_args()
    examples = [" " + x.rstrip() if "t5" in args.model_name else x.rstrip() for x in open(args.input_path).readlines()]

-    generate_summaries_or_translations(
+    generate_summaries(
        examples, args.output_path, args.model_name, batch_size=args.bs, device=args.device, fp16=args.fp16
    )
-
-    output_lns = [x.rstrip() for x in open(args.output_path).readlines()]
-    scores = {}
-    if args.reference_path is not None:
-        score_fn = {"bleu": calculate_bleu_score, "rouge": calculate_rouge}[args.metric]
+    if args.score_path is not None:
+        output_lns = [x.rstrip() for x in open(args.output_path).readlines()]
        reference_lns = [x.rstrip() for x in open(args.reference_path).readlines()]
-        scores: dict = score_fn(output_lns, reference_lns)
-        if args.score_path is not None:
-            json.dump(scores, open("score_path", "w+"))
-    return scores
+
+        rouge: dict = calculate_rouge(output_lns, reference_lns)
+
+        json.dump(rouge, open("score_path", "w+"))


 if __name__ == "__main__":
@@ -0,0 +1,267 @@
+import argparse
+import logging
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import torch
+from torch.utils.data import DataLoader
+
+from transformers import BartTokenizer
+
+from .distillation import distill_main, evaluate_checkpoint
+from .finetune import main
+from .run_eval import generate_summaries, run_generate
+from .utils import SummarizationDataset, lmap, pickle_load
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+logger = logging.getLogger()
+FP16_EVER = False
+CHEAP_ARGS = {
+    "logger": "default",
+    "num_workers": 2,
+    "alpha_hid": 0,
+    "freeze_embeds": True,
+    "enc_only": False,
+    "tgt_suffix": "",
+    "resume_from_checkpoint": None,
+    "sortish_sampler": True,
+    "student_decoder_layers": 1,
+    "val_check_interval": 1.0,
+    "output_dir": "",
+    "fp16": False,
+    "no_teacher": False,
+    "fp16_opt_level": "O1",
+    "gpus": 1 if torch.cuda.is_available() else 0,
+    "n_tpu_cores": 0,
+    "max_grad_norm": 1.0,
+    "do_train": True,
+    "do_predict": True,
+    "gradient_accumulation_steps": 1,
+    "server_ip": "",
+    "server_port": "",
+    "seed": 42,
+    "model_type": "bart",
+    "model_name_or_path": "sshleifer/bart-tiny-random",
+    "config_name": "",
+    "tokenizer_name": "facebook/bart-large",
+    "cache_dir": "",
+    "do_lower_case": False,
+    "learning_rate": 3e-05,
+    "weight_decay": 0.0,
+    "adam_epsilon": 1e-08,
+    "warmup_steps": 0,
+    "num_train_epochs": 1,
+    "train_batch_size": 2,
+    "eval_batch_size": 2,
+    "max_source_length": 12,
+    "max_target_length": 12,
+    "val_max_target_length": 12,
+    "test_max_target_length": 12,
+    "fast_dev_run": False,
+    "no_cache": False,
+    "n_train": -1,
+    "n_val": -1,
+    "n_test": -1,
+    "student_encoder_layers": 1,
+    "alpha_loss_encoder": 0.0,
+    "freeze_encoder": False,
+    "auto_scale_batch_size": False,
+}
+
+
+def _dump_articles(path: Path, articles: list):
+    with path.open("w") as f:
+        f.write("\n".join(articles))
+
+
+MSG = "T5 is broken at the moment"
+T5_TINY = "patrickvonplaten/t5-tiny-random"
+
+
+def make_test_data_dir():
+    tmp_dir = Path(tempfile.gettempdir())
+    articles = [" Sam ate lunch today", "Sams lunch ingredients"]
+    summaries = ["A very interesting story about what I ate for lunch.", "Avocado, celery, turkey, coffee"]
+    for split in ["train", "val", "test"]:
+        _dump_articles((tmp_dir / f"{split}.source"), articles)
+        _dump_articles((tmp_dir / f"{split}.target"), summaries)
+    return tmp_dir
+
+
+class TestSummarizationDistiller(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
+        return cls
+
+    @unittest.skipUnless(torch.cuda.device_count() > 1, "skipping multiGPU test")
+    def test_bdc_multigpu(self):
+        updates = dict(
+            student_encoder_layers=2,
+            student_decoder_layers=1,
+            no_teacher=True,
+            freeze_encoder=True,
+            gpus=2,
+            sortish_sampler=False,
+            fp16_opt_level="O1",
+            fp16=FP16_EVER,
+        )
+        self._bart_distiller_cli(updates)
+
+    def test_bdc_t5_train(self):
+        updates = dict(
+            fp16=FP16_EVER,
+            gpus=1 if torch.cuda.is_available() else 0,
+            model_type="t5",
+            model_name_or_path=T5_TINY,
+            do_train=True,
+            do_predict=True,
+            tokenizer_name=T5_TINY,
+            no_teacher=True,
+            alpha_hid=2.0,
+        )
+        self._bart_distiller_cli(updates)
+
+    def test_bdc_no_teacher(self):
+        updates = dict(student_encoder_layers=2, student_decoder_layers=1, no_teacher=True,)
+        self._bart_distiller_cli(updates)
+
+    def test_bdc_yes_teacher(self):
+        updates = dict(student_encoder_layers=2, student_decoder_layers=1,)
+        self._bart_distiller_cli(updates)
+
+    def test_bdc_checkpointing(self):
+        updates = dict(
+            student_encoder_layers=2,
+            student_decoder_layers=1,
+            num_train_epochs=4,
+            val_check_interval=0.25,
+            alpha_hid=2.0,
+        )
+        model = self._bart_distiller_cli(updates, check_contents=False)
+
+        ckpts = list(Path(model.output_dir).glob("*.ckpt"))
+        self.assertEqual(1, len(ckpts))
+        transformer_ckpts = list(Path(model.output_dir).glob("**/*.bin"))
+        self.assertEqual(len(transformer_ckpts), len(ckpts))
+        new_transformer_ckpts = list(Path(model.output_dir).glob("**/*.bin"))
+        self.assertEqual(len(new_transformer_ckpts), 1)
+        examples = lmap(str.strip, model.hparams.data_dir.joinpath("test.source").open().readlines())
+        out_path = tempfile.mktemp()
+        generate_summaries(examples, out_path, new_transformer_ckpts[0].parent)
+        self.assertTrue(Path(out_path).exists())
+
+        evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
+
+    def _bart_distiller_cli(self, updates, check_contents=True):
+        default_updates = dict(
+            train_batch_size=1,
+            eval_batch_size=2,
+            num_train_epochs=2,
+            alpha_mlm=0.2,
+            alpha_ce=0.8,
+            do_predict=True,
+            gpus=1 if torch.cuda.is_available() else 0,
+            model_name_or_path="sshleifer/tinier_bart",
+            teacher=CHEAP_ARGS["model_name_or_path"],
+            val_check_interval=0.5,
+            alpha_encoder_loss=0.4,
+        )
+        default_updates.update(updates)
+        args_d: dict = CHEAP_ARGS.copy()
+        tmp_dir = make_test_data_dir()
+        output_dir = tempfile.mkdtemp(prefix="output_")
+
+        args_d.update(data_dir=tmp_dir, output_dir=output_dir, **default_updates)
+        model = distill_main(argparse.Namespace(**args_d))
+        if not check_contents:
+            return model
+        contents = os.listdir(output_dir)
+        ckpt_name = "val_avg_rouge2=0.0000-step_count=2.ckpt"  # "val_avg_rouge2=0.0000-epoch=1.ckpt"  # "epoch=1-val_avg_rouge2=0.0000.ckpt"
+        contents = {os.path.basename(p) for p in contents}
+        self.assertIn(ckpt_name, contents)
+        self.assertIn("metrics.pkl", contents)
+        self.assertIn("test_generations.txt", contents)
+        self.assertIn("val_generations_00001.txt", contents)
+        self.assertIn("val_results_00001.txt", contents)
+        self.assertIn("test_results.txt", contents)
+
+        metrics = pickle_load(Path(output_dir) / "metrics.pkl")
+        desired_n_evals = int(args_d["num_train_epochs"] * (1 / args_d["val_check_interval"]) + 1)
+        self.assertEqual(len(metrics["val"]), desired_n_evals)
+        self.assertEqual(len(metrics["train"]), 0)  # doesn't get logged here
+        return model
+
+
+class TestBartExamples(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        stream_handler = logging.StreamHandler(sys.stdout)
+        logger.addHandler(stream_handler)
+        logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
+        return cls
+
+    def test_bart_cnn_cli(self):
+        tmp = Path(tempfile.gettempdir()) / "utest_generations_bart_sum.hypo"
+        output_file_name = Path(tempfile.gettempdir()) / "utest_output_bart_sum.hypo"
+        articles = [" New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County."]
+        _dump_articles(tmp, articles)
+        testargs = ["run_eval.py", str(tmp), str(output_file_name), "sshleifer/bart-tiny-random"]
+        with patch.object(sys, "argv", testargs):
+            run_generate()
+            self.assertTrue(Path(output_file_name).exists())
+            os.remove(Path(output_file_name))
+
+    def test_t5_run_sum_cli(self):
+        args_d: dict = CHEAP_ARGS.copy()
+
+        tmp_dir = make_test_data_dir()
+        output_dir = tempfile.mkdtemp(prefix="output_")
+        args_d.update(
+            data_dir=tmp_dir,
+            model_name_or_path=T5_TINY,
+            tokenizer_name=None,  # T5_TINY,
+            train_batch_size=2,
+            eval_batch_size=2,
+            gpus=0,
+            output_dir=output_dir,
+            do_predict=True,
+        )
+        assert "n_train" in args_d
+        args = argparse.Namespace(**args_d)
+        main(args)
+
+    def test_bart_summarization_dataset(self):
+        tmp_dir = Path(tempfile.gettempdir())
+        articles = [" Sam ate lunch today", "Sams lunch ingredients"]
+        summaries = ["A very interesting story about what I ate for lunch.", "Avocado, celery, turkey, coffee"]
+        _dump_articles((tmp_dir / "train.source"), articles)
+        _dump_articles((tmp_dir / "train.target"), summaries)
+        tokenizer = BartTokenizer.from_pretrained("facebook/bart-large")
+        max_len_source = max(len(tokenizer.encode(a)) for a in articles)
+        max_len_target = max(len(tokenizer.encode(a)) for a in summaries)
+        trunc_target = 4
+        train_dataset = SummarizationDataset(
+            tokenizer, data_dir=tmp_dir, type_path="train", max_source_length=20, max_target_length=trunc_target,
+        )
+        dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
+        for batch in dataloader:
+            self.assertEqual(batch["attention_mask"].shape, batch["input_ids"].shape)
+            # show that articles were trimmed.
+            self.assertEqual(batch["input_ids"].shape[1], max_len_source)
+            self.assertGreater(20, batch["input_ids"].shape[1])  # trimmed significantly
+
+            # show that targets were truncated
+            self.assertEqual(batch["decoder_input_ids"].shape[1], trunc_target)  # Truncated
+            self.assertGreater(max_len_target, trunc_target)  # Truncated
+
+
+def list_to_text_file(lst, path):
+    dest = Path(path)
+    dest.open("w+").writelines(lst)
@@ -3,13 +3,12 @@ import json
 import os
 import pickle
 from pathlib import Path
-from typing import Callable, Dict, Iterable, List
+from typing import Dict, Iterable, List

 import git
 import numpy as np
 import torch
 from rouge_score import rouge_scorer, scoring
-from sacrebleu import corpus_bleu
 from torch import nn
 from torch.utils.data import Dataset, Sampler
 from tqdm import tqdm
@@ -42,7 +41,7 @@ def encode_file(
    examples = []
    for text in tqdm(lns, desc=f"Tokenizing {data_path.name}"):
        tokenized = tokenizer.batch_encode_plus(
-            [text],
+            [text],  # DONT ADD SPACES
            max_length=max_length,
            pad_to_max_length=pad_to_max_length,
            add_prefix_space=True,
@@ -55,13 +54,11 @@ def encode_file(
    return examples


-def lmap(f: Callable, x: Iterable) -> List:
-    """list(map(f, x))"""
+def lmap(f, x):
    return list(map(f, x))


-def calculate_bleu_score(output_lns, refs_lns) -> dict:
-    return {"bleu": corpus_bleu(output_lns, [refs_lns]).score}
+T5_PREFIX = "summarize: "  # HACK, fixme


 def trim_batch(
@@ -98,8 +95,6 @@ class SummarizationDataset(Dataset):
            tok_name=tok_name,
        )
        tgt_path = os.path.join(data_dir, type_path + ".target")
-        if hasattr(tokenizer, "set_lang"):
-            tokenizer.set_lang("ro_RO")  # HACK: only applies to mbart
        self.target = encode_file(
            tokenizer, tgt_path, max_target_length, overwrite_cache=overwrite_cache, tok_name=tok_name
        )
@@ -194,20 +189,14 @@ def flatten_list(summary_ids: List[List]):
    return [x for x in itertools.chain.from_iterable(summary_ids)]


-def save_git_info(folder_path: str) -> None:
-    """Save git information to output_dir/git_log.json"""
+def save_git_info(folder_path: str):
+    """
+    Log commit info.
+    """
    repo_infos = get_git_info()
-    save_json(repo_infos, os.path.join(folder_path, "git_log.json"))

-
-def save_json(content, path):
-    with open(path, "w") as f:
-        json.dump(content, f, indent=4)
-
-
-def load_json(path):
-    with open(path) as f:
-        return json.load(f)
+    with open(os.path.join(folder_path, "git_log.json"), "w") as f:
+        json.dump(repo_infos, f, indent=4)


 def get_git_info():
@@ -74,6 +74,18 @@ python run_glue.py \
  --learning_rate 2e-5 \
  --num_train_epochs 3.0 \
  --output_dir /tmp/$TASK_NAME/
+
+python run_nlp_glue.py \
+  --model_name_or_path bert-base-cased \
+  --task_name mrpc \
+  --do_train \
+  --do_eval \
+  --max_seq_length 128 \
+  --per_device_train_batch_size 32 \
+  --learning_rate 2e-5 \
+  --num_train_epochs 3.0 \
+  --output_dir /tmp/mrpc/ \
+  --overwrite_output_dir
 ```

 where task name can be one of CoLA, SST-2, MRPC, STS-B, QQP, MNLI, QNLI, RTE, WNLI.
@@ -0,0 +1,311 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning the library models for sequence classification on GLUE (Bert, XLM, XLNet, RoBERTa, Albert, XLM-RoBERTa)."""
+
+
+import logging
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Callable, Dict, Optional
+
+import nlp
+import numpy as np
+
+from transformers import (
+    AutoConfig,
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
+    EvalPrediction,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    glue_compute_metrics,
+    set_seed,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class GlueDataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+
+    Using `HfArgumentParser` we can turn this class
+    into argparse arguments to be able to specify them on
+    the command line.
+    """
+
+    task_name: str = field(
+        metadata={
+            "help": "The name of the task to train and/or evaluate on: "
+            "['cola', 'sst2', 'mrpc', 'qqp', 'stsb', 'mnli', 'mnli_mismatched', 'mnli_matched', 'qnli', 'rte', 'wnli', 'ax']"
+        }
+    )
+
+    max_seq_length: int = field(
+        default=128,
+        metadata={
+            "help": "The maximum total input sequence length after tokenization. Sequences longer "
+            "than this will be truncated, sequences shorter will be padded."
+        },
+    )
+
+    def __post_init__(self):
+        self.task_name = self.task_name.lower().replace("-", "")  # We used to allow 'sts-b' for 'stsb'
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
+    """
+
+    model_name_or_path: str = field(
+        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
+    )
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser((ModelArguments, GlueDataTrainingArguments, TrainingArguments))
+
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    if (
+        os.path.exists(training_args.output_dir)
+        and os.listdir(training_args.output_dir)
+        and training_args.do_train
+        and not training_args.overwrite_output_dir
+    ):
+        raise ValueError(
+            f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
+        )
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,
+    )
+    logger.warning(
+        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+        training_args.local_rank,
+        training_args.device,
+        training_args.n_gpu,
+        bool(training_args.local_rank != -1),
+        training_args.fp16,
+    )
+    logger.info("Training/evaluation parameters %s", training_args)
+
+    # Set seed
+    set_seed(training_args.seed)
+
+    # Get tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+        # use_fast=True,
+    )
+
+    # Download, tokenize and prepare datasets for training a PyTorch model
+    task_name = data_args.task_name.replace("-", "")  # We used to allow 'sts-b' for 'stsb'
+    glue = nlp.load_dataset("glue", name=task_name)
+
+    def tokenize(batch):
+        """ Tokenize the dataset and:
+            - ``return_lengths=True`` => add a column with the length of the encoded sequences
+            - ``truncate=True`` => Truncate to the max input length of the model
+        """
+        return tokenizer(batch["sentence1"], batch["sentence2"], return_lengths=True, truncate=True)
+
+    def batch_dataset(batch):
+        """ Pad a batch to the length of the longest sequence in the batch (``padding=True``)
+            And then group the batch in a single example so the sequences will stay together
+        """
+        padded_batch = tokenizer.pad(batch, padding=True)
+        padded_batch = {key: [array] for key, array in padded_batch.items()}
+        return padded_batch
+
+    for split_name, split in glue.items():
+        split = split.map(
+            tokenize, batched=True
+        )  # Tokenize the dataset (adds columns to the dataset w. tokenized inputs)
+        split = split.sort("length")  # Sort the dataset by length
+
+        # Build batches by gathering sequences of similar lengths
+        batch_size = training_args.train_batch_size if split_name == "train" else training_args.eval_batch_size
+        split = split.map(batch_dataset, batched=True, batch_size=batch_size)
+
+        # Set the format of the dataset to output only the columns our model can digest
+        glue[split_name].set_format(columns=["input_ids", "attention_mask", "token_type_ids", "label"])
+
+    # Get the splits
+    train_dataset, eval_dataset, test_dataset = None, None, None
+    if training_args.do_train:
+        train_dataset = glue["train"]
+    if training_args.do_eval:
+        eval_dataset = glue["validation" + ("_matched" if task_name == "mnli" else "")]
+    if training_args.do_predict:
+        test_dataset = glue["test" + ("_matched" if task_name == "mnli" else "")]
+
+    # Define output mode (regression or classification) and num labels (to define the size of the last layer of the model)
+    output_mode = "regression" if "float" in glue["train"].features["label"].dtype else "classification"
+    if output_mode == "regression":
+        num_labels = 1
+    else:
+        num_labels = len(train_dataset.unique("label"))
+
+    if task_name in ["mnli", "mnli-mm"] and tokenizer.__class__.__name__ in (
+        "RobertaTokenizer",
+        "RobertaTokenizerFast",
+        "XLMRobertaTokenizer",
+    ):
+        raise NotImplementedError(
+            """
+        # HACK(label indices are swapped in RoBERTa pretrained model)
+        label_list[1], label_list[2] = label_list[2], label_list[1]
+        """
+        )
+
+    # Load pretrained model and tokenizer
+    #
+    # Distributed training:
+    # The .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+
+    config = AutoConfig.from_pretrained(
+        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
+        num_labels=num_labels,
+        finetuning_task=data_args.task_name,
+        cache_dir=model_args.cache_dir,
+    )
+    model = AutoModelForSequenceClassification.from_pretrained(
+        model_args.model_name_or_path,
+        from_tf=bool(".ckpt" in model_args.model_name_or_path),
+        config=config,
+        cache_dir=model_args.cache_dir,
+    )
+
+    def build_compute_metrics_fn(task_name: str) -> Callable[[EvalPrediction], Dict]:
+        def compute_metrics_fn(p: EvalPrediction):
+            if output_mode == "classification":
+                preds = np.argmax(p.predictions, axis=1)
+            elif output_mode == "regression":
+                preds = np.squeeze(p.predictions)
+            return glue_compute_metrics(task_name, preds, p.label_ids)
+
+        return compute_metrics_fn
+
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+        compute_metrics=build_compute_metrics_fn(data_args.task_name),
+    )
+
+    # Training
+    if training_args.do_train:
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
+        )
+        trainer.save_model()
+        # For convenience, we also re-save the tokenizer to the same directory,
+        # so that you can share your model easily on huggingface.co/models =)
+        if trainer.is_world_master():
+            tokenizer.save_pretrained(training_args.output_dir)
+
+    # Evaluation
+    eval_results = {}
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        # Loop to handle MNLI double evaluation (matched, mis-matched)
+        eval_datasets = [eval_dataset]
+        if data_args.task_name == "mnli":
+            eval_datasets.append(glue["validation_mismatched"])
+
+        for eval_dataset in eval_datasets:
+            trainer.compute_metrics = build_compute_metrics_fn(eval_dataset.args.task_name)
+            eval_result = trainer.evaluate(eval_dataset=eval_dataset)
+
+            output_eval_file = os.path.join(
+                training_args.output_dir, f"eval_results_{eval_dataset.args.task_name}.txt"
+            )
+            if trainer.is_world_master():
+                with open(output_eval_file, "w") as writer:
+                    logger.info("***** Eval results {} *****".format(eval_dataset.args.task_name))
+                    for key, value in eval_result.items():
+                        logger.info("  %s = %s", key, value)
+                        writer.write("%s = %s\n" % (key, value))
+
+            eval_results.update(eval_result)
+
+    if training_args.do_predict:
+        logging.info("*** Test ***")
+        test_datasets = [test_dataset]
+        if data_args.task_name == "mnli":
+            test_datasets.append(glue["test_mismatched"])
+
+        for test_dataset in test_datasets:
+            predictions = trainer.predict(test_dataset=test_dataset).predictions
+            if output_mode == "classification":
+                predictions = np.argmax(predictions, axis=1)
+
+            output_test_file = os.path.join(
+                training_args.output_dir, f"test_results_{test_dataset.args.task_name}.txt"
+            )
+            if trainer.is_world_master():
+                with open(output_test_file, "w") as writer:
+                    logger.info("***** Test results {} *****".format(test_dataset.args.task_name))
+                    writer.write("index\tprediction\n")
+                    for index, item in enumerate(predictions):
+                        if output_mode == "regression":
+                            writer.write("%d\t%3.3f\n" % (index, item))
+                        else:
+                            item = test_dataset.get_labels()[item]
+                            writer.write("%d\t%s\n" % (index, item))
+    return eval_results
+
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,51 @@
+***This script evaluates the multitask pre-trained checkpoint for ``t5-base`` (see paper [here](https://arxiv.org/pdf/1910.10683.pdf)) on the English to German WMT dataset. Please note that the results in the paper were attained using a model fine-tuned on translation, so that results will be worse here by approx. 1.5 BLEU points***
+
+### Intro
+
+This example shows how T5 (here the official [paper](https://arxiv.org/abs/1910.10683)) can be
+evaluated on the WMT English-German dataset.
+
+### Get the WMT Data
+
+To be able to reproduce the authors' results on WMT English to German, you first need to download 
+the WMT14 en-de news datasets.
+Go on Stanford's official NLP [website](https://nlp.stanford.edu/projects/nmt/) and find "newstest2014.en" and "newstest2014.de" under WMT'14 English-German data or download the dataset directly via:
+
+```bash
+curl https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/newstest2014.en > newstest2014.en
+curl https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/newstest2014.de > newstest2014.de
+```
+
+You should have 2737 sentences in each file. You can verify this by running:
+
+```bash
+wc -l newstest2014.en  # should give 2737
+```
+
+### Usage
+
+Let's check the longest and shortest sentence in our file to find reasonable decoding hyperparameters: 
+
+Get the longest and shortest sentence:
+
+```bash 
+awk '{print NF}' newstest2014.en | sort -n | head -1 # shortest sentence has 2 word
+awk '{print NF}' newstest2014.en | sort -n | tail -1 # longest sentence has 91 words
+```
+
+We will set our `max_length` to ~3 times the longest sentence and leave `min_length` to its default value of 0.
+We decode with beam search `num_beams=4` as proposed in the paper. Also as is common in beam search we set `early_stopping=True` and `length_penalty=2.0`.
+
+To create translation for each in dataset and get a final BLEU score, run:
+```bash
+python evaluate_wmt.py <path_to_newstest2014.en> newstest2014_de_translations.txt <path_to_newstest2014.de> newsstest2014_en_de_bleu.txt
+```
+the default batch size, 16, fits in 16GB GPU memory, but may need to be adjusted to fit your system.
+
+### Where is the code?
+The core model is in `src/transformers/modeling_t5.py`. This directory only contains examples.
+
+### BLEU Scores
+
+The BLEU score is calculated using [sacrebleu](https://github.com/mjpost/sacreBLEU) by mjpost.
+To get the BLEU score we used 
@@ -0,0 +1,103 @@
+import argparse
+from pathlib import Path
+
+import torch
+from sacrebleu import corpus_bleu
+from tqdm import tqdm
+
+from transformers import T5ForConditionalGeneration, T5Tokenizer
+
+
+def chunks(lst, n):
+    """Yield successive n-sized chunks from lst."""
+    for i in range(0, len(lst), n):
+        yield lst[i : i + n]
+
+
+def generate_translations(lns, output_file_path, model_size, batch_size, device):
+    model = T5ForConditionalGeneration.from_pretrained(model_size)
+    model.to(device)
+
+    tokenizer = T5Tokenizer.from_pretrained(model_size)
+
+    # update config with summarization specific params
+    task_specific_params = model.config.task_specific_params
+    if task_specific_params is not None:
+        model.config.update(task_specific_params.get("translation_en_to_de", {}))
+
+    with Path(output_file_path).open("w") as output_file:
+        for batch in tqdm(list(chunks(lns, batch_size))):
+            batch = [model.config.prefix + text for text in batch]
+
+            dct = tokenizer.batch_encode_plus(batch, max_length=512, return_tensors="pt", pad_to_max_length=True)
+
+            input_ids = dct["input_ids"].to(device)
+            attention_mask = dct["attention_mask"].to(device)
+
+            translations = model.generate(input_ids=input_ids, attention_mask=attention_mask)
+            dec = [
+                tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in translations
+            ]
+
+            for hypothesis in dec:
+                output_file.write(hypothesis + "\n")
+
+
+def calculate_bleu_score(output_lns, refs_lns, score_path):
+    bleu = corpus_bleu(output_lns, [refs_lns])
+    result = "BLEU score: {}".format(bleu.score)
+    with Path(score_path).open("w") as score_file:
+        score_file.write(result)
+
+
+def run_generate():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "model_size",
+        type=str,
+        help="T5 model size, either 't5-small', 't5-base', 't5-large', 't5-3b', 't5-11b'. Defaults to 't5-base'.",
+        default="t5-base",
+    )
+    parser.add_argument(
+        "input_path", type=str, help="like wmt/newstest2014.en",
+    )
+    parser.add_argument(
+        "output_path", type=str, help="where to save translation",
+    )
+    parser.add_argument(
+        "reference_path", type=str, help="like wmt/newstest2014.de",
+    )
+    parser.add_argument(
+        "score_path", type=str, help="where to save the bleu score",
+    )
+    parser.add_argument(
+        "--batch_size", type=int, default=16, required=False, help="batch size: how many to summarize at a time",
+    )
+    parser.add_argument(
+        "--no_cuda", default=False, type=bool, help="Whether to force the execution on CPU.",
+    )
+
+    args = parser.parse_args()
+    args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+
+    dash_pattern = (" ##AT##-##AT## ", "-")
+
+    # Read input lines into python
+    with open(args.input_path, "r") as input_file:
+        input_lns = [x.strip().replace(dash_pattern[0], dash_pattern[1]) for x in input_file.readlines()]
+
+    generate_translations(input_lns, args.output_path, args.model_size, args.batch_size, args.device)
+
+    # Read generated lines into python
+    with open(args.output_path, "r") as output_file:
+        output_lns = [x.strip() for x in output_file.readlines()]
+
+    # Read reference lines into python
+    with open(args.reference_path, "r") as reference_file:
+        refs_lns = [x.strip().replace(dash_pattern[0], dash_pattern[1]) for x in reference_file.readlines()]
+
+    calculate_bleu_score(output_lns, refs_lns, args.score_path)
+
+
+if __name__ == "__main__":
+    run_generate()
@@ -0,0 +1,50 @@
+import logging
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from .evaluate_wmt import run_generate
+
+
+text = ["When Liana Barrientos was 23 years old, she got married in Westchester County."]
+translation = ["Als Liana Barrientos 23 Jahre alt war, heiratete sie in Westchester County."]
+
+output_file_name = "output_t5_trans.txt"
+score_file_name = "score_t5_trans.txt"
+
+logging.basicConfig(level=logging.DEBUG)
+
+logger = logging.getLogger()
+
+
+class TestT5Examples(unittest.TestCase):
+    def test_t5_cli(self):
+        stream_handler = logging.StreamHandler(sys.stdout)
+        logger.addHandler(stream_handler)
+
+        tmp_source = Path(tempfile.gettempdir()) / "utest_generations_t5_trans.hypo"
+        with tmp_source.open("w") as f:
+            f.write("\n".join(text))
+
+        tmp_target = Path(tempfile.gettempdir()) / "utest_generations_t5_trans.target"
+        with tmp_target.open("w") as f:
+            f.write("\n".join(translation))
+
+        output_file_name = Path(tempfile.gettempdir()) / "utest_output_trans.hypo"
+        score_file_name = Path(tempfile.gettempdir()) / "utest_score.hypo"
+
+        testargs = [
+            "evaluate_wmt.py",
+            "patrickvonplaten/t5-tiny-random",
+            str(tmp_source),
+            str(output_file_name),
+            str(tmp_target),
+            str(score_file_name),
+        ]
+
+        with patch.object(sys, "argv", testargs):
+            run_generate()
+            self.assertTrue(Path(output_file_name).exists())
+            self.assertTrue(Path(score_file_name).exists())
@@ -1,2 +1,2 @@
 This model is used detecting **hatespeech** in **Arabic language**. The mono in the name refers to the monolingual setting, where the model is trained using only Arabic language data. It is finetuned on multilingual bert model.
-The model is trained with different learning rates and the best validation score achieved is 0.877609 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
+The model is trained with different learning rates and the best validation score achieved is 0.8674776 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
@@ -1,2 +1,2 @@
 This model is used detecting **hatespeech** in **English language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model.
-The model is trained with different learning rates and the best validation score achieved is 0.726030 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
+The model is trained with different learning rates and the best validation score achieved is 0.7069374 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
@@ -1,2 +0,0 @@
-This model is used detecting **hatespeech** in **French language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model.
-The model is trained with different learning rates and the best validation score achieved is 0.692094 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
@@ -1,2 +0,0 @@
-This model is used detecting **hatespeech** in **German language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model.
-The model is trained with different learning rates and the best validation score achieved is 0.649794 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
@@ -1,2 +0,0 @@
-This model is used detecting **hatespeech** in **Indonesian language**. The mono in the name refers to the monolingual setting, where the model is trained using only Arabic language data. It is finetuned on multilingual bert model.
-The model is trained with different learning rates and the best validation score achieved is 0.844494 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
@@ -1,2 +0,0 @@
-This model is used detecting **hatespeech** in **Italian language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model.
-The model is trained with different learning rates and the best validation score achieved is 0.837288 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
@@ -1,2 +0,0 @@
-This model is used detecting **hatespeech** in **Polish language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model.
-The model is trained with different learning rates and the best validation score achieved is 0.723254 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
@@ -1,2 +0,0 @@
-This model is used detecting **hatespeech** in **Portuguese language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model.
-The model is trained with different learning rates and the best validation score achieved is 0.716119 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
@@ -1,2 +0,0 @@
-This model is used detecting **hatespeech** in **Spanish language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model.
-The model is trained with different learning rates and the best validation score achieved is 0.740287 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT)
@@ -1,231 +1,10 @@
 ---
-language: english
 tags:
 - exbert
+
 license: apache-2.0
-datasets:
- bookcorpus
- wikipedia
 ---

-# BERT base model (uncased)
-
-Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in
-[this paper](https://arxiv.org/abs/1810.04805) and first released in
-[this repository](https://github.com/google-research/bert). This model is uncased: it does not make a difference
-between english and English.
-
-Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by
-the Hugging Face team.
-
-## Model description
-
-BERT is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it
-was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of
-publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it
-was pretrained with two objectives:
-
- Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run
-  the entire masked sentence through the model and has to predict the masked words. This is different from traditional
-  recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like
-  GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the
-  sentence.
- Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes
-  they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to
-  predict if the two sentences were following each other or not.
-
-This way, the model learns an inner representation of the English language that can then be used to extract features
-useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard
-classifier using the features produced by the BERT model as inputs.
-
-## Intended uses & limitations
-
-You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to
-be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for
-fine-tuned versions on a task that interests you.
-
-Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked)
-to make decisions, such as sequence classification, token classification or question answering. For tasks such as text
-generation you should look at model like GPT2.
-
-### How to use
-
-You can use this model directly with a pipeline for masked language modeling:
-
-```python
->>> from transformers import pipeline
->>> unmasker = pipeline('fill-mask', model='bert-base-uncased')
->>> unmasker("Hello I'm a [MASK] model.")
-
-[{'sequence': "[CLS] hello i'm a fashion model. [SEP]",
-  'score': 0.1073106899857521,
-  'token': 4827,
-  'token_str': 'fashion'},
- {'sequence': "[CLS] hello i'm a role model. [SEP]",
-  'score': 0.08774490654468536,
-  'token': 2535,
-  'token_str': 'role'},
- {'sequence': "[CLS] hello i'm a new model. [SEP]",
-  'score': 0.05338378623127937,
-  'token': 2047,
-  'token_str': 'new'},
- {'sequence': "[CLS] hello i'm a super model. [SEP]",
-  'score': 0.04667217284440994,
-  'token': 3565,
-  'token_str': 'super'},
- {'sequence': "[CLS] hello i'm a fine model. [SEP]",
-  'score': 0.027095865458250046,
-  'token': 2986,
-  'token_str': 'fine'}]
-```
-
-Here is how to use this model to get the features of a given text in PyTorch:
-
-```python
-from transformers import BertTokenizer, BertModel
-tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-model = BertModel.from_pretrained("bert-base-uncased")
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='pt')
-output = model(**encoded_input)
-```
-
-and in TensorFlow:
-
-```python
-from transformers import BertTokenizer, BertModel
-tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-model = BertModel.from_pretrained("bert-base-uncased")
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='tf')
-output = model(encoded_input)
-```
-
-### Limitations and bias
-
-Even if the training data used for this model could be characterized as fairly neutral, this model can have biased
-predictions:
-
-```python
->>> from transformers import pipeline
->>> unmasker = pipeline('fill-mask', model='bert-base-uncased')
->>> unmasker("The man worked as a [MASK].")
-
-[{'sequence': '[CLS] the man worked as a carpenter. [SEP]',
-  'score': 0.09747550636529922,
-  'token': 10533,
-  'token_str': 'carpenter'},
- {'sequence': '[CLS] the man worked as a waiter. [SEP]',
-  'score': 0.0523831807076931,
-  'token': 15610,
-  'token_str': 'waiter'},
- {'sequence': '[CLS] the man worked as a barber. [SEP]',
-  'score': 0.04962705448269844,
-  'token': 13362,
-  'token_str': 'barber'},
- {'sequence': '[CLS] the man worked as a mechanic. [SEP]',
-  'score': 0.03788609802722931,
-  'token': 15893,
-  'token_str': 'mechanic'},
- {'sequence': '[CLS] the man worked as a salesman. [SEP]',
-  'score': 0.037680890411138535,
-  'token': 18968,
-  'token_str': 'salesman'}]
-
->>> unmasker("The woman worked as a [MASK].")
-
-[{'sequence': '[CLS] the woman worked as a nurse. [SEP]',
-  'score': 0.21981462836265564,
-  'token': 6821,
-  'token_str': 'nurse'},
- {'sequence': '[CLS] the woman worked as a waitress. [SEP]',
-  'score': 0.1597415804862976,
-  'token': 13877,
-  'token_str': 'waitress'},
- {'sequence': '[CLS] the woman worked as a maid. [SEP]',
-  'score': 0.1154729500412941,
-  'token': 10850,
-  'token_str': 'maid'},
- {'sequence': '[CLS] the woman worked as a prostitute. [SEP]',
-  'score': 0.037968918681144714,
-  'token': 19215,
-  'token_str': 'prostitute'},
- {'sequence': '[CLS] the woman worked as a cook. [SEP]',
-  'score': 0.03042375110089779,
-  'token': 5660,
-  'token_str': 'cook'}]
-```
-
-This bias will also affect all fine-tuned versions of this model.
-
-## Training data
-
-The BERT model was pretrained on [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038
-unpublished books and [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and
-headers).
-
-## Training procedure
-
-### Preprocessing
-
-The texts are lowercased and tokenized using WordPiece and a vocabulary size of 30,000. The inputs of the model are
-then of the form:
-
-```
-[CLS] Sentence A [SEP] Sentence B [SEP]
-```
-
-With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in
-the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a
-consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two
-"sentences" has a combined length of less than 512 tokens.
-
-The details of the masking procedure for each sentence are the following:
- 15% of the tokens are masked.
- In 80% of the cases, the masked tokens are replaced by `[MASK]`.
- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace.
- In the 10% remaining cases, the masked tokens are left as is.
-
-### Pretraining
-
-The model was trained on 4 cloud TPUs in Pod configuration (16 TPU chips total) for one million steps with a batch size
-of 256. The sequence length was limited to 128 tokens for 90% of the steps and 512 for the remaining 10%. The optimizer
-used is Adam with a learning rate of 1e-4, \\(\beta_{1} = 0.9\\) and \\(\beta_{2} = 0.999\\), a weight decay of 0.01,
-learning rate warmup for 10,000 steps and linear decay of the learning rate after.
-
-## Evaluation results
-
-When fine-tuned on downstream tasks, this model achieves the following results:
-
-Glue test results:
-
-| Task | MNLI-(m/mm) | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  | Average |
-|:----:|:-----------:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|:-------:|
-|      | 84.6/83.4   | 71.2 | 90.5 | 93.5  | 52.1 | 85.8  | 88.9 | 66.4 | 79.6    |
-
-
-### BibTeX entry and citation info
-
-```bibtex
-@article{DBLP:journals/corr/abs-1810-04805,
-  author    = {Jacob Devlin and
-               Ming{-}Wei Chang and
-               Kenton Lee and
-               Kristina Toutanova},
-  title     = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
-               Understanding},
-  journal   = {CoRR},
-  volume    = {abs/1810.04805},
-  year      = {2018},
-  url       = {http://arxiv.org/abs/1810.04805},
-  archivePrefix = {arXiv},
-  eprint    = {1810.04805},
-  timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},
-  biburl    = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib},
-  bibsource = {dblp computer science bibliography, https://dblp.org}
-}
-```
-
 <a href="https://huggingface.co/exbert/?model=bert-base-uncased">
 	<img width="300px" src="https://hf-dinosaur.huggingface.co/exbert/button.png">
 </a>
@@ -1,9 +1,4 @@
 ---
 datasets:
 - squad
-widget:
- text: "Which name is also used to describe the Amazon rainforest in English?"
-  context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
- text: "How many square kilometers of rainforest is covered in the basin?"
-  context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
 ---
@@ -1,147 +1,10 @@
 ---
-language: english
 tags:
 - exbert

 license: mit
 ---

-
-# GPT-2
-
-Pretrained model on English language using a causal language modeling (CLM) objective. It was introduced in
-[this paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf)
-and first released at [this page](https://openai.com/blog/better-language-models/).
-
-Disclaimer: The team releasing GPT-2 did not write a model card for this model so this model card has been written by
-the Hugging Face team.
-
-## Model description
-
-GPT-2 is a transformers model pretrained on a very large corpus of English data in a self-supervised fashion. This
-means it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots
-of publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely,
-it was trained to guess the next word in sentences.
-
-More precisely, inputs are sequences of continuous text of a certain length and the targets are the same sequence,
-shifted one token (word or piece of word) to the right. The model uses internally a mask-mechanism to make sure the
-predictions for the token `i` only uses the inputs from `1` to `i` but not the future tokens.
-
-This way, the model learns an inner representation of the English language that can then be used to extract features
-useful for downstream tasks. The model is best at what it was pretrained for however, which is generating texts from a
-prompt.
-
-## Intended uses & limitations
-
-You can use the raw model for text generation or fine-tune it to a downstream task. See the
-[model hub](https://huggingface.co/models?filter=gpt2) to look for fine-tuned versions on a task that interests you.
-
-### How to use
-
-You can use this model directly with a pipeline for text generation. Since the generation relies on some randomness, we
-set a seed for reproducibility:
-
-```python
->>> from transformers import pipeline, set_seed
->>> generator = pipeline('text-generation', model='gpt2')
->>> set_seed(42)
->>> generator("Hello, I'm a language model,", max_length=30, num_return_sequences=5)
-
-[{'generated_text': "Hello, I'm a language model, a language for thinking, a language for expressing thoughts."},
- {'generated_text': "Hello, I'm a language model, a compiler, a compiler library, I just want to know how I build this kind of stuff. I don"},
- {'generated_text': "Hello, I'm a language model, and also have more than a few of your own, but I understand that they're going to need some help"},
- {'generated_text': "Hello, I'm a language model, a system model. I want to know my language so that it might be more interesting, more user-friendly"},
- {'generated_text': 'Hello, I\'m a language model, not a language model"\n\nThe concept of "no-tricks" comes in handy later with new'}]
-```
-
-Here is how to use this model to get the features of a given text in PyTorch:
-
-```python
-from transformers import GPT2Tokenizer, GPT2Model
-tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
-model = GPT2Model.from_pretrained('gpt2')
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='pt')
-output = model(**encoded_input)
-```
-
-and in TensorFlow:
-
-```python
-from transformers import GPT2Tokenizer, TFGPT2Model
-tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
-model = TFGPT2Model.from_pretrained('gpt2')
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='tf')
-output = model(encoded_input)
-```
-
-### Limitations and bias
-
-The training data used for this model has not been released as a dataset one can browse. We know it contains a lot of
-unfiltered from the internet, which is far from neutral. Therefore, the model can have biased predictions:
-
-```python
->>> from transformers import pipeline, set_seed
->>> generator = pipeline('text-generation', model='gpt2')
->>> set_seed(42)
->>> generator("The White man worked as a", max_length=10, num_return_sequences=5)
-
-[{'generated_text': 'The White man worked as a mannequin for'},
- {'generated_text': 'The White man worked as a maniser of the'},
- {'generated_text': 'The White man worked as a bus conductor by day'},
- {'generated_text': 'The White man worked as a plumber at the'},
- {'generated_text': 'The White man worked as a journalist. He had'}]
-
->>> set_seed(42)
->>> generator("The Black man worked as a", max_length=10, num_return_sequences=5)
-
-[{'generated_text': 'The Black man worked as a man at a restaurant'},
- {'generated_text': 'The Black man worked as a car salesman in a'},
- {'generated_text': 'The Black man worked as a police sergeant at the'},
- {'generated_text': 'The Black man worked as a man-eating monster'},
- {'generated_text': 'The Black man worked as a slave, and was'}]
-```
-
-This bias will also affect all fine-tuned versions of this model.
-
-## Training data
-
-The OpenAI team wanted to train this model on a corpus as large as possible. To build it, they scraped all the web
-pages from outbound links on Reddit which received at least 3 karma. Note that all Wikipedia pages were removed from
-this dataset, so the model was not trained on any part of Wikipedia. The resulting dataset (called WebText) weights
-40GB of texts but has not been publicly released.
-
-## Training procedure
-
-### Preprocessing
-
-The texts are tokenized using a byte-level version of Byte Pair Encoding (BPE) (for unicode characters) and a
-vocabulary size of 50,257. The inputs are sequences of 1024 consecutive tokens.
-
-The larger model was trained on 256 cloud TPU v3 cores. The training duration was not disclosed, nor were the exact
-details of training.
-
-## Evaluation results
-
-The model achieves the following results without any fine-tuning (zero-shot):
-
-| Dataset  | LAMBADA | LAMBADA | CBT-CN | CBT-NE | WikiText2 | PTB    | enwiki8 | text8  | WikiText103 | 1BW   |
-|:--------:|:-------:|:-------:|:------:|:------:|:---------:|:------:|:-------:|:------:|:-----------:|:-----:|
-| (metric) | (PPL)   | (ACC)   | (ACC)  | (ACC)  | (PPL)     | (PPL)  | (BPB)   | (BPC)  | (PPL)       | (PPL) |
-|          | 35.13   | 45.99   | 87.65  | 83.4   | 29.41     | 65.85  | 1.16    | 1,17   | 37.50       | 75.20 |
-
-
-### BibTeX entry and citation info
-
-```bibtex
-@article{radford2019language,
-  title={Language Models are Unsupervised Multitask Learners},
-  author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
-  year={2019}
-}
-```
-
 <a href="https://huggingface.co/exbert/?model=gpt2">
 	<img width="300px" src="https://hf-dinosaur.huggingface.co/exbert/button.png">
 </a>
@@ -1,8 +1,6 @@
 ---
 language: esperanto
 thumbnail: https://huggingface.co/blog/assets/EsperBERTo-thumbnail-v2.png
-widget:
- text: "Mi estas viro kej estas tago varma."
 ---

 # EsperBERTo: RoBERTa-like Language model trained on Esperanto
@@ -1,10 +1,6 @@
 ---
 language: esperanto
 thumbnail: https://huggingface.co/blog/assets/EsperBERTo-thumbnail-v2.png
-widget:
- text: "Jen la komenco de bela <mask>."
- text: "Uno du <mask>"
- text: "Jen finiĝas bela <mask>."
 ---

 # EsperBERTo: RoBERTa-like Language model trained on Esperanto
@@ -1,23 +0,0 @@
-# Flaubert-base-cased-ecology_crisis
-
-An adapted [__Flaubert/Flaubert_base-cased model__](https://github.com/getalp/Flaubert) Trained further on a Language modeling Task of unlabeled French tweets used to create the [CrisisDataset](https://github.com/DiegoKoz/french_ecological_crisis), The intermediate task of masqued language modeling helped us improve the results on our [paper](http://www.sciencedirect.com/science/article/pii/S0306457320300650) compared to the standard flaubert-base-cased model.
-
-If you use this pretrained model on your work, please cite us as follows 🤗
-
-```
-@article{Kozlowski-et-al2020,
-title = "A three-level classification of French tweets in ecological crises",
-journal = "Information Processing & Management",
-volume = "57",
-number = "5",
-pages = "102284",
-year = "2020",
-issn = "0306-4573",
-doi = "https://doi.org/10.1016/j.ipm.2020.102284",
-url = "http://www.sciencedirect.com/science/article/pii/S0306457320300650",
-author = "Diego Kozlowski and Elisa Lannelongue and Frédéric Saudemont and Farah Benamara and Alda Mari and Véronique Moriceau and Abdelmoumene Boumadane",
-keywords = "Crisis response from social media, Machine learning, Natural language processing, Transfer learning",
-}
-
-```
-
@@ -17,7 +17,6 @@ Pull Request so it can be included under the Community notebooks.
 | [How to train a language model](https://github.com/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb)| Highlight all the steps to effectively train Transformer model on custom data | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb)|
 | [How to generate text](https://github.com/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)| How to use different decoding methods for language generation with transformers | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)|
 | [How to export model to ONNX](https://github.com/huggingface/transformers/blob/master/notebooks/04-onnx-export.ipynb) | Highlight how to export and run inference workloads through ONNX |
-| [How to use Benchmarks](https://github.com/huggingface/transformers/blob/master/notebooks/05-benchmark.ipynb) | How to benchmark models with transformers | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/transformers/blob/master/notebooks/05-benchmark.ipynb)|


 ## Community notebooks:
@@ -20,7 +20,6 @@ known_third_party =
    pandas
    PIL
    psutil
-    pytest
    pytorch_lightning
    rouge_score
    sacrebleu
@@ -86,7 +86,7 @@ extras["all"] = extras["serving"] + ["tensorflow", "torch"]

 extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "psutil"]
 # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
-extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
+extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3"]
 extras["quality"] = [
    "black",
    "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
@@ -169,8 +169,7 @@ if is_sklearn_available():

 # Modeling
 if is_torch_available():
-    from .modeling_utils import PreTrainedModel, prune_layer, Conv1D, apply_chunking_to_forward
-    from .modeling_generation_utils import top_k_top_p_filtering
+    from .modeling_utils import PreTrainedModel, prune_layer, Conv1D, top_k_top_p_filtering, apply_chunking_to_forward
    from .modeling_auto import (
        AutoModel,
        AutoModelForPreTraining,
@@ -407,11 +406,9 @@ if is_torch_available():

 # TensorFlow
 if is_tf_available():
-    from .modeling_tf_generation_utils import (
+    from .modeling_tf_utils import (
        shape_list,
        tf_top_k_top_p_filtering,
-    )
-    from .modeling_tf_utils import (
        TFPreTrainedModel,
        TFSequenceSummary,
        TFSharedEmbeddings,
@@ -87,18 +87,8 @@ class PyTorchBenchmark(Benchmark):

        if self.args.torchscript:
            config.torchscript = True
-
-        has_model_class_in_config = hasattr(config, "architecture") and len(config.architectures) > 1
-        if not self.args.only_pretrain_model and has_model_class_in_config:
-            try:
-                model_class = config.architectures[0]
-                transformers_module = __import__("transformers", fromlist=[model_class])
-                model_cls = getattr(transformers_module, model_class)
-                model = model_cls(config)
-            except ImportError:
-                raise ImportError(
-                    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to set `--only_pretrain_model` or `args.only_pretrain_model=True`."
-                )
+        if self.args.with_lm_head:
+            model = MODEL_WITH_LM_HEAD_MAPPING[config.__class__](config)
        else:
            model = MODEL_MAPPING[config.__class__](config)

@@ -137,20 +127,7 @@ class PyTorchBenchmark(Benchmark):

    def _prepare_train_func(self, model_name: str, batch_size: int, sequence_length: int) -> Callable[[], None]:
        config = self.config_dict[model_name]
-
-        has_model_class_in_config = hasattr(config, "architecture") and len(config.architectures) > 1
-        if not self.args.only_pretrain_model and has_model_class_in_config:
-            try:
-                model_class = config.architectures[0]
-                transformers_module = __import__("transformers", fromlist=[model_class])
-                model_cls = getattr(transformers_module, model_class)
-                model = model_cls(config)
-            except ImportError:
-                raise ImportError(
-                    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to set `--only_pretrain_model` or `args.only_pretrain_model=True`."
-                )
-        else:
-            model = MODEL_WITH_LM_HEAD_MAPPING[config.__class__](config)
+        model = MODEL_WITH_LM_HEAD_MAPPING[config.__class__](config)

        if self.args.torchscript:
            raise NotImplementedError("Training for torchscript is currently not implemented")
@@ -74,6 +74,12 @@ class BenchmarkArguments:
            "help": "Don't use multiprocessing for memory and speed measurement. It is highly recommended to use multiprocessing for accurate CPU and GPU memory measurements. This option should only be used for debugging / testing and on TPU."
        },
    )
+    with_lm_head: bool = field(
+        default=False,
+        metadata={
+            "help": "Use model with its language model head (MODEL_WITH_LM_HEAD_MAPPING instead of MODEL_MAPPING)"
+        },
+    )
    inference_time_csv_file: str = field(
        default=f"inference_time_{round(time())}.csv",
        metadata={"help": "CSV filename used if saving time results to csv."},
@@ -99,12 +105,6 @@ class BenchmarkArguments:
        metadata={"help": "Log filename used if print statements are saved in log."},
    )
    repeat: int = field(default=3, metadata={"help": "Times an experiment will be run."})
-    only_pretrain_model: bool = field(
-        default=False,
-        metadata={
-            "help": "Instead of loading the model as defined in `config.architectures` if exists, just load the pretrain model weights."
-        },
-    )

    def to_json_string(self):
        """
@@ -24,7 +24,13 @@ import timeit
 from functools import wraps
 from typing import Callable, Optional

-from transformers import TF_MODEL_MAPPING, PretrainedConfig, is_py3nvml_available, is_tf_available
+from transformers import (
+    TF_MODEL_MAPPING,
+    TF_MODEL_WITH_LM_HEAD_MAPPING,
+    PretrainedConfig,
+    is_py3nvml_available,
+    is_tf_available,
+)

 from .benchmark_utils import (
    Benchmark,
@@ -119,17 +125,8 @@ class TensorflowBenchmark(Benchmark):
        if self.args.fp16:
            raise NotImplementedError("Mixed precision is currently not supported.")

-        has_model_class_in_config = hasattr(config, "architecture") and len(config.architectures) > 1
-        if not self.args.only_pretrain_model and has_model_class_in_config:
-            try:
-                model_class = "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
-                transformers_module = __import__("transformers", fromlist=[model_class])
-                model_cls = getattr(transformers_module, model_class)
-                model = model_cls(config)
-            except ImportError:
-                raise ImportError(
-                    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to set `--only_pretrain_model` or `args.only_pretrain_model=True`."
-                )
+        if self.args.with_lm_head:
+            model = TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](config)
        else:
            model = TF_MODEL_MAPPING[config.__class__](config)

@@ -752,7 +752,6 @@ class Benchmark(ABC):
            info["time"] = datetime.time(datetime.now())
            info["fp16"] = self.args.fp16
            info["use_multiprocessing"] = self.args.do_multi_processing
-            info["only_pretrain_model"] = self.args.only_pretrain_model

            if is_psutil_available():
                info["cpu_ram_mb"] = bytes_to_mega_bytes(psutil.virtual_memory().total)
@@ -807,7 +806,7 @@ class Benchmark(ABC):
                    else:
                        result = str(result)
                    self.print_fn(
-                        model_name[:30].center(30) + str(batch_size).center(15),
+                        model_name.center(30) + str(batch_size).center(15),
                        str(sequence_length).center(15),
                        result.center(15),
                    )
@@ -81,22 +81,22 @@ class AlbertConfig(PretrainedConfig):

        Example::

-            >>> from transformers import AlbertConfig, AlbertModel
-            >>> # Initializing an ALBERT-xxlarge style configuration
-            >>> albert_xxlarge_configuration = AlbertConfig()
+            from transformers import AlbertConfig, AlbertModel
+            # Initializing an ALBERT-xxlarge style configuration
+            albert_xxlarge_configuration = AlbertConfig()

-            >>> # Initializing an ALBERT-base style configuration
-            >>> albert_base_configuration = AlbertConfig(
-            ...      hidden_size=768,
-            ...      num_attention_heads=12,
-            ...      intermediate_size=3072,
-            ...  )
+            # Initializing an ALBERT-base style configuration
+            albert_base_configuration = AlbertConfig(
+                hidden_size=768,
+                num_attention_heads=12,
+                intermediate_size=3072,
+            )

-            >>> # Initializing a model from the ALBERT-base style configuration
-            >>> model = AlbertModel(albert_xxlarge_configuration)
+            # Initializing a model from the ALBERT-base style configuration
+            model = AlbertModel(albert_xxlarge_configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """

    model_type = "albert"
@@ -73,13 +73,9 @@ class BartConfig(PretrainedConfig):
    ):
        r"""
            :class:`~transformers.BartConfig` is the configuration class for `BartModel`.
-
-            Examples::
-
-                >>> from transformers import BartConfig, BartModel
-
-                >>> config = BartConfig.from_pretrained('facebook/bart-large')
-                >>> model = BartModel(config)
+            Examples:
+                config = BartConfig.from_pretrained('bart-large')
+                model = BartModel(config)
        """
        if "hidden_size" in common_kwargs:
            raise ValueError("hidden size is called d_model")
@@ -95,16 +95,16 @@ class BertConfig(PretrainedConfig):

        Example::

-            >>> from transformers import BertModel, BertConfig
+            from transformers import BertModel, BertConfig

-            >>> # Initializing a BERT bert-base-uncased style configuration
-            >>> configuration = BertConfig()
+            # Initializing a BERT bert-base-uncased style configuration
+            configuration = BertConfig()

-            >>> # Initializing a model from the bert-base-uncased style configuration
-            >>> model = BertModel(configuration)
+            # Initializing a model from the bert-base-uncased style configuration
+            model = BertModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """
    model_type = "bert"

@@ -66,16 +66,16 @@ class CTRLConfig(PretrainedConfig):

        Example::

-            >>> from transformers import CTRLModel, CTRLConfig
+            from transformers import CTRLModel, CTRLConfig

-            >>> # Initializing a CTRL configuration
-            >>> configuration = CTRLConfig()
+            # Initializing a CTRL configuration
+            configuration = CTRLConfig()

-            >>> # Initializing a model from the configuration
-            >>> model = CTRLModel(configuration)
+            # Initializing a model from the configuration
+            model = CTRLModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """

    model_type = "ctrl"
@@ -80,16 +80,16 @@ class DistilBertConfig(PretrainedConfig):

        Example::

-            >>> from transformers import DistilBertModel, DistilBertConfig
+            from transformers import DistilBertModel, DistilBertConfig

-            >>> # Initializing a DistilBERT configuration
-            >>> configuration = DistilBertConfig()
+            # Initializing a DistilBERT configuration
+            configuration = DistilBertConfig()

-            >>> # Initializing a model from the configuration
-            >>> model = DistilBertModel(configuration)
+            # Initializing a model from the configuration
+            model = DistilBertModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """
    model_type = "distilbert"

@@ -101,16 +101,16 @@ class ElectraConfig(PretrainedConfig):

        Example::

-            >>> from transformers import ElectraModel, ElectraConfig
+            from transformers import ElectraModel, ElectraConfig

-            >>> # Initializing a ELECTRA electra-base-uncased style configuration
-            >>> configuration = ElectraConfig()
+            # Initializing a ELECTRA electra-base-uncased style configuration
+            configuration = ElectraConfig()

-            >>> # Initializing a model from the electra-base-uncased style configuration
-            >>> model = ElectraModel(configuration)
+            # Initializing a model from the electra-base-uncased style configuration
+            model = ElectraModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """
    model_type = "electra"

@@ -42,20 +42,20 @@ class EncoderDecoderConfig(PretrainedConfig):

        Example::

-            >>> from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel
+            from transformers import BertConfig, EncoderDecoderConfig, EncoderDecoderModel

-            >>> # Initializing a BERT bert-base-uncased style configuration
-            >>> config_encoder = BertConfig()
-            >>> config_decoder = BertConfig()
+            # Initializing a BERT bert-base-uncased style configuration
+            config_encoder = BertConfig()
+            config_decoder = BertConfig()

-            >>> config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)
+            config = EncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)

-            >>> # Initializing a Bert2Bert model from the bert-base-uncased style configurations
-            >>> model = EncoderDecoderModel(config=config)
+            # Initializing a Bert2Bert model from the bert-base-uncased style configurations
+            model = EncoderDecoderModel(config=config)

-            >>> # Accessing the model configuration
-            >>> config_encoder = model.config.encoder
-            >>> config_decoder  = model.config.decoder
+            # Accessing the model configuration
+            config_encoder = model.config.encoder
+            config_decoder  = model.config.decoder
    """
    model_type = "encoder_decoder"

@@ -100,16 +100,16 @@ class GPT2Config(PretrainedConfig):

        Example::

-            >>> from transformers import GPT2Model, GPT2Config
+            from transformers import GPT2Model, GPT2Config

-            >>> # Initializing a GPT2 configuration
-            >>> configuration = GPT2Config()
+            # Initializing a GPT2 configuration
+            configuration = GPT2Config()

-            >>> # Initializing a model from the configuration
-            >>> model = GPT2Model(configuration)
+            # Initializing a model from the configuration
+            model = GPT2Model(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """

    model_type = "gpt2"
@@ -49,16 +49,16 @@ class LongformerConfig(RobertaConfig):

        Example::

-            >>> from transformers import LongformerConfig, LongformerModel
+            from transformers import LongformerConfig, LongformerModel

-            >>> # Initializing a Longformer configuration
-            >>> configuration = LongformerConfig()
+            # Initializing a Longformer configuration
+            configuration = LongformerConfig()

-            >>> # Initializing a model from the configuration
-            >>> model = LongformerModel(configuration)
+            # Initializing a model from the configuration
+            model = LongformerModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """
    model_type = "longformer"

@@ -85,16 +85,16 @@ class MobileBertConfig(PretrainedConfig):

        Example:

-            >>> from transformers import MobileBertModel, MobileBertConfig
+            from transformers import MobileBertModel, MobileBertConfig

-            >>> # Initializing a MobileBERT configuration
-            >>> configuration = MobileBertConfig()
+            # Initializing a MobileBERT configuration
+            configuration = MobileBertConfig()

-            >>> # Initializing a model from the configuration above
-            >>> model = MobileBertModel(configuration)
+            # Initializing a model from the configuration above
+            model = MobileBertModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config

        Attributes:
            pretrained_config_archive_map (Dict[str, str]):
@@ -98,16 +98,16 @@ class OpenAIGPTConfig(PretrainedConfig):

        Example::

-            >>> from transformers import OpenAIGPTConfig, OpenAIGPTModel
+            from transformers import OpenAIGPTConfig, OpenAIGPTModel

-            >>> # Initializing a GPT configuration
-            >>> configuration = OpenAIGPTConfig()
+            # Initializing a GPT configuration
+            configuration = OpenAIGPTConfig()

-            >>> # Initializing a model from the configuration
-            >>> model = OpenAIGPTModel(configuration)
+            # Initializing a model from the configuration
+            model = OpenAIGPTModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """

    model_type = "openai-gpt"
@@ -125,16 +125,16 @@ class ReformerConfig(PretrainedConfig):

        Example::

-            >>> from transformers import ReformerModel, ReformerConfig
+            from transformers import ReformerModel, ReformerConfig

-            >>> # Initializing a Reformer configuration
-            >>> configuration = ReformerConfig()
+            # Initializing a Reformer configuration
+            configuration = ReformerConfig()

-            >>> # Initializing a Reformer model
-            >>> model = ReformerModel(configuration)
+            # Initializing a Reformer model
+            model = ReformerModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """
    model_type = "reformer"

@@ -49,16 +49,16 @@ class RobertaConfig(BertConfig):

        Example::

-            >>> from transformers import RobertaConfig, RobertaModel
+            from transformers import RobertaConfig, RobertaModel

-            >>> # Initializing a RoBERTa configuration
-            >>> configuration = RobertaConfig()
+            # Initializing a RoBERTa configuration
+            configuration = RobertaConfig()

-            >>> # Initializing a model from the configuration
-            >>> model = RobertaModel(configuration)
+            # Initializing a model from the configuration
+            model = RobertaModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """
    model_type = "roberta"

@@ -100,16 +100,16 @@ class TransfoXLConfig(PretrainedConfig):

        Example::

-            >>> from transformers import TransfoXLConfig, TransfoXLModel
+            from transformers import TransfoXLConfig, TransfoXLModel

-            >>> # Initializing a Transformer XL configuration
-            >>> configuration = TransfoXLConfig()
+            # Initializing a Transformer XL configuration
+            configuration = TransfoXLConfig()

-            >>> # Initializing a model from the configuration
-            >>> model = TransfoXLModel(configuration)
+            # Initializing a model from the configuration
+            model = TransfoXLModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """

    model_type = "transfo-xl"
@@ -142,16 +142,16 @@ class XLMConfig(PretrainedConfig):

        Example::

-            >>> from transformers import XLMConfig, XLMModel
+            from transformers import XLMConfig, XLMModel

-            >>> # Initializing a XLM configuration
-            >>> configuration = XLMConfig()
+            # Initializing a XLM configuration
+            configuration = XLMConfig()

-            >>> # Initializing a model from the configuration
-            >>> model = XLMModel(configuration)
+            # Initializing a model from the configuration
+            model = XLMModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """

    model_type = "xlm"
@@ -113,16 +113,16 @@ class XLNetConfig(PretrainedConfig):

        Example::

-            >>> from transformers import XLNetConfig, XLNetModel
+            from transformers import XLNetConfig, XLNetModel

-            >>> # Initializing a XLNet configuration
-            >>> configuration = XLNetConfig()
+            # Initializing a XLNet configuration
+            configuration = XLNetConfig()

-            >>> # Initializing a model from the configuration
-            >>> model = XLNetModel(configuration)
+            # Initializing a model from the configuration
+            model = XLNetModel(configuration)

-            >>> # Accessing the model configuration
-            >>> configuration = model.config
+            # Accessing the model configuration
+            configuration = model.config
    """

    model_type = "xlnet"
@@ -114,21 +114,15 @@ def infer_shapes(nlp: Pipeline, framework: str) -> Tuple[List[str], List[str], D
    return input_vars, output_names, dynamic_axes, tokens


-def load_graph_from_args(pipeline_name: str, framework: str, model: str, tokenizer: Optional[str] = None) -> Pipeline:
+def load_graph_from_args(framework: str, model: str, tokenizer: Optional[str] = None) -> Pipeline:
    # If no tokenizer provided
    if tokenizer is None:
        tokenizer = model

-    # Check the wanted framework is available
-    if framework == "pt" and not is_torch_available():
-        raise Exception("Cannot convert because PyTorch is not installed. Please install torch first.")
-    if framework == "tf" and not is_tf_available():
-        raise Exception("Cannot convert because TF is not installed. Please install tensorflow first.")
-
    print("Loading pipeline (model: {}, tokenizer: {})".format(model, tokenizer))

    # Allocate tokenizer and model
-    return pipeline(pipeline_name, model=model, tokenizer=tokenizer, framework=framework)
+    return pipeline(args.pipeline, model=model, tokenizer=tokenizer, framework=framework)


 def convert_pytorch(nlp: Pipeline, opset: int, output: str, use_external_format: bool):
@@ -160,7 +154,9 @@ def convert_pytorch(nlp: Pipeline, opset: int, output: str, use_external_format:

 def convert_tensorflow(nlp: Pipeline, opset: int, output: str):
    if not is_tf_available():
-        raise Exception("Cannot convert because TF is not installed. Please install tensorflow first.")
+        raise Exception(
+            "Cannot convert {} because TF is not installed. Please install torch first.".format(args.model)
+        )

    print("/!\\ Please note TensorFlow doesn't support exporting model > 2Gb /!\\")

@@ -191,12 +187,11 @@ def convert(
    opset: int,
    tokenizer: Optional[str] = None,
    use_external_format: bool = False,
-    pipeline_name: str = "feature-extraction",
 ):
    print("ONNX opset version set to: {}".format(opset))

    # Load the pipeline
-    nlp = load_graph_from_args(pipeline_name, framework, model, tokenizer)
+    nlp = load_graph_from_args(framework, model, tokenizer)

    parent = dirname(output)
    if not exists(parent):
@@ -234,15 +229,7 @@ if __name__ == "__main__":

    try:
        # Convert
-        convert(
-            args.framework,
-            args.model,
-            args.output,
-            args.opset,
-            args.tokenizer,
-            args.use_external_format,
-            args.pipeline,
-        )
+        convert(args.framework, args.model, args.output, args.opset, args.tokenizer, args.use_external_format)

        # And verify
        if args.check_loading:
@@ -82,9 +82,7 @@ class DataCollatorForLanguageModeling:
            inputs, labels = self.mask_tokens(batch)
            return {"input_ids": inputs, "labels": labels}
        else:
-            labels = batch.clone().detach()
-            labels[labels == self.tokenizer.pad_token_id] = -100
-            return {"input_ids": batch, "labels": labels}
+            return {"input_ids": batch, "labels": batch}

    def _tensorize_batch(self, examples: List[torch.Tensor]) -> torch.Tensor:
        length_of_first = examples[0].size(0)
@@ -488,11 +488,11 @@ class SquadProcessor(DataProcessor):

        Examples::

-            >>> import tensorflow_datasets as tfds
-            >>> dataset = tfds.load("squad")
+            import tensorflow_datasets as tfds
+            dataset = tfds.load("squad")

-            >>> training_examples = get_examples_from_dataset(dataset, evaluate=False)
-            >>> evaluation_examples = get_examples_from_dataset(dataset, evaluate=True)
+            training_examples = get_examples_from_dataset(dataset, evaluate=False)
+            evaluation_examples = get_examples_from_dataset(dataset, evaluate=True)
        """

        if evaluate:
@@ -186,263 +186,6 @@ def add_end_docstrings(*docstr):
    return docstring_decorator


-PT_TOKEN_CLASSIFICATION_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import torch
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-        >>> labels = torch.tensor([1] * inputs["input_ids"].size(1)).unsqueeze(0)  # Batch size 1
-
-        >>> outputs = model(**inputs, labels=labels)
-        >>> loss, scores = outputs[:2]
-"""
-
-PT_QUESTION_ANSWERING_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import torch
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-        >>> start_positions = torch.tensor([1])
-        >>> end_positions = torch.tensor([3])
-
-        >>> outputs = model(**inputs, start_positions=start_positions, end_positions=end_positions)
-        >>> loss, start_scores, end_scores = outputs[:3]
-"""
-
-PT_SEQUENCE_CLASSIFICATION_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import torch
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-        >>> labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
-        >>> outputs = model(**inputs, labels=labels)
-        >>> loss, logits = outputs[:2]
-"""
-
-PT_MASKED_LM_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import torch
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> input_ids = tokenizer("Hello, my dog is cute", return_tensors="pt")["input_ids"]
-
-        >>> outputs = model(input_ids, labels=input_ids)
-        >>> loss, prediction_scores = outputs[:2]
-"""
-
-PT_BASE_MODEL_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import torch
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-        >>> outputs = model(**inputs)
-
-        >>> last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
-"""
-
-PT_MULTIPLE_CHOICE_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import torch
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
-        >>> choice0 = "It is eaten with a fork and a knife."
-        >>> choice1 = "It is eaten while held in the hand."
-        >>> labels = torch.tensor(0).unsqueeze(0)  # choice0 is correct (according to Wikipedia ;)), batch size 1
-
-        >>> encoding = tokenizer([[prompt, prompt], [choice0, choice1]], return_tensors='pt', pad_to_max_length=True)
-        >>> outputs = model(**{{k: v.unsqueeze(0) for k,v in encoding.items()}}, labels=labels)  # batch size is 1
-
-        >>> # the linear classifier still needs to be trained
-        >>> loss, logits = outputs[:2]
-"""
-
-PT_CAUSAL_LM_SAMPLE = r"""
-    Example::
-
-        >>> import torch
-        >>> from transformers import {tokenizer_class}, {model_class}
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-        >>> outputs = model(**inputs, labels=inputs["input_ids"])
-        >>> loss, logits = outputs[:2]
-"""
-
-TF_TOKEN_CLASSIFICATION_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import tensorflow as tf
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")
-        >>> input_ids = inputs["input_ids"]
-        >>> inputs["labels"] = tf.reshape(tf.constant([1] * tf.size(input_ids).numpy()), (-1, tf.size(input_ids))) # Batch size 1
-
-        >>> outputs = model(inputs)
-        >>> loss, scores = outputs[:2]
-"""
-
-TF_QUESTION_ANSWERING_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import tensorflow as tf
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
-        >>> input_dict = tokenizer(question, text, return_tensors='tf')
-        >>> start_scores, end_scores = model(input_dict)
-
-        >>> all_tokens = tokenizer.convert_ids_to_tokens(input_dict["input_ids"].numpy()[0])
-        >>> answer = ' '.join(all_tokens[tf.math.argmax(start_scores, 1)[0] : tf.math.argmax(end_scores, 1)[0]+1])
-"""
-
-TF_SEQUENCE_CLASSIFICATION_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import tensorflow as tf
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")
-        >>> inputs["labels"] = tf.reshape(tf.constant(1), (-1, 1)) # Batch size 1
-
-        >>> outputs = model(inputs)
-        >>> loss, logits = outputs[:2]
-"""
-
-TF_MASKED_LM_SAMPLE = r"""
-    Example::
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import tensorflow as tf
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :]  # Batch size 1
-
-        >>> outputs = model(input_ids)
-        >>> prediction_scores = outputs[0]
-"""
-
-TF_BASE_MODEL_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import tensorflow as tf
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")
-        >>> outputs = model(inputs)
-
-        >>> last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
-"""
-
-TF_MULTIPLE_CHOICE_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import tensorflow as tf
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
-        >>> choice0 = "It is eaten with a fork and a knife."
-        >>> choice1 = "It is eaten while held in the hand."
-
-        >>> encoding = tokenizer([[prompt, prompt], [choice0, choice1]], return_tensors='tf', pad_to_max_length=True)
-        >>> inputs = {{k: tf.expand_dims(v, 0) for k, v in encoding.items()}}
-        >>> outputs = model(inputs)  # batch size is 1
-
-        >>> # the linear classifier still needs to be trained
-        >>> logits = outputs[0]
-"""
-
-TF_CAUSAL_LM_SAMPLE = r"""
-    Example::
-
-        >>> from transformers import {tokenizer_class}, {model_class}
-        >>> import tensorflow as tf
-
-        >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}')
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")
-        >>> outputs = model(inputs)
-        >>> logits = outputs[0]
-"""
-
-
-def add_code_sample_docstrings(*docstr, tokenizer_class=None, checkpoint=None):
-    def docstring_decorator(fn):
-        model_class = fn.__qualname__.split(".")[0]
-        is_tf_class = model_class[:2] == "TF"
-
-        if "SequenceClassification" in model_class:
-            code_sample = TF_SEQUENCE_CLASSIFICATION_SAMPLE if is_tf_class else PT_SEQUENCE_CLASSIFICATION_SAMPLE
-        elif "QuestionAnswering" in model_class:
-            code_sample = TF_QUESTION_ANSWERING_SAMPLE if is_tf_class else PT_QUESTION_ANSWERING_SAMPLE
-        elif "TokenClassification" in model_class:
-            code_sample = TF_TOKEN_CLASSIFICATION_SAMPLE if is_tf_class else PT_TOKEN_CLASSIFICATION_SAMPLE
-        elif "MultipleChoice" in model_class:
-            code_sample = TF_MULTIPLE_CHOICE_SAMPLE if is_tf_class else PT_MULTIPLE_CHOICE_SAMPLE
-        elif "MaskedLM" in model_class:
-            code_sample = TF_MASKED_LM_SAMPLE if is_tf_class else PT_MASKED_LM_SAMPLE
-        elif "LMHead" in model_class:
-            code_sample = TF_CAUSAL_LM_SAMPLE if is_tf_class else PT_CAUSAL_LM_SAMPLE
-        elif "Model" in model_class:
-            code_sample = TF_BASE_MODEL_SAMPLE if is_tf_class else PT_BASE_MODEL_SAMPLE
-        else:
-            raise ValueError(f"Docstring can't be built for model {model_class}")
-
-        built_doc = code_sample.format(model_class=model_class, tokenizer_class=tokenizer_class, checkpoint=checkpoint)
-        fn.__doc__ = (fn.__doc__ or "") + "".join(docstr) + built_doc
-        return fn
-
-    return docstring_decorator
-
-
 def is_remote_url(url_or_filename):
    parsed = urlparse(url_or_filename)
    return parsed.scheme in ("http", "https")
@@ -80,7 +80,6 @@ class ModelInfo:
        author: Optional[str] = None,
        downloads: Optional[int] = None,
        tags: List[str] = [],
-        pipeline_tag: Optional[str] = None,
        siblings: Optional[List[Dict]] = None,  # list of files that constitute the model
        **kwargs
    ):
@@ -89,7 +88,6 @@ class ModelInfo:
        self.author = author
        self.downloads = downloads
        self.tags = tags
-        self.pipeline_tag = pipeline_tag
        self.siblings = [S3Object(**x) for x in siblings] if siblings is not None else None
        for k, v in kwargs.items():
            setattr(self, k, v)
@@ -24,15 +24,13 @@ import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss

 from .configuration_albert import AlbertConfig
-from .file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_callable
+from .file_utils import add_start_docstrings, add_start_docstrings_to_callable
 from .modeling_bert import ACT2FN, BertEmbeddings, BertSelfAttention, prune_linear_layer
 from .modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices


 logger = logging.getLogger(__name__)

-_TOKENIZER_FOR_DOC = "AlbertTokenizer"
-

 ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
    "albert-base-v1",
@@ -487,7 +485,6 @@ class AlbertModel(AlbertPreTrainedModel):
            self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads)

    @add_start_docstrings_to_callable(ALBERT_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="albert-base-v2")
    def forward(
        self,
        input_ids=None,
@@ -524,6 +521,18 @@ class AlbertModel(AlbertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Example::
+
+        from transformers import AlbertModel, AlbertTokenizer
+        import torch
+
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = AlbertModel.from_pretrained('albert-base-v2')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids)
+        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
+
        """

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
@@ -648,16 +657,16 @@ class AlbertForPreTraining(AlbertPreTrainedModel):

    Examples::

-        >>> from transformers import AlbertTokenizer, AlbertForPreTraining
-        >>> import torch
+        from transformers import AlbertTokenizer, AlbertForPreTraining
+        import torch

-        >>> tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
-        >>> model = AlbertForPreTraining.from_pretrained('albert-base-v2')
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = AlbertForPreTraining.from_pretrained('albert-base-v2')

-        >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
-        >>> outputs = model(input_ids)
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids)

-        >>> prediction_scores, sop_scores = outputs[:2]
+        prediction_scores, sop_scores = outputs[:2]

        """

@@ -754,7 +763,6 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
        return self.predictions.decoder

    @add_start_docstrings_to_callable(ALBERT_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="albert-base-v2")
    def forward(
        self,
        input_ids=None,
@@ -794,6 +802,18 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Example::
+
+        from transformers import AlbertTokenizer, AlbertForMaskedLM
+        import torch
+
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = AlbertForMaskedLM.from_pretrained('albert-base-v2')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=input_ids)
+        loss, prediction_scores = outputs[:2]
+
        """
        if "masked_lm_labels" in kwargs:
            warnings.warn(
@@ -843,7 +863,6 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):
        self.init_weights()

    @add_start_docstrings_to_callable(ALBERT_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="albert-base-v2")
    def forward(
        self,
        input_ids=None,
@@ -880,6 +899,19 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+        Examples::
+
+            from transformers import AlbertTokenizer, AlbertForSequenceClassification
+            import torch
+
+            tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+            model = AlbertForSequenceClassification.from_pretrained('albert-base-v2')
+            input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+            labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+            outputs = model(input_ids, labels=labels)
+            loss, logits = outputs[:2]
+
        """

        outputs = self.albert(
@@ -930,7 +962,6 @@ class AlbertForTokenClassification(AlbertPreTrainedModel):
        self.init_weights()

    @add_start_docstrings_to_callable(ALBERT_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="albert-base-v2")
    def forward(
        self,
        input_ids=None,
@@ -965,6 +996,21 @@ class AlbertForTokenClassification(AlbertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Examples::
+
+        from transformers import AlbertTokenizer, AlbertForTokenClassification
+        import torch
+
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = AlbertForTokenClassification.from_pretrained('albert-base-v2')
+
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+
+        loss, scores = outputs[:2]
+
        """

        outputs = self.albert(
@@ -1016,7 +1062,6 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):
        self.init_weights()

    @add_start_docstrings_to_callable(ALBERT_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="albert-base-v2")
    def forward(
        self,
        input_ids=None,
@@ -1059,6 +1104,21 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Examples::
+
+        # The checkpoint albert-base-v2 is not fine-tuned for question answering. Please see the
+        # examples/question-answering/run_squad.py example to see how to fine-tune a model to a question answering task.
+
+        from transformers import AlbertTokenizer, AlbertForQuestionAnswering
+        import torch
+
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = AlbertForQuestionAnswering.from_pretrained('albert-base-v2')
+        question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
+        input_dict = tokenizer.encode_plus(question, text, return_tensors='pt')
+        start_scores, end_scores = model(**input_dict)
+
        """

        outputs = self.albert(
@@ -1116,7 +1176,6 @@ class AlbertForMultipleChoice(AlbertPreTrainedModel):
        self.init_weights()

    @add_start_docstrings_to_callable(ALBERT_INPUTS_DOCSTRING.format("(batch_size, num_choices, sequence_length)"))
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="albert-base-v2")
    def forward(
        self,
        input_ids=None,
@@ -1154,6 +1213,25 @@ class AlbertForMultipleChoice(AlbertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Examples::
+
+        from transformers import AlbertTokenizer, AlbertForMultipleChoice
+        import torch
+
+        tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+        model = AlbertForMultipleChoice.from_pretrained('albert-base-v2')
+
+        prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
+        choice0 = "It is eaten with a fork and a knife."
+        choice1 = "It is eaten while held in the hand."
+        labels = torch.tensor(0).unsqueeze(0)  # choice0 is correct (according to Wikipedia ;)), batch size 1
+
+        encoding = tokenizer.batch_encode_plus([[prompt, choice0], [prompt, choice1]], return_tensors='pt', pad_to_max_length=True)
+        outputs = model(**{k: v.unsqueeze(0) for k,v in encoding.items()}, labels=labels) # batch size is 1
+
+        # the linear classifier still needs to be trained
+        loss, logits = outputs[:2]
        """
        num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]

@@ -392,8 +392,8 @@ class AutoModel:

        Examples::

-            >>> config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
-            >>> model = AutoModel.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
+            config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
+            model = AutoModel.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
        """
        for config_class, model_class in MODEL_MAPPING.items():
            if isinstance(config, config_class):
@@ -480,7 +480,8 @@ class AutoModel:
        Examples::

            model = AutoModel.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
-            assert model.config.output_attentions == True
+            model = AutoModel.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
+            assert model.config.output_attention == True
            # Loading from a TF checkpoint file instead of a PyTorch model (slower)
            config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
            model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
@@ -546,8 +547,8 @@ class AutoModelForPreTraining:

        Examples::

-            >>> config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
-            >>> model = AutoModelForPreTraining.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
+            config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
+            model = AutoModelForPreTraining.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
        """
        for config_class, model_class in MODEL_FOR_PRETRAINING_MAPPING.items():
            if isinstance(config, config_class):
@@ -27,19 +27,12 @@ from torch.nn import CrossEntropyLoss

 from .activations import ACT2FN
 from .configuration_bart import BartConfig
-from .file_utils import (
-    add_code_sample_docstrings,
-    add_end_docstrings,
-    add_start_docstrings,
-    add_start_docstrings_to_callable,
-)
+from .file_utils import add_start_docstrings, add_start_docstrings_to_callable
 from .modeling_utils import PreTrainedModel


 logger = logging.getLogger(__name__)

-_TOKENIZER_FOR_DOC = "BartTokenizer"
-

 BART_PRETRAINED_MODEL_ARCHIVE_LIST = [
    "facebook/bart-large",
@@ -63,17 +56,14 @@ BART_START_DOCSTRING = r"""

 """
 BART_GENERATION_EXAMPLE = r"""
-    Summarization example::
+    Examples::

        from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig
-
        # see ``examples/summarization/bart/run_eval.py`` for a longer example
        model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
        tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')
-
        ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
-        inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
-
+        inputs = tokenizer.batch_encode_plus([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
        # Generate Summary
        summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True)
        print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
@@ -817,7 +807,6 @@ class BartModel(PretrainedBartModel):
        self.init_weights()

    @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="facebook/bart-large")
    def forward(
        self,
        input_ids,
@@ -826,19 +815,14 @@ class BartModel(PretrainedBartModel):
        encoder_outputs: Optional[Tuple] = None,
        decoder_attention_mask=None,
        decoder_cached_states=None,
-        use_cache=None,
+        use_cache=False,
        output_attentions=None,
        output_hidden_states=None,
    ):
-
-        if decoder_input_ids is None:
-            use_cache = False
-
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
-        use_cache = use_cache if use_cache is not None else self.config.use_cache

        # make masks if user doesn't supply
        if not use_cache:
@@ -894,7 +878,8 @@ class BartModel(PretrainedBartModel):


@add_start_docstrings(
-    "The BART Model with a language modeling head. Can be used for summarization.", BART_START_DOCSTRING
+    "The BART Model with a language modeling head. Can be used for summarization.",
+    BART_START_DOCSTRING + BART_GENERATION_EXAMPLE,
 )
 class BartForConditionalGeneration(PretrainedBartModel):
    base_model_prefix = "model"
@@ -921,7 +906,6 @@ class BartForConditionalGeneration(PretrainedBartModel):
        self.register_buffer("final_logits_bias", new_bias)

    @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)
-    @add_end_docstrings(BART_GENERATION_EXAMPLE)
    def forward(
        self,
        input_ids,
@@ -931,7 +915,7 @@ class BartForConditionalGeneration(PretrainedBartModel):
        decoder_attention_mask=None,
        decoder_cached_states=None,
        labels=None,
-        use_cache=None,
+        use_cache=False,
        output_attentions=None,
        output_hidden_states=None,
        **unused,
@@ -962,21 +946,18 @@ class BartForConditionalGeneration(PretrainedBartModel):
            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.

-    Conditional generation example::
+    Examples::

            # Mask filling only works for bart-large
            from transformers import BartTokenizer, BartForConditionalGeneration
-            tokenizer = BartTokenizer.from_pretrained('facebook/bart-large')
+            tokenizer = BartTokenizer.from_pretrained('bart-large')
            TXT = "My friends are <mask> but they eat too many carbs."
-
-            model = BartForConditionalGeneration.from_pretrained('facebook/bart-large')
-            input_ids = tokenizer([TXT], return_tensors='pt')['input_ids']
+            model = BartForConditionalGeneration.from_pretrained('bart-large')
+            input_ids = tokenizer.batch_encode_plus([TXT], return_tensors='pt')['input_ids']
            logits = model(input_ids)[0]
-
            masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
            probs = logits[0, masked_index].softmax(dim=0)
            values, predictions = probs.topk(5)
-
            tokenizer.decode(predictions).split()
            # ['good', 'great', 'all', 'really', 'very']
        """
@@ -987,9 +968,6 @@ class BartForConditionalGeneration(PretrainedBartModel):
            )
            labels = unused.pop("lm_labels")

-        if labels is not None:
-            use_cache = False
-
        outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
@@ -1082,7 +1060,6 @@ class BartForSequenceClassification(PretrainedBartModel):
        self.model._init_weights(self.classification_head.out_proj)

    @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="facebook/bart-large")
    def forward(
        self,
        input_ids,
@@ -1093,7 +1070,6 @@ class BartForSequenceClassification(PretrainedBartModel):
        labels=None,
        output_attentions=None,
        output_hidden_states=None,
-        use_cache=None,
    ):
        r"""
        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
@@ -1103,23 +1079,33 @@ class BartForSequenceClassification(PretrainedBartModel):

    Returns:
        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BartConfig`) and inputs:
-            loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
-                Classification loss (cross entropy)
-            logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
-                Classification (or regression if config.num_labels==1) scores (before SoftMax).
-            hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
-                of shape :obj:`(batch_size, sequence_length, hidden_size)`.
-                Hidden-states of the model at the output of each layer plus the initial embedding outputs.
-            attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or ``config.output_attentions=True``):
-                Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
-                Attentions weights after the attention softmax, used to compute the weighted average in the
-                self-attention
-                heads.
-        """
-        if labels is not None:
-            use_cache = False
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
+            Classification loss (cross entropy)
+        logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.

+    Examples::
+
+        from transformers import BartTokenizer, BartForSequenceClassification
+        import torch
+
+        tokenizer = BartTokenizer.from_pretrained('bart-large')
+        model = BartForSequenceClassification.from_pretrained('bart-large')
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute",
+        add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+        loss, logits = outputs[:2]
+
+        """
        outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
@@ -1128,7 +1114,6 @@ class BartForSequenceClassification(PretrainedBartModel):
            encoder_outputs=encoder_outputs,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
-            use_cache=use_cache,
        )
        x = outputs[0]  # last hidden state
        eos_mask = input_ids.eq(self.config.eos_token_id)
@@ -1163,7 +1148,6 @@ class BartForQuestionAnswering(PretrainedBartModel):
        self.model._init_weights(self.qa_outputs)

    @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="facebook/bart-large")
    def forward(
        self,
        input_ids,
@@ -1175,7 +1159,6 @@ class BartForQuestionAnswering(PretrainedBartModel):
        end_positions=None,
        output_attentions=None,
        output_hidden_states=None,
-        use_cache=None,
    ):
        r"""
        start_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
@@ -1203,9 +1186,26 @@ class BartForQuestionAnswering(PretrainedBartModel):
            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Examples::
+
+        # The checkpoint bart-large is not fine-tuned for question answering. Please see the
+        # examples/question-answering/run_squad.py example to see how to fine-tune a model to a question answering task.
+
+        from transformers import BartTokenizer, BartForQuestionAnswering
+        import torch
+
+        tokenizer = BartTokenizer.from_pretrained('facebook/bart-large')
+        model = BartForQuestionAnswering.from_pretrained('facebook/bart-large')
+
+        question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
+        input_ids = tokenizer.encode(question, text)
+        start_scores, end_scores = model(torch.tensor([input_ids]))
+
+        all_tokens = tokenizer.convert_ids_to_tokens(input_ids)
+        answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])
+
        """
-        if start_positions is not None and end_positions is not None:
-            use_cache = False

        outputs = self.model(
            input_ids,
@@ -1215,7 +1215,6 @@ class BartForQuestionAnswering(PretrainedBartModel):
            encoder_outputs=encoder_outputs,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
-            use_cache=use_cache,
        )

        sequence_output = outputs[0]
@@ -1243,7 +1242,7 @@ class BartForQuestionAnswering(PretrainedBartModel):
            total_loss = (start_loss + end_loss) / 2
            outputs = (total_loss,) + outputs

-        return outputs  # return outputs  # (loss), start_logits, end_logits, encoder_outputs, (hidden_states), (attentions)
+        return outputs  # (loss), start_logits, end_logits, (hidden_states), (attentions)


 class SinusoidalPositionalEmbedding(nn.Embedding):
@@ -28,14 +28,12 @@ from torch.nn import CrossEntropyLoss, MSELoss

 from .activations import gelu, gelu_new, swish
 from .configuration_bert import BertConfig
-from .file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_callable
+from .file_utils import add_start_docstrings, add_start_docstrings_to_callable
 from .modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices, prune_linear_layer


 logger = logging.getLogger(__name__)

-_TOKENIZER_FOR_DOC = "BertTokenizer"
-
 BERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
    "bert-base-uncased",
    "bert-large-uncased",
@@ -666,7 +664,6 @@ class BertModel(BertPreTrainedModel):
            self.encoder.layer[layer].attention.prune_heads(heads)

    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="bert-base-uncased")
    def forward(
        self,
        input_ids=None,
@@ -705,6 +702,20 @@ class BertModel(BertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Examples::
+
+        from transformers import BertModel, BertTokenizer
+        import torch
+
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertModel.from_pretrained('bert-base-uncased')
+
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids)
+
+        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
+
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
@@ -840,16 +851,16 @@ class BertForPreTraining(BertPreTrainedModel):

    Examples::

-        >>> from transformers import BertTokenizer, BertForPreTraining
-        >>> import torch
+        from transformers import BertTokenizer, BertForPreTraining
+        import torch

-        >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-        >>> model = BertForPreTraining.from_pretrained('bert-base-uncased')
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertForPreTraining.from_pretrained('bert-base-uncased')

-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-        >>> outputs = model(**inputs)
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids)

-        >>> prediction_scores, seq_relationship_scores = outputs[:2]
+        prediction_scores, seq_relationship_scores = outputs[:2]

        """
        if "masked_lm_labels" in kwargs:
@@ -947,20 +958,19 @@ class BertLMHeadModel(BertPreTrainedModel):
            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.

-    Example::
+        Examples::

-        >>> from transformers import BertTokenizer, BertLMHeadModel, BertConfig
-        >>> import torch
+            from transformers import BertTokenizer, BertLMHeadModel
+            import torch

-        >>> tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
-        >>> config = BertConfig.from_pretrained("bert-base-cased")
-        >>> config.is_decoder = True
-        >>> model = BertLMHeadModel.from_pretrained('bert-base-cased', config=config)
+            tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+            model = BertLMHeadModel.from_pretrained('bert-base-uncased', is_decoder=True)

-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-        >>> outputs = model(**inputs)
+            input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+            outputs = model(input_ids, labels=input_ids)
+
+            loss, prediction_scores = outputs[:2]

-        >>> last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
        """

        outputs = self.bert(
@@ -1018,7 +1028,6 @@ class BertForMaskedLM(BertPreTrainedModel):
        return self.cls.predictions.decoder

    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="bert-base-uncased")
    def forward(
        self,
        input_ids=None,
@@ -1060,6 +1069,20 @@ class BertForMaskedLM(BertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+        Examples::
+
+            from transformers import BertTokenizer, BertForMaskedLM
+            import torch
+
+            tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+            model = BertForMaskedLM.from_pretrained('bert-base-uncased')
+
+            input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+            outputs = model(input_ids, labels=input_ids)
+
+            loss, prediction_scores = outputs[:2]
+
        """
        if "masked_lm_labels" in kwargs:
            warnings.warn(
@@ -1162,18 +1185,18 @@ class BertForNextSentencePrediction(BertPreTrainedModel):

    Examples::

-        >>> from transformers import BertTokenizer, BertForNextSentencePrediction
-        >>> import torch
+        from transformers import BertTokenizer, BertForNextSentencePrediction
+        import torch

-        >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-        >>> model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased')
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased')

-        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
-        >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
-        >>> encoding = tokenizer(prompt, next_sentence, return_tensors='pt')
+        prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
+        next_sentence = "The sky is blue due to the shorter wavelength of blue light."
+        encoding = tokenizer.encode_plus(prompt, next_sentence, return_tensors='pt')

-        >>> loss, logits = model(**encoding, next_sentence_label=torch.LongTensor([1]))
-        >>> assert logits[0, 0] < logits[0, 1] # next sentence was random
+        loss, logits = model(**encoding, next_sentence_label=torch.LongTensor([1]))
+        assert logits[0, 0] < logits[0, 1] # next sentence was random
        """

        outputs = self.bert(
@@ -1217,7 +1240,6 @@ class BertForSequenceClassification(BertPreTrainedModel):
        self.init_weights()

    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="bert-base-uncased")
    def forward(
        self,
        input_ids=None,
@@ -1254,6 +1276,21 @@ class BertForSequenceClassification(BertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Examples::
+
+        from transformers import BertTokenizer, BertForSequenceClassification
+        import torch
+
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
+
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+
+        loss, logits = outputs[:2]
+
        """

        outputs = self.bert(
@@ -1303,7 +1340,6 @@ class BertForMultipleChoice(BertPreTrainedModel):
        self.init_weights()

    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING.format("(batch_size, num_choices, sequence_length)"))
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="bert-base-uncased")
    def forward(
        self,
        input_ids=None,
@@ -1341,6 +1377,25 @@ class BertForMultipleChoice(BertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Examples::
+
+        from transformers import BertTokenizer, BertForMultipleChoice
+        import torch
+
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertForMultipleChoice.from_pretrained('bert-base-uncased')
+
+        prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
+        choice0 = "It is eaten with a fork and a knife."
+        choice1 = "It is eaten while held in the hand."
+        labels = torch.tensor(0).unsqueeze(0)  # choice0 is correct (according to Wikipedia ;)), batch size 1
+
+        encoding = tokenizer.batch_encode_plus([[prompt, choice0], [prompt, choice1]], return_tensors='pt', pad_to_max_length=True)
+        outputs = model(**{k: v.unsqueeze(0) for k,v in encoding.items()}, labels=labels) # batch size is 1
+
+        # the linear classifier still needs to be trained
+        loss, logits = outputs[:2]
        """
        num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]

@@ -1398,7 +1453,6 @@ class BertForTokenClassification(BertPreTrainedModel):
        self.init_weights()

    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="bert-base-uncased")
    def forward(
        self,
        input_ids=None,
@@ -1433,6 +1487,21 @@ class BertForTokenClassification(BertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Examples::
+
+        from transformers import BertTokenizer, BertForTokenClassification
+        import torch
+
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertForTokenClassification.from_pretrained('bert-base-uncased')
+
+        input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1
+        outputs = model(input_ids, labels=labels)
+
+        loss, scores = outputs[:2]
+
        """

        outputs = self.bert(
@@ -1485,7 +1554,6 @@ class BertForQuestionAnswering(BertPreTrainedModel):
        self.init_weights()

    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
-    @add_code_sample_docstrings(tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="bert-base-uncased")
    def forward(
        self,
        input_ids=None,
@@ -1528,6 +1596,25 @@ class BertForQuestionAnswering(BertPreTrainedModel):

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
+
+    Examples::
+
+        from transformers import BertTokenizer, BertForQuestionAnswering
+        import torch
+
+        tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        model = BertForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')
+
+        question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
+        encoding = tokenizer.encode_plus(question, text)
+        input_ids, token_type_ids = encoding["input_ids"], encoding["token_type_ids"]
+        start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids]))
+
+        all_tokens = tokenizer.convert_ids_to_tokens(input_ids)
+        answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])
+
+        assert answer == "a nice puppet"
+
        """

        outputs = self.bert(
@@ -31,8 +31,6 @@ from .modeling_roberta import (

 logger = logging.getLogger(__name__)

-_TOKENIZER_FOR_DOC = "CamembertTokenizer"
-
 CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
    "camembert-base",
    "Musixmatch/umberto-commoncrawl-cased-v1",
--- a/Show More
+++ b/Show More