fix #5693

Minor doc fix.

.circleci/deploy.sh

@@ -47,4 +47,4 @@ deploy_doc "e7cfc1a" v2.9.0
 deploy_doc "7cb203f" v2.9.1
 deploy_doc "10d7239" v2.10.0 
 deploy_doc "b42586e" v2.11.0
-deploy_doc "b62ca59" #v3.0.0 Latest stable release
+deploy_doc "1158e56" #v3.0.2 Latest stable release

.github/ISSUE_TEMPLATE/question-help.md

@@ -1,6 +1,6 @@
 ---
 name: "❓ Questions & Help"
-about: Post your general questions on Stack Overflow tagged huggingface-transformers
+about: Post your general questions on the Hugging Face forum or Stack Overflow tagged huggingface-transformers
 title: ''
 labels: ''
 assignees: ''
@@ -11,19 +11,17 @@ assignees: ''
 
 <!-- The GitHub issue tracker is primarly intended for bugs, feature requests,
      new models and benchmarks, and migration questions. For all other questions,
-     we direct you to Stack Overflow (SO) where a whole community of PyTorch and
-     Tensorflow enthusiast can help you out. Make sure to tag your question with the
-     right deep learning framework as well as the huggingface-transformers tag: 
+     we direct you to the Hugging Face forum: https://discuss.huggingface.co/ .
+     You can also try Stack Overflow (SO) where a whole community of PyTorch and
+     Tensorflow enthusiast can help you out. In this case, make sure to tag your
+     question with the right deep learning framework as well as the
+     huggingface-transformers tag: 
      https://stackoverflow.com/questions/tagged/huggingface-transformers 
-     
-     If your question wasn't answered after a period of time on Stack Overflow, you
-     can always open a question on GitHub. You should then link to the SO question 
-     that you posted.
      -->
 
 ## Details
 <!-- Description of your issue -->
 
-<!-- You should first ask your question on SO, and only if
+<!-- You should first ask your question on the forum or SO, and only if
      you didn't get an answer ask it here on GitHub. -->
-**A link to original question on Stack Overflow**:
+**A link to original question on Stack Overflow**:

.github/workflows/self-push.yml

@@ -51,11 +51,4 @@ jobs:
         USE_CUDA: yes
       run: |
         source .env/bin/activate
-        python -m pytest -n 2 --dist=loadfile -s ./tests/ | tee output.txt
-    - name: cat output.txt
-      run: cat output.txt
-    - name: Upload output.txt
-      uses: actions/upload-artifact@v1
-      with:
-        name: pytest_output
-        path: output.txt
+        python -m pytest -n 2 --dist=loadfile -s ./tests/

.github/workflows/self-scheduled.yml

@@ -46,11 +46,4 @@ jobs:
         USE_CUDA: yes
       run: |
         source .env/bin/activate
-        python -m pytest -n 1 --dist=loadfile -s ./tests/  | tee output.txt
-    - name: cat output.txt
-      run: cat output.txt
-    - name: Upload output.txt
-      uses: actions/upload-artifact@v1
-      with:
-        name: pytest_output
-        path: output.txt
+        python -m pytest -n 1 --dist=loadfile -s ./tests/

CONTRIBUTING.md

@@ -66,7 +66,7 @@ If you are willing to contribute the model yourself, let us know so we can best
 guide you.
 
 We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them 
-in the [`templates`](https://github.com/huggingface/transformers/templates) folder.
+in the [`templates`](https://github.com/huggingface/transformers/tree/master/templates) folder.
 
 ### Do you want a new feature (that is not a model)?
 
@@ -88,7 +88,7 @@ If your issue is well written we're already 80% of the way there by the time you
 post it.
 
 We have added **templates** to guide you in the process of adding a new example script for training or testing the 
-models in the library. You can find them in the [`templates`](https://github.com/huggingface/transformers/templates) 
+models in the library. You can find them in the [`templates`](https://github.com/huggingface/transformers/tree/master/templates) 
 folder.
 
 ## Start contributing! (Pull Requests)

docs/source/_static/js/custom.js

@@ -1,10 +1,10 @@
 // These two things need to be updated at each release for the version selector.
 // Last stable version
-const stableVersion = "v3.0.0"
+const stableVersion = "v3.0.2"
 // Dictionary doc folder to label
 const versionMapping = {
     "master": "master",
-    "": "v3.0.0 (stable)",
+    "": "v3.0.0/v3.0.1/v3.0.2 (stable)",
     "v2.11.0": "v2.11.0",
     "v2.10.0": "v2.10.0",
     "v2.9.1": "v2.9.0/v2.9.1",
@@ -149,6 +149,7 @@ function addHfMenu() {
     <div class="menu">
         <a href="/welcome">🔥 Sign in</a>
         <a href="/models">🚀 Models</a>
+        <a href="http://discuss.huggingface.co">💬 Forum</a>
     </div>
     `;
     document.body.insertAdjacentHTML('afterbegin', div);

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'3.0.0'
+release = u'3.0.2'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/index.rst

@@ -121,7 +121,10 @@ conversion utilities for the following models:
     trained using `OPUS <http://opus.nlpl.eu/>`_ pretrained_models data by Jörg Tiedemann.
 21. `Longformer <https://github.com/allenai/longformer>`_ (from AllenAI) released with the paper `Longformer: The
     Long-Document Transformer <https://arxiv.org/abs/2004.05150>`_ by Iz Beltagy, Matthew E. Peters, and Arman Cohan.
-22. `Other community models <https://huggingface.co/models>`_, contributed by the `community
+22. `DPR <https://github.com/facebookresearch/DPR>`_ (from Facebook) released with the paper `Dense Passage Retrieval
+    for Open-Domain Question Answering <https://arxiv.org/abs/2004.04906>`_ by Vladimir Karpukhin, Barlas Oğuz, Sewon
+    Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
+23. `Other community models <https://huggingface.co/models>`_, contributed by the `community
     <https://huggingface.co/users>`_.
 
 .. toctree::
@@ -142,6 +145,7 @@ conversion utilities for the following models:
     preprocessing
     training
     model_sharing
+    tokenizer_summary
     multilingual
 
 .. toctree::
@@ -161,6 +165,7 @@ conversion utilities for the following models:
     :caption: Research
 
     bertology
+    perplexity
     benchmarks
 
 .. toctree::
@@ -168,6 +173,7 @@ conversion utilities for the following models:
     :caption: Package Reference
 
     main_classes/configuration
+    main_classes/output
     main_classes/model
     main_classes/tokenizer
     main_classes/pipelines
@@ -198,3 +204,4 @@ conversion utilities for the following models:
     model_doc/longformer
     model_doc/retribert
     model_doc/mobilebert
+    model_doc/dpr

docs/source/main_classes/configuration.rst

@@ -1,7 +1,9 @@
 Configuration
 ----------------------------------------------------
 
-The base class ``PretrainedConfig`` implements the common methods for loading/saving a configuration either from a local file or directory, or from a pretrained model configuration provided by the library (downloaded from HuggingFace's AWS S3 repository).
+The base class ``PretrainedConfig`` implements the common methods for loading/saving a configuration either from a
+local file or directory, or from a pretrained model configuration provided by the library (downloaded from
+HuggingFace's AWS S3 repository).
 
 ``PretrainedConfig``
 ~~~~~~~~~~~~~~~~~~~~~

docs/source/main_classes/output.rst

@@ -0,0 +1,141 @@
+Model outputs
+-------------
+
+PyTorch models have outputs that are instances of subclasses of :class:`~transformers.file_utils.ModelOutput`. Those
+are data structures containing all the information returned by the model, but that can also be used as tuples or
+dictionaries.
+
+Let's see of this looks on an example:
+
+.. code-block::
+
+    from transformers import BertTokenizer, BertForSequenceClassification
+    import torch
+
+    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+    model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
+
+    inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
+    labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+    outputs = model(**inputs, labels=labels)
+
+The ``outputs`` object is a :class:`~transformers.modeling_outputs.SequenceClassifierOutput`, as we can see in the
+documentation of that class below, it means it has an optional ``loss``, a ``logits`` an optional ``hidden_states`` and
+an optional ``attentions`` attribute. Here we have the ``loss`` since we passed along ``labels``, but we don't have
+``hidden_states`` and ``attentions`` because we didn't pass ``output_hidden_states=True`` or
+``output_attentions=True``.
+
+You can access each attribute as you would usually do, and if that attribute has not been returned by the model, you
+will get ``None``. Here for instance ``outputs.loss`` is the loss computed by the model, and ``outputs.attentions`` is
+``None``.
+
+When considering our ``outputs`` object as tuple, it only considers the attributes that don't have ``None`` values.
+Here for instance, it has two elements, ``loss`` then ``logits``, so
+
+.. code-block::
+
+    outputs[:2]
+
+will return the tuple ``(outputs.loss, outputs.logits)`` for instance.
+
+When considering our ``outputs`` object as dictionary, it only considers the attributes that don't have ``None``
+values. Here for instance, it has two keys that are ``loss`` and ``logits``.
+
+We document here the generic model outputs that are used by more than one model type. Specific output types are
+documented on their corresponding model page.
+
+``ModelOutput``
+~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.file_utils.ModelOutput
+    :members:
+
+``BaseModelOutput``
+~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.BaseModelOutput
+    :members:
+
+``BaseModelOutputWithPooling``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.BaseModelOutputWithPooling
+    :members:
+
+``BaseModelOutputWithPast``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.BaseModelOutputWithPast
+    :members:
+
+``Seq2SeqModelOutput``
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.Seq2SeqModelOutput
+    :members:
+
+``CausalLMOutput``
+~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.CausalLMOutput
+    :members:
+
+``CausalLMOutputWithPast``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.CausalLMOutputWithPast
+    :members:
+
+``MaskedLMOutput``
+~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.MaskedLMOutput
+    :members:
+
+``Seq2SeqLMOutput``
+~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.Seq2SeqLMOutput
+    :members:
+
+``NextSentencePredictorOutput``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.NextSentencePredictorOutput
+    :members:
+
+``SequenceClassifierOutput``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.SequenceClassifierOutput
+    :members:
+
+``Seq2SeqSequenceClassifierOutput``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.Seq2SeqSequenceClassifierOutput
+    :members:
+
+``MultipleChoiceModelOutput``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.MultipleChoiceModelOutput
+    :members:
+
+``TokenClassifierOutput``
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.TokenClassifierOutput
+    :members:
+
+``QuestionAnsweringModelOutput``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.QuestionAnsweringModelOutput
+    :members:
+
+``Seq2SeqQuestionAnsweringModelOutput``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.Seq2SeqQuestionAnsweringModelOutput
+    :members:

docs/source/model_doc/albert.rst

@@ -47,6 +47,13 @@ AlbertTokenizer
         create_token_type_ids_from_sequences, save_vocabulary
 
 
+Albert specific outputs
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_albert.AlbertForPretrainingOutput
+    :members:
+
+
 AlbertModel
 ~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/bart.rst

@@ -39,6 +39,18 @@ BartTokenizer
     :members:
 
 
+MBartTokenizer
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MBartTokenizer
+    :members: build_inputs_with_special_tokens, prepare_translation_batch
+
+BartForConditionalGeneration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.BartForConditionalGeneration
+    :members: generate, forward
+
 BartModel
 ~~~~~~~~~~~~~
 
@@ -62,10 +74,3 @@ BartForQuestionAnswering
     :members: forward
 
 
-BartForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.BartForConditionalGeneration
-    :members: generate, forward
-
-

docs/source/model_doc/bert.rst

@@ -59,6 +59,13 @@ BertTokenizerFast
     :members:
 
 
+Bert specific outputs
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_bert.BertForPretrainingOutput
+    :members:
+
+
 BertModel
 ~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/dpr.rst

@@ -0,0 +1,102 @@
+DPR
+----------------------------------------------------
+
+Overview
+~~~~~~~~~~~~~~~~~~~~~
+
+Dense Passage Retrieval (DPR) - is a set of tools and models for state-of-the-art open-domain Q&A research.
+It is based on the following paper:
+
+Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, Wen-tau Yih, Dense Passage Retrieval for Open-Domain Question Answering.
+
+The abstract from the paper is the following:
+
+*Open-domain question answering relies on efficient passage retrieval to select candidate contexts, where traditional
+sparse vector space models, such as TF-IDF or BM25, are the de facto method. In this work, we show that retrieval can
+be practically implemented using dense representations alone, where embeddings are learned from a small number of
+questions and passages by a simple dual-encoder framework. When evaluated on a wide range of open-domain QA datasets,
+our dense retriever outperforms a strong Lucene-BM25 system largely by 9%-19% absolute in terms of top-20 passage
+retrieval accuracy, and helps our end-to-end QA system establish new state-of-the-art on multiple open-domain QA
+benchmarks.*
+
+The original code can be found `here <https://github.com/facebookresearch/DPR>`_.
+
+
+DPRConfig
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRConfig
+    :members:
+
+
+DPRContextEncoderTokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRContextEncoderTokenizer
+    :members:
+
+
+DPRContextEncoderTokenizerFast
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRContextEncoderTokenizerFast
+    :members:
+
+DPRQuestionEncoderTokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRQuestionEncoderTokenizer
+    :members:
+
+
+DPRQuestionEncoderTokenizerFast
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRQuestionEncoderTokenizerFast
+    :members:
+
+DPRReaderTokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRReaderTokenizer
+    :members:
+
+
+DPRReaderTokenizerFast
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRReaderTokenizerFast
+    :members:
+
+
+DPR specific outputs
+~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_dpr.DPRContextEncoderOutput
+    :members:
+
+.. autoclass:: transformers.modeling_dpr.DPRQuestionEncoderOutput
+    :members:
+
+.. autoclass:: transformers.modeling_dpr.DPRReaderOutput
+    :members:
+
+
+DPRContextEncoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRContextEncoder
+    :members:
+
+DPRQuestionEncoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRQuestionEncoder
+    :members:
+
+
+DPRReader
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.DPRReader
+    :members:

docs/source/model_doc/electra.rst

@@ -71,6 +71,13 @@ ElectraTokenizerFast
     :members:
 
 
+Electra specific outputs
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_electra.ElectraForPretrainingOutput
+    :members:
+
+
 ElectraModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/flaubert.rst

@@ -61,6 +61,13 @@ FlaubertForSequenceClassification
     :members:
 
 
+FlaubertForTokenClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaubertForTokenClassification
+    :members:
+
+
 FlaubertForQuestionAnsweringSimple
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -114,4 +121,4 @@ TFFlaubertForQuestionAnsweringSimple
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.TFFlaubertForQuestionAnsweringSimple
-    :members:
+    :members:

docs/source/model_doc/gpt.rst

@@ -71,6 +71,13 @@ OpenAIGPTTokenizerFast
     :members:
 
 
+OpenAI specific outputs
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_openai.OpenAIGPTDoubleHeadsModelOutput
+    :members:
+
+
 OpenAIGPTModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/gpt2.rst

@@ -58,6 +58,13 @@ GPT2TokenizerFast
     :members:
 
 
+GPT2 specific outputs
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_gpt2.GPT2DoubleHeadsModelOutput
+    :members:
+
+
 GPT2Model
 ~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/mobilebert.rst

@@ -56,6 +56,13 @@ MobileBertTokenizerFast
     :members:
 
 
+MobileBert specific outputs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_mobilebert.MobileBertForPretrainingOutput
+    :members:
+
+
 MobileBertModel
 ~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/transformerxl.rst

@@ -54,6 +54,16 @@ TransfoXLTokenizerFast
     :members:
 
 
+TransfoXL specific outputs
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_transfo_xl.TransfoXLModelOutput
+    :members:
+
+.. autoclass:: transformers.modeling_transfo_xl.TransfoXLLMHeadModelOutput
+    :members:
+
+
 TransfoXLModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/xlm.rst

@@ -46,6 +46,14 @@ XLMTokenizer
     :members: build_inputs_with_special_tokens, get_special_tokens_mask,
         create_token_type_ids_from_sequences, save_vocabulary
 
+
+XLM specific outputs
+~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_xlm.XLMForQuestionAnsweringOutput
+    :members:
+
+
 XLMModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/xlnet.rst

@@ -50,6 +50,31 @@ XLNetTokenizer
         create_token_type_ids_from_sequences, save_vocabulary
 
 
+XLNet specific outputs
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_xlnet.XLNetModelOutput
+    :members:
+
+.. autoclass:: transformers.modeling_xlnet.XLNetLMHeadModelOutput
+    :members:
+
+.. autoclass:: transformers.modeling_xlnet.XLNetForSequenceClassificationOutput
+    :members:
+
+.. autoclass:: transformers.modeling_xlnet.XLNetForMultipleChoiceOutput
+    :members:
+
+.. autoclass:: transformers.modeling_xlnet.XLNetForTokenClassificationOutput
+    :members:
+
+.. autoclass:: transformers.modeling_xlnet.XLNetForQuestionAnsweringSimpleOutput
+    :members:
+
+.. autoclass:: transformers.modeling_xlnet.XLNetForQuestionAnsweringOutput
+    :members:
+
+
 XLNetModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_summary.rst

@@ -55,7 +55,7 @@ Original GPT
    <a href="https://huggingface.co/models?filter=openai-gpt">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-openai--gpt-blueviolet">
    </a>
-   <a href="/model_doc/gpt">
+   <a href="model_doc/gpt.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-openai--gpt-blueviolet">
    </a>
 
@@ -75,7 +75,7 @@ GPT-2
    <a href="https://huggingface.co/models?filter=gpt2">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-gpt2-blueviolet">
    </a>
-   <a href="/model_doc/gpt2">
+   <a href="model_doc/gpt2.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-gpt2-blueviolet">
    </a>
 
@@ -96,7 +96,7 @@ CTRL
    <a href="https://huggingface.co/models?filter=ctrl">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-ctrl-blueviolet">
    </a>
-   <a href="/model_doc/ctrl">
+   <a href="model_doc/ctrl.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-ctrl-blueviolet">
    </a>
 
@@ -117,7 +117,7 @@ Transformer-XL
    <a href="https://huggingface.co/models?filter=transfo-xl">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-transfo--xl-blueviolet">
    </a>
-   <a href="/model_doc/transformerxl">
+   <a href="model_doc/transformerxl.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-transfo--xl-blueviolet">
    </a>
 
@@ -148,7 +148,7 @@ Reformer
    <a href="https://huggingface.co/models?filter=reformer">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-reformer-blueviolet">
    </a>
-   <a href="/model_doc/reformer">
+   <a href="model_doc/reformer.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-reformer-blueviolet">
    </a>
 
@@ -183,7 +183,7 @@ XLNet
    <a href="https://huggingface.co/models?filter=xlnet">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-xlnet-blueviolet">
    </a>
-   <a href="/model_doc/xlnet">
+   <a href="model_doc/xlnet.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-xlnet-blueviolet">
    </a>
 
@@ -217,7 +217,7 @@ BERT
    <a href="https://huggingface.co/models?filter=bert">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-bert-blueviolet">
    </a>
-   <a href="/model_doc/bert">
+   <a href="model_doc/bert.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-bert-blueviolet">
    </a>
 
@@ -246,7 +246,7 @@ ALBERT
    <a href="https://huggingface.co/models?filter=albert">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-albert-blueviolet">
    </a>
-   <a href="/model_doc/albert">
+   <a href="model_doc/albert.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-albert-blueviolet">
    </a>
 
@@ -275,7 +275,7 @@ RoBERTa
    <a href="https://huggingface.co/models?filter=roberta">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-roberta-blueviolet">
    </a>
-   <a href="/model_doc/roberta">
+   <a href="model_doc/roberta.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-roberta-blueviolet">
    </a>
 
@@ -301,7 +301,7 @@ DistilBERT
    <a href="https://huggingface.co/models?filter=distilbert">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-distilbert-blueviolet">
    </a>
-   <a href="/model_doc/distilbert">
+   <a href="model_doc/distilbert.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-distilbert-blueviolet">
    </a>
 
@@ -326,7 +326,7 @@ XLM
    <a href="https://huggingface.co/models?filter=xlm">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-xlm-blueviolet">
    </a>
-   <a href="/model_doc/xlm">
+   <a href="model_doc/xlm.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-xlm-blueviolet">
    </a>
 
@@ -361,7 +361,7 @@ XLM-RoBERTa
    <a href="https://huggingface.co/models?filter=xlm-roberta">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-xlm--roberta-blueviolet">
    </a>
-   <a href="/model_doc/xlmroberta">
+   <a href="model_doc/xlmroberta.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-xlm--roberta-blueviolet">
    </a>
 
@@ -383,7 +383,7 @@ FlauBERT
    <a href="https://huggingface.co/models?filter=flaubert">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-flaubert-blueviolet">
    </a>
-   <a href="/model_doc/flaubert">
+   <a href="model_doc/flaubert.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-flaubert-blueviolet">
    </a>
 
@@ -401,7 +401,7 @@ ELECTRA
    <a href="https://huggingface.co/models?filter=electra">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-electra-blueviolet">
    </a>
-   <a href="/model_doc/electra">
+   <a href="model_doc/electra.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-electra-blueviolet">
    </a>
 
@@ -427,7 +427,7 @@ Longformer
    <a href="https://huggingface.co/models?filter=longformer">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-longformer-blueviolet">
    </a>
-   <a href="/model_doc/longformer">
+   <a href="model_doc/longformer.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-longformer-blueviolet">
    </a>
 
@@ -461,7 +461,7 @@ BART
    <a href="https://huggingface.co/models?filter=bart">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-bart-blueviolet">
    </a>
-   <a href="/model_doc/bart">
+   <a href="model_doc/bart.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-bart-blueviolet">
    </a>
 
@@ -488,7 +488,7 @@ MarianMT
    <a href="https://huggingface.co/models?filter=marian">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-marian-blueviolet">
    </a>
-   <a href="/model_doc/marian">
+   <a href="model_doc/marian.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-marian-blueviolet">
    </a>
 
@@ -506,7 +506,7 @@ T5
    <a href="https://huggingface.co/models?filter=t5">
        <img alt="Models" src="https://img.shields.io/badge/All_model_pages-t5-blueviolet">
    </a>
-   <a href="/model_doc/t5">
+   <a href="model_doc/t5.html">
        <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-t5-blueviolet">
    </a>
 
@@ -555,7 +555,7 @@ model know which part of the input vector corresponds to the text or the image.
 The pretrained model only works for classification.
 
 ..
-    More information in this :doc:`model documentation </model_doc/mmbt>`.
+    More information in this :doc:`model documentation </model_doc/mmbt.html>`.
     TODO: write this page
 
 More technical aspects

docs/source/perplexity.rst

@@ -0,0 +1,151 @@
+Perplexity of fixed-length models
+=================================
+
+Perplexity (PPL) is one of the most common metrics for evaluating language
+models. Before diving in, we should note that the metric applies specifically
+to classical language models (sometimes called autoregressive or causal
+language models) and is not well defined for masked language models like BERT
+(see :doc:`summary of the models <model_summary>`).
+
+Perplexity is defined as the exponentiated average log-likelihood of a
+sequence. If we have a tokenized sequence :math:`X = (x_0, x_1, \dots, x_t)`,
+then the perplexity of :math:`X` is,
+
+.. math::
+
+    \text{PPL}(X)
+    = \exp \left\{ {-\frac{1}{t}\sum_i^t \log p_\theta (x_i|x_{<i}) } \right\}
+
+where :math:`\log p_\theta (x_i|x_{<i})` is the log-likelihood of the ith
+token conditioned on the preceding tokens :math:`x_{<i}` according to our
+model. Intuitively, it can be thought of as an evaluation of the model's
+ability to predict uniformly among the set of specified tokens in a corpus.
+Importantly, this means that the tokenization procedure has a direct impact
+on a model's perplexity which should always be taken into consideration when
+comparing different models.
+
+This is also equivalent to the exponentiation of the cross-entropy between
+the data and model predictions. For more intuition about perplexity and its
+relationship to Bits Per Character (BPC) and data compression, check out this
+`fantastic blog post on The Gradient
+<https://thegradient.pub/understanding-evaluation-metrics-for-language-models/>`_.
+
+Calculating PPL with fixed-length models
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If we weren't limited by a model's context size, we would evaluate the
+model's perplexity by autoregressively factorizing a sequence and
+conditioning on the entire preceding subsequence at each step, as shown
+below.
+
+.. image:: imgs/ppl_full.gif
+    :width: 600
+    :alt: Full decomposition of a sequence with unlimited context length
+
+When working with approximate models, however, we typically have a constraint
+on the number of tokens the model can process. The largest version
+of :doc:`GPT-2 <model_doc/gpt2>`, for example, has a fixed length of 1024
+tokens, so we cannot calculate :math:`p_\theta(x_t|x_{<t})` directly when
+:math:`t` is greater than 1024.
+
+Instead, the sequence is typically broken into subsequences equal to the
+model's maximum input size. If a model's max input size is :math:`k`, we
+then approximate the likelihood of a token :math:`x_t` by conditioning only
+on the :math:`k-1` tokens that precede it rather than the entire context.
+When evaluating the model's perplexity of a sequence, a tempting but
+suboptimal approach is to break the sequence into disjoint chunks and
+add up the decomposed log-likelihoods of each segment independently.
+
+.. image:: imgs/ppl_chunked.gif
+    :width: 600
+    :alt: Suboptimal PPL not taking advantage of full available context
+
+This is quick to compute since the perplexity of each segment can be computed
+in one forward pass, but serves as a poor approximation of the
+fully-factorized perplexity and will typically yield a higher (worse) PPL
+because the model will have less context at most of the prediction steps.
+
+Instead, the PPL of fixed-length models should be evaluated with a
+sliding-window strategy. This involves repeatedly sliding the
+context window so that the model has more context when making each
+prediction.
+
+.. image:: imgs/ppl_sliding.gif
+    :width: 600
+    :alt: Sliding window PPL taking advantage of all available context
+
+This is a closer approximation to the true decomposition of the
+sequence probability and will typically yield a more favorable score.
+The downside is that it requires a separate forward pass for each token in
+the corpus. A good practical compromise is to employ a strided sliding
+window, moving the context by larger strides rather than sliding by 1 token a
+time. This allows computation to procede much faster while still giving the
+model a large context to make predictions at each step.
+
+Example: Calculating perplexity with GPT-2 in 🤗 Transformers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Let's demonstrate this process with GPT-2.
+
+.. code-block:: python
+
+    from transformers import GPT2LMHeadModel, GPT2TokenizerFast
+    device = 'cuda'
+    model_id = 'gpt2-large'
+    model = GPT2LMHeadModel.from_pretrained(model_id).to(device)
+    tokenizer = GPT2TokenizerFast.from_pretrained(model_id)
+
+We'll load in the WikiText-2 dataset and evaluate the perplexity using a few
+different sliding-window strategies. Since this dataset is small and we're
+just doing one forward pass over the set, we can just load and encode the
+entire dataset in memory.
+
+.. code-block:: python
+
+    from nlp import load_dataset
+    test = load_dataset('wikitext', 'wikitext-2-raw-v1', split='test')
+    encodings = tokenizer('\n\n'.join(test['text']), return_tensors='pt')
+
+With 🤗 Transformers, we can simply pass the ``input_ids`` as the ``labels``
+to our model, and the average log-likelihood for each token is returned as
+the loss. With our sliding window approach, however, there is overlap in the
+tokens we pass to the model at each iteration. We don't want the
+log-likelihood for the tokens we're just treating as context to be included
+in our loss, so we can set these targets to ``-100`` so that they are
+ignored. The following is an example of how we could do this with a stride of
+``512``. This means that the model will have at least 512 tokens for context
+when calculating the conditional likelihood of any one token (provided there
+are 512 preceding tokens available to condition on).
+
+.. code-block:: python
+
+    max_length = model.config.n_positions
+    stride = 512
+
+    lls = []
+    for i in tqdm(range(1, encodings.input_ids.size(1), stride)):
+        begin_loc = max(i + stride - max_length, 0)
+        end_loc = i + stride
+        input_ids = encodings.input_ids[:,begin_loc:end_loc].to(device)
+        target_ids = input_ids.clone()
+        target_ids[:,:-stride] = -100
+
+        with torch.no_grad():
+            outputs = model(input_ids, labels=target_ids)
+            log_likelihood = outputs[0] * stride
+
+        lls.append(log_likelihood)
+    
+    ppl = torch.exp(torch.stack(lls).sum() / i)
+
+Running this with the stride length equal to the max input length is
+equivalent to the suboptimal, non-sliding-window strategy we discussed above.
+The smaller the stride, the more context the model will have in making each
+prediction, and the better the reported perplexity will typically be.
+
+When we run the above with ``stride = 1024``, i.e. no overlap, the resulting
+PPL is ``19.64``, which is about the same as the ``19.93`` reported in the
+GPT-2 paper. By using ``stride = 512`` and thereby employing our striding
+window strategy, this jumps down to ``16.53``. This is not only a more
+favorable score, but is calculated in a way that is closer to the true
+autoregressive decomposition of a sequence likelihood.

docs/source/quicktour.rst

@@ -108,11 +108,11 @@ any other model from the model hub):
     >>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
     >>> model = AutoModelForSequenceClassification.from_pretrained(model_name)
     >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
-    >>> pipe = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
+    >>> classifier = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
     >>> ## TENSORFLOW CODE
     >>> model_name = "nlptown/bert-base-multilingual-uncased-sentiment"
     >>> # This model only exists in PyTorch, so we use the `from_pt` flag to import that model in TensorFlow.
-    >>> model = TFAutoModelForSequenceClassification.from_pretrained(model_name, from_pt=True) 
+    >>> model = TFAutoModelForSequenceClassification.from_pretrained(model_name, from_pt=True)
     >>> tokenizer = AutoTokenizer.from_pretrained(model_name)
     >>> classifier = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer)
 
@@ -146,8 +146,9 @@ Using the tokenizer
 
 We mentioned the tokenizer is responsible for the preprocessing of your texts. First, it will split a given text in
 words (or part of words, punctuation symbols, etc.) usually called `tokens`. There are multiple rules that can govern
-that process, which is why we need to instantiate the tokenizer using the name of the model, to make sure we use the
-same rules as when the model was pretrained.
+that process (you can learn more about them in the :doc:`tokenizer_summary <tokenizer_summary>`, which is why we need
+to instantiate the tokenizer using the name of the model, to make sure we use the same rules as when the model was
+pretrained.
 
 The second step is to convert those `tokens` into numbers, to be able to build a tensor out of them and feed them to
 the model. To do this, the tokenizer has a `vocab`, which is the part we download when we instantiate it with the
@@ -190,7 +191,7 @@ and get tensors back. You can specify all of that to the tokenizer:
     ...     return_tensors="tf"
     ... )
 
-The padding is automatically applied on the side the model expect it (in this case, on the right), with the
+The padding is automatically applied on the side expected by the model (in this case, on the right), with the
 padding token the model was pretrained with. The attention mask is also adapted to take the padding into account:
 
 .. code-block::
@@ -211,9 +212,9 @@ You can learn more about tokenizers :doc:`here <preprocessing>`.
 Using the model
 ^^^^^^^^^^^^^^^
 
-Once your input has been preprocessed by the tokenizer, you can directly send it to the model. As we mentioned, it will
-contain all the relevant information the model needs. If you're using a TensorFlow model, you can directly pass the
-dictionary keys to tensor, for a PyTorch model, you need to unpack the dictionary by adding :obj:`**`.
+Once your input has been preprocessed by the tokenizer, you can send it directly to the model. As we mentioned, it will
+contain all the relevant information the model needs. If you're using a TensorFlow model, you can pass the
+dictionary keys directly to tensor, for a PyTorch model, you need to unpack the dictionary by adding :obj:`**`.
 
 .. code-block::
 
@@ -284,7 +285,7 @@ training loop. 🤗 Transformers also provides a :class:`~transformers.Trainer`
 you are using TensorFlow) class to help with your training (taking care of things such as distributed training, mixed
 precision, etc.). See the :doc:`training tutorial <training>` for more details.
 
-Once your model is fine-tuned, you can save it with its tokenizer the following way:
+Once your model is fine-tuned, you can save it with its tokenizer in the following way:
 
 ::
 
@@ -328,7 +329,9 @@ pretrained model. Behind the scenes, the library has one model class per combina
 code is easy to access and tweak if you need to.
 
 In our previous example, the model was called "distilbert-base-uncased-finetuned-sst-2-english", which means it's
-using the :doc:`DistilBERT </model_doc/distilbert>` architecture. The model automatically created is then a
+using the :doc:`DistilBERT </model_doc/distilbert>` architecture. As
+:class:`~transformers.AutoModelForSequenceClassification` (or  :class:`~transformers.TFAutoModelForSequenceClassification`
+if you are using TensorFlow)` was used, the model automatically created is then a
 :class:`~transformers.DistilBertForSequenceClassification`. You can look at its documentation for all details relevant
 to that specific model, or browse the source code. This is how you would directly instantiate model and tokenizer
 without the auto magic:
@@ -351,7 +354,7 @@ Customizing the model
 
 If you want to change how the model itself is built, you can define your custom configuration class. Each architecture
 comes with its own relevant configuration (in the case of DistilBERT, :class:`~transformers.DistilBertConfig`) which
-allows you to specify any of the hidden dimension, dropout rate etc. If you do core modifications, like changing the
+allows you to specify any of the hidden dimension, dropout rate, etc. If you do core modifications, like changing the
 hidden size, you won't be able to use a pretrained model anymore and will need to train from scratch. You would then
 instantiate the model directly from this configuration.
 

docs/source/tokenizer_summary.rst

@@ -0,0 +1,243 @@
+Tokenizer summary
+-----------------
+
+In this page, we will have a closer look at tokenization. As we saw in
+:doc:`the preprocessing tutorial <preprocessing>`, tokenizing a text is splitting it into words or subwords, which then
+are converted to ids. The second part is pretty straightforward, here we will focus on the first part. More
+specifically, we will look at the three main different kinds of tokenizers used in 🤗 Transformers:
+:ref:`Byte-Pair Encoding (BPE) <byte-pair-encoding>`, :ref:`WordPiece <wordpiece>` and
+:ref:`SentencePiece <sentencepiece>`, and provide examples of models using each of those.
+
+Note that on each model page, you can look at the documentation of the associated tokenizer to know which of those
+algorithms the pretrained model used. For instance, if we look at :class:`~transformers.BertTokenizer`, we can see it's
+using :ref:`WordPiece <wordpiece>`.
+
+Introduction to tokenization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Splitting a text in smaller chunks is a task that's harder than it looks, and there are multiple ways of doing it. For
+instance, let's look at the sentence "Don't you love 🤗 Transformers? We sure do." A first simple way of tokenizing
+this text is just to split it by spaces, which would give:
+
+::
+
+    ["Don't", "you", "love", "🤗", "Transformers?", "We", "sure", "do."]
+
+This is a nice first step, but if we look at the tokens "Transformers?" or "do.", we can see we can do better. Those
+will be different than the tokens "Transformers" and "do" for our model, so we should probably take the punctuation
+into account. This would give:
+
+::
+
+    ["Don", "'", "t", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]
+
+which is better already. One thing that is annoying though is how it dealt with "Don't". "Don't" stands for do not, so
+it should probably be better tokenized as ``["Do", "n't"]``. This is where things start getting more complicated, and
+part of the reason each kind of model has its own tokenizer class. Depending on the rules we apply to split our texts
+into tokens, we'll get different tokenized versions of the same text. And of course, a given pretrained model won't
+perform properly if you don't use the exact same rules as the persons who pretrained it.
+
+`spaCy <https://spacy.io/>`__ and `Moses <http://www.statmt.org/moses/?n=Development.GetStarted>`__ are two popular
+rule-based tokenizers. On the text above, they'd output something like:
+
+::
+
+    ["Do", "n't", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]
+
+Space/punctuation-tokenization and rule-based tokenization are both examples of word tokenization, which is splitting a
+sentence into words. While it's the most intuitive way to separate texts in smaller chunks, it can have a problem when
+you have a huge corpus: it usually yields a very big vocabulary (the set of all unique tokens used).
+:doc:`Transformer XL <model_doc/transformerxl>` for instance uses space/punctuation-tokenization, and has a vocabulary
+size of 267,735!
+
+A huge vocabulary size means a huge embedding matrix at the start of the model, which will cause memory problems.
+TransformerXL deals with it by using a special kind of embeddings called adaptive embeddings, but in general,
+transformers models rarely have a vocabulary size greater than 50,000, especially if they are trained on a single
+language.
+
+So if tokenizing on words is unsatisfactory, we could go on the opposite direction and simply tokenize on characters.
+While it's very simple and would save a lot of memory, this doesn't allow the model to learn representations of texts
+as meaningful as when using a word tokenization, leading to a loss of performance. So to get the best of both worlds,
+all transformers models use a hybrid between word-level and character-level tokenization called subword tokenization.
+
+Subword tokenization
+^^^^^^^^^^^^^^^^^^^^
+
+Subword tokenization algorithms rely on the principle that most common words should be left as is, but rare words
+should be decomposed in meaningful subword units. For instance "annoyingly" might be considered a rare word and
+decomposed as "annoying" and "ly". This is especially useful in agglutinative languages such as Turkish, where you can
+form (almost) arbitrarily long complex words by stringing together some subwords.
+
+This allows the model to keep a reasonable vocabulary while still learning useful representations for common words or
+subwords. This also enables the model to process words it has never seen before, by decomposing them into
+subwords it knows. For instance, the base :class:`~transformers.BertTokenizer` will tokenize "I have a new GPU!" like
+this:
+
+::
+
+    >>> from transformers import BertTokenizer
+    >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+    >>> tokenizer.tokenize("I have a new GPU!")
+    ['i', 'have', 'a', 'new', 'gp', '##u', '!']
+
+Since we are considering the uncased model, the sentence was lowercased first. Then all the words were present in the
+vocabulary of the tokenizer, except for "gpu", so the tokenizer split it in subwords it knows: "gp" and "##u". The "##"
+means that the rest of the token should be attached to the previous one, without space (for when we need to decode
+predictions and reverse the tokenization).
+
+Another example is when we use the base :class:`~transformers.XLNetTokenizer` to tokenize our previous text:
+
+::
+
+    >>> from transformers import XLNetTokenizer
+    >>> tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
+    >>> tokenizer.tokenize("Don't you love 🤗 Transformers? We sure do.")
+    ['▁Don', "'", 't', '▁you', '▁love', '▁', '🤗', '▁', 'Transform', 'ers', '?', '▁We', '▁sure', '▁do', '.']
+
+We'll get back to the meaning of those '▁' when we look at :ref:`SentencePiece <sentencepiece>` but you can see
+Transformers has been split into "Transform" and "ers".
+
+Let's now look at how the different subword tokenization algorithms work. Note that they all rely on some form of
+training which is usually done on the corpus the corresponding model will be trained on.
+
+.. _byte-pair-encoding:
+
+Byte-Pair Encoding
+~~~~~~~~~~~~~~~~~~
+
+Byte-Pair Encoding was introduced in `this paper <https://arxiv.org/abs/1508.07909>`__. It relies on a pretokenizer
+splitting the training data into words, which can be a simple space tokenization
+(:doc:`GPT-2 <model_doc/gpt2>` and :doc:`Roberta <model_doc/roberta>` uses this for instance) or a rule-based tokenizer
+(:doc:`XLM <model_doc/xlm>` use Moses for most languages, as does :doc:`FlauBERT <model_doc/flaubert>`),
+
+:doc:`GPT <model_doc/gpt>` uses Spacy and ftfy, and counts the frequency of each word in the training corpus.
+
+It then begins from the list of all characters, and will learn merge rules to form a new token from two symbols in the
+vocabulary until it has learned a vocabulary of the desired size (this is a hyperparameter to pick).
+
+Let's say that after the pre-tokenization we have the following words (the number indicating the frequency of each
+word):
+
+::
+
+    ('hug', 10), ('pug', 5), ('pun', 12), ('bun', 4), ('hugs', 5)
+
+Then the base vocabulary is ['b', 'g', 'h', 'n', 'p', 's', 'u'] and all our words are first split by character:
+
+::
+
+    ('h' 'u' 'g', 10), ('p' 'u' 'g', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'u' 'g' 's', 5)
+
+We then take each pair of symbols and look at the most frequent. For instance 'hu' is present `10 + 5 = 15` times (10
+times in the 10 occurrences of 'hug', 5 times in the 5 occurrences of 'hugs'). The most frequent here is 'ug', present
+`10 + 5 + 2 + 5 = 22` times in total. So the first merge rule the tokenizer learns is to group all 'u' and 'g' together
+then it adds 'ug' to the vocabulary. Our corpus then becomes
+
+::
+
+    ('h' 'ug', 10), ('p' 'ug', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'ug' 's', 5)
+
+and we continue by looking at the next most common pair of symbols. It's 'un', present 16 times, so we merge those two
+and add 'un' to the vocabulary. Then it's 'hug' (as 'h' + 'ug'), present 15 times, so we merge those two and add 'hug'
+to the vocabulary.
+
+At this stage, the vocabulary is ``['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']`` and our corpus is
+represented as
+
+::
+
+    ('hug', 10), ('p' 'ug', 5), ('p' 'un', 12), ('b' 'un', 4), ('hug' 's', 5)
+
+If we stop there, the tokenizer can apply the rules it learned to new words (as long as they don't contain characters that
+were not in the base vocabulary). For instance 'bug' would be tokenized as ``['b', 'ug']`` but mug would be tokenized as
+``['<unk>', 'ug']`` since the 'm' is not in the base vocabulary. This doesn't happen to letters in general (since the
+base corpus uses all of them), but to special characters like emojis.
+
+As we said before, the vocabulary size (which is the base vocabulary size + the number of merges) is a hyperparameter
+to choose. For instance :doc:`GPT <model_doc/gpt>` has a vocabulary size of 40,478 since they have 478 base characters
+and chose to stop the training of the tokenizer at 40,000 merges.
+
+Byte-level BPE
+^^^^^^^^^^^^^^
+
+To deal with the fact the base vocabulary needs to get all base characters, which can be quite big if one allows for
+all unicode characters, the
+`GPT-2 paper <https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf>`__
+introduces a clever trick, which is to use bytes as the base vocabulary (which gives a size of 256). With some
+additional rules to deal with punctuation, this manages to be able to tokenize every text without needing an unknown
+token. For instance, the :doc:`GPT-2 model <model_doc/gpt>` has a vocabulary size of 50,257, which corresponds to the
+256 bytes base tokens, a special end-of-text token and the symbols learned with 50,000 merges.
+
+.. _wordpiece:
+
+WordPiece
+=========
+
+WordPiece is the subword tokenization algorithm used for :doc:`BERT <model_doc/bert>` (as well as
+:doc:`DistilBERT <model_doc/distilbert>` and :doc:`Electra <model_doc/electra>`) and was outlined in
+`this paper <https://static.googleusercontent.com/media/research.google.com/ja//pubs/archive/37842.pdf>`__. It relies
+on the same base as BPE, which is to initialize the vocabulary to every character present in the corpus and
+progressively learn a given number of merge rules, the difference is that it doesn't choose the pair that is the most
+frequent but the one that will maximize the likelihood on the corpus once merged.
+
+What does this mean? Well, in the previous example, it means we would only merge 'u' and 'g' if the probability of
+having 'ug' divided by the probability of having 'u' then 'g' is greater than for any other pair of symbols. It's
+subtly different from what BPE does in the sense that it evaluates what it "loses" by merging two symbols and makes
+sure it's `worth it`.
+
+.. _unigram:
+
+Unigram
+=======
+
+Unigram is a subword tokenization algorithm introduced in `this paper <https://arxiv.org/pdf/1804.10959.pdf>`__.
+Instead of starting with a group of base symbols and learning merges with some rule, like BPE or WordPiece, it starts
+from a large vocabulary (for instance, all pretokenized words and the most common substrings) that it will trim down
+progressively. It's not used directly for any of the pretrained models in the library, but it's used in conjunction
+with :ref:`SentencePiece <sentencepiece>`.
+
+More specifically, at a given step, unigram computes a loss from the corpus we have and the current vocabulary, then,
+for each subword, evaluate how much the loss would augment if the subword was removed from the vocabulary. It then
+sorts the subwords by this quantity (that represents how worse the loss becomes if the token is removed) and removes
+all the worst p tokens (for instance p could be 10% or 20%). It then repeats the process until the vocabulary has
+reached the desired size, always keeping the base characters (to be able to tokenize any word written with them, like
+BPE or WordPiece).
+
+Contrary to BPE and WordPiece that work out rules in a certain order that you can then apply in the same order when
+tokenizing new text, Unigram will have several ways of tokenizing a new text. For instance, if it ends up with the
+vocabulary
+
+::
+
+    ['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']
+
+we had before, it could tokenize "hugs" as ``['hug', 's']``, ``['h', 'ug', 's']`` or ``['h', 'u', 'g', 's']``. So which
+one choose? On top of saving the vocabulary, the trained tokenizer will save the probability of each token in the
+training corpus. You can then give a probability to each tokenization (which is the product of the probabilities of the
+tokens forming it) and pick the most likely one (or if you want to apply some data augmentation, you could sample one
+of the tokenization according to their probabilities).
+
+Those probabilities define the loss that trains the tokenizer: if our corpus consists of the
+words :math:`x_{1}, \dots, x_{N}` and if for the word :math:`x_{i}` we note :math:`S(x_{i})` the set of all possible
+tokenizations of :math:`x_{i}` (with the current vocabulary), then the loss is defined as
+
+.. math::
+    \mathcal{L} = -\sum_{i=1}^{N} \log \left ( \sum_{x \in S(x_{i})} p(x) \right )
+
+.. _sentencepiece:
+
+SentencePiece
+=============
+
+All the methods we have been looking at so far required some form of pretokenization, which has a central problem: not
+all languages use spaces to separate words. This is a problem :doc:`XLM <model_doc/xlm>` solves by using specific
+pretokenizers for each of those languages (in this case, Chinese, Japanese and Thai). To solve this problem,
+SentencePiece (introduced in `this paper <https://arxiv.org/pdf/1808.06226.pdf>`__) treats the input as a raw stream,
+includes the space in the set of characters to use, then uses BPE or unigram to construct the appropriate vocabulary.
+
+That's why in the example we saw before using :class:`~transformers.XLNetTokenizer` (which uses SentencePiece), we had
+the '▁' character, that represents space. Decoding a tokenized text is then super easy: we just have to concatenate
+all of them together and replace '▁' with space.
+
+All transformers models in the library that use SentencePiece use it with unigram. Examples of models using it are
+:doc:`ALBERT <model_doc/albert>`, :doc:`XLNet <model_doc/xlnet>` or the :doc:`Marian framework <model_doc/marian>`.

docs/source/training.rst

@@ -39,7 +39,7 @@ of the specified model are used to initialize the model. The
 library also includes a number of task-specific final layers or 'heads' whose
 weights are instantiated randomly when not present in the specified
 pre-trained model. For example, instantiating a model with
-``BertForSequenceClassification.from_pretrained('bert-base-uncased', num_classes=2)``
+``BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)``
 will create a BERT model instance with encoder weights copied from the
 ``bert-base-uncased`` model and a randomly initialized sequence
 classification head on top of the encoder with an output size of 2. Models
@@ -272,7 +272,7 @@ optimize.
 :func:`~transformers.Trainer` uses a built-in default function to collate
 batches and prepare them to be fed into the model. If needed, you can also
 use the ``data_collator`` argument to pass your own collator function which
-takes in the data in the format provides by your dataset and returns a
+takes in the data in the format provided by your dataset and returns a
 batch ready to be fed into the model. Note that
 :func:`~transformers.TFTrainer` expects the passed datasets to be dataset
 objects from ``tensorflow_datasets``.

examples/README.md

@@ -21,7 +21,7 @@ This is still a work-in-progress – in particular documentation is still sparse
 | [**`text-classification`**](https://github.com/huggingface/transformers/tree/master/examples/text-classification)   | GLUE, XNLI      | ✅ | ✅ | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/trainer/01_text_classification.ipynb)
 | [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER       | ✅ | ✅ | ✅ | -
 | [**`multiple-choice`**](https://github.com/huggingface/transformers/tree/master/examples/multiple-choice)           | SWAG, RACE, ARC | ✅ | ✅ | -  | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ViktorAlm/notebooks/blob/master/MPC_GPU_Demo_for_TF_and_PT.ipynb)
-| [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering)     | SQuAD           | -  | ✅ | -  | -
+| [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering)     | SQuAD           | ✅ | ✅ | -  | -
 | [**`text-generation`**](https://github.com/huggingface/transformers/tree/master/examples/text-generation)           | -               | n/a | n/a | n/a | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)
 | [**`distillation`**](https://github.com/huggingface/transformers/tree/master/examples/distillation)       | All               | -  | -  | -  | -
 | [**`summarization`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq)     | CNN/Daily Mail    | -  | -  | ✅  | -

examples/benchmarking/README.md

@@ -0,0 +1,10 @@
+# 🤗 Benchmark results
+
+Here, you can find a list of the different benchmark results created by the community.
+
+If you would like to list benchmark results on your favorite models of the [model hub](https://huggingface.co/models) here, please open a Pull Request and add it below.
+
+| Benchmark description | Results | Environment info |      Author      |
+|:----------|:-------------|:-------------|------:|
+| PyTorch Benchmark on inference for `bert-base-cased` |[memory](https://github.com/patrickvonplaten/files_to_link_to/blob/master/bert_benchmark/inference_memory.csv) | [env](https://github.com/patrickvonplaten/files_to_link_to/blob/master/bert_benchmark/env.csv) | [Partick von Platen](https://github.com/patrickvonplaten) | 
+| PyTorch Benchmark on inference for `bert-base-cased` |[time](https://github.com/patrickvonplaten/files_to_link_to/blob/master/bert_benchmark/inference_time.csv) | [env](https://github.com/patrickvonplaten/files_to_link_to/blob/master/bert_benchmark/env.csv) | [Partick von Platen](https://github.com/patrickvonplaten) | 

examples/deebert/README.md

@@ -0,0 +1,54 @@
+# DeeBERT: Early Exiting for *BERT
+
+This is the code base for the paper [DeeBERT: Dynamic Early Exiting for Accelerating BERT Inference](https://www.aclweb.org/anthology/2020.acl-main.204/), modified from its [original code base](https://github.com/castorini/deebert).
+
+The original code base also has information for downloading sample models that we have trained in advance.
+
+## Usage
+
+There are three scripts in the folder which can be run directly.
+
+In each script, there are several things to modify before running:
+
+* `PATH_TO_DATA`: path to the GLUE dataset.
+* `--output_dir`: path for saving fine-tuned models. Default: `./saved_models`.
+* `--plot_data_dir`: path for saving evaluation results. Default: `./results`. Results are printed to stdout and also saved to `npy` files in this directory to facilitate plotting figures and further analyses.
+* `MODEL_TYPE`: bert or roberta
+* `MODEL_SIZE`: base or large
+* `DATASET`: SST-2, MRPC, RTE, QNLI, QQP, or MNLI
+
+#### train_deebert.sh
+
+This is for fine-tuning DeeBERT models.
+
+#### eval_deebert.sh
+
+This is for evaluating each exit layer for fine-tuned DeeBERT models.
+
+#### entropy_eval.sh
+
+This is for evaluating fine-tuned DeeBERT models, given a number of different early exit entropy thresholds.
+
+
+
+## Citation
+
+Please cite our paper if you find the resource useful:
+```
+@inproceedings{xin-etal-2020-deebert,
+    title = "{D}ee{BERT}: Dynamic Early Exiting for Accelerating {BERT} Inference",
+    author = "Xin, Ji  and
+      Tang, Raphael  and
+      Lee, Jaejun  and
+      Yu, Yaoliang  and
+      Lin, Jimmy",
+    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
+    month = jul,
+    year = "2020",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/2020.acl-main.204",
+    pages = "2246--2251",
+}
+```
+

examples/deebert/entropy_eval.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+export CUDA_VISIBLE_DEVICES=0
+
+PATH_TO_DATA=/h/xinji/projects/GLUE
+
+MODEL_TYPE=bert  # bert or roberta
+MODEL_SIZE=base  # base or large
+DATASET=MRPC  # SST-2, MRPC, RTE, QNLI, QQP, or MNLI
+
+MODEL_NAME=${MODEL_TYPE}-${MODEL_SIZE}
+if [ $MODEL_TYPE = 'bert' ]
+then
+  MODEL_NAME=${MODEL_NAME}-uncased
+fi
+
+ENTROPIES="0 0.1 0.2 0.3 0.4 0.5 0.6 0.7"
+
+for ENTROPY in $ENTROPIES; do
+  python -u run_glue_deebert.py \
+    --model_type $MODEL_TYPE \
+    --model_name_or_path ./saved_models/${MODEL_TYPE}-${MODEL_SIZE}/$DATASET/two_stage \
+    --task_name $DATASET \
+    --do_eval \
+    --do_lower_case \
+    --data_dir $PATH_TO_DATA/$DATASET \
+    --output_dir ./saved_models/${MODEL_TYPE}-${MODEL_SIZE}/$DATASET/two_stage \
+    --plot_data_dir ./results/ \
+    --max_seq_length 128 \
+    --early_exit_entropy $ENTROPY \
+    --eval_highway \
+    --overwrite_cache \
+    --per_gpu_eval_batch_size=1
+done

examples/deebert/eval_deebert.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+export CUDA_VISIBLE_DEVICES=0
+
+PATH_TO_DATA=/h/xinji/projects/GLUE
+
+MODEL_TYPE=bert  # bert or roberta
+MODEL_SIZE=base  # base or large
+DATASET=MRPC  # SST-2, MRPC, RTE, QNLI, QQP, or MNLI
+
+MODEL_NAME=${MODEL_TYPE}-${MODEL_SIZE}
+if [ $MODEL_TYPE = 'bert' ]
+then
+  MODEL_NAME=${MODEL_NAME}-uncased
+fi
+
+
+python -u run_glue_deebert.py  \
+  --model_type $MODEL_TYPE \
+  --model_name_or_path ./saved_models/${MODEL_TYPE}-${MODEL_SIZE}/$DATASET/two_stage \
+  --task_name $DATASET \
+  --do_eval \
+  --do_lower_case \
+  --data_dir $PATH_TO_DATA/$DATASET \
+  --output_dir ./saved_models/${MODEL_TYPE}-${MODEL_SIZE}/$DATASET/two_stage \
+  --plot_data_dir ./results/ \
+  --max_seq_length 128 \
+  --eval_each_highway \
+  --eval_highway \
+  --overwrite_cache \
+  --per_gpu_eval_batch_size=1

examples/deebert/run_glue_deebert.py

@@ -0,0 +1,720 @@
+from __future__ import absolute_import, division, print_function
+
+import argparse
+import glob
+import logging
+import os
+import random
+import time
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
+from torch.utils.data.distributed import DistributedSampler
+from tqdm import tqdm, trange
+
+from src.modeling_highway_bert import DeeBertForSequenceClassification
+from src.modeling_highway_roberta import DeeRobertaForSequenceClassification
+from transformers import (
+    WEIGHTS_NAME,
+    AdamW,
+    BertConfig,
+    BertTokenizer,
+    RobertaConfig,
+    RobertaTokenizer,
+    get_linear_schedule_with_warmup,
+)
+from transformers import glue_compute_metrics as compute_metrics
+from transformers import glue_convert_examples_to_features as convert_examples_to_features
+from transformers import glue_output_modes as output_modes
+from transformers import glue_processors as processors
+
+
+try:
+    from torch.utils.tensorboard import SummaryWriter
+except ImportError:
+    from tensorboardX import SummaryWriter
+
+
+logger = logging.getLogger(__name__)
+
+
+MODEL_CLASSES = {
+    "bert": (BertConfig, DeeBertForSequenceClassification, BertTokenizer),
+    "roberta": (RobertaConfig, DeeRobertaForSequenceClassification, RobertaTokenizer),
+}
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def get_wanted_result(result):
+    if "spearmanr" in result:
+        print_result = result["spearmanr"]
+    elif "f1" in result:
+        print_result = result["f1"]
+    elif "mcc" in result:
+        print_result = result["mcc"]
+    elif "acc" in result:
+        print_result = result["acc"]
+    else:
+        raise ValueError("Primary metric unclear in the results")
+    return print_result
+
+
+def train(args, train_dataset, model, tokenizer, train_highway=False):
+    """ Train the model """
+    if args.local_rank in [-1, 0]:
+        tb_writer = SummaryWriter()
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
+    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
+
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+    else:
+        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
+
+    # Prepare optimizer and schedule (linear warmup and decay)
+    no_decay = ["bias", "LayerNorm.weight"]
+    if train_highway:
+        optimizer_grouped_parameters = [
+            {
+                "params": [
+                    p
+                    for n, p in model.named_parameters()
+                    if ("highway" in n) and (not any(nd in n for nd in no_decay))
+                ],
+                "weight_decay": args.weight_decay,
+            },
+            {
+                "params": [
+                    p for n, p in model.named_parameters() if ("highway" in n) and (any(nd in n for nd in no_decay))
+                ],
+                "weight_decay": 0.0,
+            },
+        ]
+    else:
+        optimizer_grouped_parameters = [
+            {
+                "params": [
+                    p
+                    for n, p in model.named_parameters()
+                    if ("highway" not in n) and (not any(nd in n for nd in no_decay))
+                ],
+                "weight_decay": args.weight_decay,
+            },
+            {
+                "params": [
+                    p
+                    for n, p in model.named_parameters()
+                    if ("highway" not in n) and (any(nd in n for nd in no_decay))
+                ],
+                "weight_decay": 0.0,
+            },
+        ]
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
+    scheduler = get_linear_schedule_with_warmup(
+        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total
+    )
+    if args.fp16:
+        try:
+            from apex import amp
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
+
+    # multi-gpu training (should be after apex fp16 initialization)
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
+    # Distributed training (should be after apex fp16 initialization)
+    if args.local_rank != -1:
+        model = torch.nn.parallel.DistributedDataParallel(
+            model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True
+        )
+
+    # Train!
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info(
+        "  Total train batch size (w. parallel, distributed & accumulation) = %d",
+        args.train_batch_size
+        * args.gradient_accumulation_steps
+        * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),
+    )
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
+    set_seed(args)  # Added here for reproductibility (even between python 2 and 3)
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
+        for step, batch in enumerate(epoch_iterator):
+            model.train()
+            batch = tuple(t.to(args.device) for t in batch)
+            inputs = {"input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3]}
+            if args.model_type != "distilbert":
+                inputs["token_type_ids"] = (
+                    batch[2] if args.model_type in ["bert", "xlnet"] else None
+                )  # XLM, DistilBERT and RoBERTa don't use segment_ids
+            inputs["train_highway"] = train_highway
+            outputs = model(**inputs)
+            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)
+
+            if args.n_gpu > 1:
+                loss = loss.mean()  # mean() to average on multi-gpu parallel training
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+
+            if args.fp16:
+                with amp.scale_loss(loss, optimizer) as scaled_loss:
+                    scaled_loss.backward()
+            else:
+                loss.backward()
+
+            tr_loss += loss.item()
+            if (step + 1) % args.gradient_accumulation_steps == 0:
+                if args.fp16:
+                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+                else:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
+                optimizer.step()
+                scheduler.step()  # Update learning rate schedule
+                model.zero_grad()
+                global_step += 1
+
+                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
+                    # Log metrics
+                    if (
+                        args.local_rank == -1 and args.evaluate_during_training
+                    ):  # Only evaluate when single GPU otherwise metrics may not average well
+                        results = evaluate(args, model, tokenizer)
+                        for key, value in results.items():
+                            tb_writer.add_scalar("eval_{}".format(key), value, global_step)
+                    tb_writer.add_scalar("lr", scheduler.get_lr()[0], global_step)
+                    tb_writer.add_scalar("loss", (tr_loss - logging_loss) / args.logging_steps, global_step)
+                    logging_loss = tr_loss
+
+                if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
+                    # Save model checkpoint
+                    output_dir = os.path.join(args.output_dir, "checkpoint-{}".format(global_step))
+                    if not os.path.exists(output_dir):
+                        os.makedirs(output_dir)
+                    model_to_save = (
+                        model.module if hasattr(model, "module") else model
+                    )  # Take care of distributed/parallel training
+                    model_to_save.save_pretrained(output_dir)
+                    torch.save(args, os.path.join(output_dir, "training_args.bin"))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+
+    if args.local_rank in [-1, 0]:
+        tb_writer.close()
+
+    return global_step, tr_loss / global_step
+
+
+def evaluate(args, model, tokenizer, prefix="", output_layer=-1, eval_highway=False):
+    # Loop to handle MNLI double evaluation (matched, mis-matched)
+    eval_task_names = ("mnli", "mnli-mm") if args.task_name == "mnli" else (args.task_name,)
+    eval_outputs_dirs = (args.output_dir, args.output_dir + "-MM") if args.task_name == "mnli" else (args.output_dir,)
+
+    results = {}
+    for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs):
+        eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True)
+
+        if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(eval_output_dir)
+
+        args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+        # Note that DistributedSampler samples randomly
+        eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
+        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
+
+        # multi-gpu eval
+        if args.n_gpu > 1:
+            model = torch.nn.DataParallel(model)
+
+        # Eval!
+        logger.info("***** Running evaluation {} *****".format(prefix))
+        logger.info("  Num examples = %d", len(eval_dataset))
+        logger.info("  Batch size = %d", args.eval_batch_size)
+        eval_loss = 0.0
+        nb_eval_steps = 0
+        preds = None
+        out_label_ids = None
+        exit_layer_counter = {(i + 1): 0 for i in range(model.num_layers)}
+        st = time.time()
+        for batch in tqdm(eval_dataloader, desc="Evaluating"):
+            model.eval()
+            batch = tuple(t.to(args.device) for t in batch)
+
+            with torch.no_grad():
+                inputs = {"input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3]}
+                if args.model_type != "distilbert":
+                    inputs["token_type_ids"] = (
+                        batch[2] if args.model_type in ["bert", "xlnet"] else None
+                    )  # XLM, DistilBERT and RoBERTa don't use segment_ids
+                if output_layer >= 0:
+                    inputs["output_layer"] = output_layer
+                outputs = model(**inputs)
+                if eval_highway:
+                    exit_layer_counter[outputs[-1]] += 1
+                tmp_eval_loss, logits = outputs[:2]
+
+                eval_loss += tmp_eval_loss.mean().item()
+            nb_eval_steps += 1
+            if preds is None:
+                preds = logits.detach().cpu().numpy()
+                out_label_ids = inputs["labels"].detach().cpu().numpy()
+            else:
+                preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
+                out_label_ids = np.append(out_label_ids, inputs["labels"].detach().cpu().numpy(), axis=0)
+        eval_time = time.time() - st
+        logger.info("Eval time: {}".format(eval_time))
+
+        eval_loss = eval_loss / nb_eval_steps
+        if args.output_mode == "classification":
+            preds = np.argmax(preds, axis=1)
+        elif args.output_mode == "regression":
+            preds = np.squeeze(preds)
+        result = compute_metrics(eval_task, preds, out_label_ids)
+        results.update(result)
+
+        if eval_highway:
+            logger.info("Exit layer counter: {}".format(exit_layer_counter))
+            actual_cost = sum([l * c for l, c in exit_layer_counter.items()])
+            full_cost = len(eval_dataloader) * model.num_layers
+            logger.info("Expected saving: {}".format(actual_cost / full_cost))
+            if args.early_exit_entropy >= 0:
+                save_fname = (
+                    args.plot_data_dir
+                    + "/"
+                    + args.model_name_or_path[2:]
+                    + "/entropy_{}.npy".format(args.early_exit_entropy)
+                )
+                if not os.path.exists(os.path.dirname(save_fname)):
+                    os.makedirs(os.path.dirname(save_fname))
+                print_result = get_wanted_result(result)
+                np.save(save_fname, np.array([exit_layer_counter, eval_time, actual_cost / full_cost, print_result]))
+                logger.info("Entropy={}\tResult={:.2f}".format(args.early_exit_entropy, 100 * print_result))
+
+        output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt")
+        with open(output_eval_file, "w") as writer:
+            logger.info("***** Eval results {} *****".format(prefix))
+            for key in sorted(result.keys()):
+                logger.info("  %s = %s", key, str(result[key]))
+                writer.write("%s = %s\n" % (key, str(result[key])))
+
+    return results
+
+
+def load_and_cache_examples(args, task, tokenizer, evaluate=False):
+    if args.local_rank not in [-1, 0] and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    processor = processors[task]()
+    output_mode = output_modes[task]
+    # Load data features from cache or dataset file
+    cached_features_file = os.path.join(
+        args.data_dir,
+        "cached_{}_{}_{}_{}".format(
+            "dev" if evaluate else "train",
+            list(filter(None, args.model_name_or_path.split("/"))).pop(),
+            str(args.max_seq_length),
+            str(task),
+        ),
+    )
+    if os.path.exists(cached_features_file) and not args.overwrite_cache:
+        logger.info("Loading features from cached file %s", cached_features_file)
+        features = torch.load(cached_features_file)
+    else:
+        logger.info("Creating features from dataset file at %s", args.data_dir)
+        label_list = processor.get_labels()
+        if task in ["mnli", "mnli-mm"] and args.model_type in ["roberta"]:
+            # HACK(label indices are swapped in RoBERTa pretrained model)
+            label_list[1], label_list[2] = label_list[2], label_list[1]
+        examples = (
+            processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
+        )
+        features = convert_examples_to_features(
+            examples, tokenizer, label_list=label_list, max_length=args.max_seq_length, output_mode=output_mode,
+        )
+        if args.local_rank in [-1, 0]:
+            logger.info("Saving features into cached file %s", cached_features_file)
+            torch.save(features, cached_features_file)
+
+    if args.local_rank == 0 and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    # Convert to Tensors and build dataset
+    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
+    all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)
+
+    if features[0].token_type_ids is None:
+        # For RoBERTa (a potential bug!)
+        all_token_type_ids = torch.tensor([[0] * args.max_seq_length for f in features], dtype=torch.long)
+    else:
+        all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)
+    if output_mode == "classification":
+        all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
+    elif output_mode == "regression":
+        all_labels = torch.tensor([f.label for f in features], dtype=torch.float)
+
+    dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels)
+    return dataset
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    # Required parameters
+    parser.add_argument(
+        "--data_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The input data dir. Should contain the .tsv files (or other data files) for the task.",
+    )
+    parser.add_argument(
+        "--model_type",
+        default=None,
+        type=str,
+        required=True,
+        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        default=None,
+        type=str,
+        required=True,
+        help="Path to pre-trained model or shortcut name.",
+    )
+    parser.add_argument(
+        "--task_name",
+        default=None,
+        type=str,
+        required=True,
+        help="The name of the task to train selected in the list: " + ", ".join(processors.keys()),
+    )
+    parser.add_argument(
+        "--output_dir",
+        default=None,
+        type=str,
+        required=True,
+        help="The output directory where the model predictions and checkpoints will be written.",
+    )
+    parser.add_argument(
+        "--plot_data_dir",
+        default="./plotting/",
+        type=str,
+        required=False,
+        help="The directory to store data for plotting figures.",
+    )
+
+    # Other parameters
+    parser.add_argument(
+        "--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name"
+    )
+    parser.add_argument(
+        "--tokenizer_name",
+        default="",
+        type=str,
+        help="Pretrained tokenizer name or path if not the same as model_name",
+    )
+    parser.add_argument(
+        "--cache_dir",
+        default="",
+        type=str,
+        help="Where do you want to store the pre-trained models downloaded from s3",
+    )
+    parser.add_argument(
+        "--max_seq_length",
+        default=128,
+        type=int,
+        help="The maximum total input sequence length after tokenization. Sequences longer "
+        "than this will be truncated, sequences shorter will be padded.",
+    )
+    parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
+    parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.")
+    parser.add_argument(
+        "--evaluate_during_training", action="store_true", help="Rul evaluation during training at each logging step."
+    )
+    parser.add_argument(
+        "--do_lower_case", action="store_true", help="Set this flag if you are using an uncased model."
+    )
+    parser.add_argument("--eval_each_highway", action="store_true", help="Set this flag to evaluate each highway.")
+    parser.add_argument(
+        "--eval_after_first_stage",
+        action="store_true",
+        help="Set this flag to evaluate after training only bert (not highway).",
+    )
+    parser.add_argument("--eval_highway", action="store_true", help="Set this flag if it's evaluating highway models")
+
+    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, help="Batch size per GPU/CPU for training.")
+    parser.add_argument(
+        "--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for evaluation."
+    )
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Number of updates steps to accumulate before performing a backward/update pass.",
+    )
+    parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
+    parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight deay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
+    parser.add_argument(
+        "--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform."
+    )
+    parser.add_argument(
+        "--max_steps",
+        default=-1,
+        type=int,
+        help="If > 0: set total number of training steps to perform. Override num_train_epochs.",
+    )
+    parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")
+    parser.add_argument("--early_exit_entropy", default=-1, type=float, help="Entropy threshold for early exit.")
+
+    parser.add_argument("--logging_steps", type=int, default=50, help="Log every X updates steps.")
+    parser.add_argument("--save_steps", type=int, default=50, help="Save checkpoint every X updates steps.")
+    parser.add_argument(
+        "--eval_all_checkpoints",
+        action="store_true",
+        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number",
+    )
+    parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
+    parser.add_argument(
+        "--overwrite_output_dir", action="store_true", help="Overwrite the content of the output directory"
+    )
+    parser.add_argument(
+        "--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets"
+    )
+    parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
+
+    parser.add_argument(
+        "--fp16",
+        action="store_true",
+        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
+    )
+    parser.add_argument(
+        "--fp16_opt_level",
+        type=str,
+        default="O1",
+        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+        "See details at https://nvidia.github.io/apex/amp.html",
+    )
+    parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank")
+    parser.add_argument("--server_ip", type=str, default="", help="For distant debugging.")
+    parser.add_argument("--server_port", type=str, default="", help="For distant debugging.")
+    args = parser.parse_args()
+
+    if (
+        os.path.exists(args.output_dir)
+        and os.listdir(args.output_dir)
+        and args.do_train
+        and not args.overwrite_output_dir
+    ):
+        raise ValueError(
+            "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
+                args.output_dir
+            )
+        )
+
+    # Setup distant debugging if needed
+    if args.server_ip and args.server_port:
+        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
+        import ptvsd
+
+        print("Waiting for debugger attach")
+        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
+        ptvsd.wait_for_attach()
+
+    # Setup CUDA, GPU & distributed training
+    if args.local_rank == -1 or args.no_cuda:
+        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+        args.n_gpu = torch.cuda.device_count()
+    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+        torch.cuda.set_device(args.local_rank)
+        device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend="nccl")
+        args.n_gpu = 1
+    args.device = device
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
+    )
+    logger.warning(
+        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+        args.local_rank,
+        device,
+        args.n_gpu,
+        bool(args.local_rank != -1),
+        args.fp16,
+    )
+
+    # Set seed
+    set_seed(args)
+
+    # Prepare GLUE task
+    args.task_name = args.task_name.lower()
+    if args.task_name not in processors:
+        raise ValueError("Task not found: %s" % (args.task_name))
+    processor = processors[args.task_name]()
+    args.output_mode = output_modes[args.task_name]
+    label_list = processor.get_labels()
+    num_labels = len(label_list)
+
+    # Load pretrained model and tokenizer
+    if args.local_rank not in [-1, 0]:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    args.model_type = args.model_type.lower()
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(
+        args.config_name if args.config_name else args.model_name_or_path,
+        num_labels=num_labels,
+        finetuning_task=args.task_name,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+    tokenizer = tokenizer_class.from_pretrained(
+        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+        do_lower_case=args.do_lower_case,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+    model = model_class.from_pretrained(
+        args.model_name_or_path,
+        from_tf=bool(".ckpt" in args.model_name_or_path),
+        config=config,
+        cache_dir=args.cache_dir if args.cache_dir else None,
+    )
+
+    if args.model_type == "bert":
+        model.bert.encoder.set_early_exit_entropy(args.early_exit_entropy)
+        model.bert.init_highway_pooler()
+    elif args.model_type == "roberta":
+        model.roberta.encoder.set_early_exit_entropy(args.early_exit_entropy)
+        model.roberta.init_highway_pooler()
+    else:
+        raise NotImplementedError()
+
+    if args.local_rank == 0:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    model.to(args.device)
+
+    logger.info("Training/evaluation parameters %s", args)
+
+    # Training
+    if args.do_train:
+        train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)
+        global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+
+        if args.eval_after_first_stage:
+            result = evaluate(args, model, tokenizer, prefix="")
+            print_result = get_wanted_result(result)
+
+        train(args, train_dataset, model, tokenizer, train_highway=True)
+
+    # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
+        # Create output directory if needed
+        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(args.output_dir)
+
+        logger.info("Saving model checkpoint to %s", args.output_dir)
+        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
+        # They can then be reloaded using `from_pretrained()`
+        model_to_save = (
+            model.module if hasattr(model, "module") else model
+        )  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(args.output_dir)
+        tokenizer.save_pretrained(args.output_dir)
+
+        # Good practice: save your training arguments together with the trained model
+        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
+
+        # Load a trained model and vocabulary that you have fine-tuned
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
+        model.to(args.device)
+
+    # Evaluation
+    results = {}
+    if args.do_eval and args.local_rank in [-1, 0]:
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        checkpoints = [args.output_dir]
+        if args.eval_all_checkpoints:
+            checkpoints = list(
+                os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME, recursive=True))
+            )
+            logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN)  # Reduce logging
+        logger.info("Evaluate the following checkpoints: %s", checkpoints)
+        for checkpoint in checkpoints:
+            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
+            prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""
+
+            model = model_class.from_pretrained(checkpoint)
+            if args.model_type == "bert":
+                model.bert.encoder.set_early_exit_entropy(args.early_exit_entropy)
+            elif args.model_type == "roberta":
+                model.roberta.encoder.set_early_exit_entropy(args.early_exit_entropy)
+            else:
+                raise NotImplementedError()
+
+            model.to(args.device)
+            result = evaluate(args, model, tokenizer, prefix=prefix, eval_highway=args.eval_highway)
+            print_result = get_wanted_result(result)
+            logger.info("Result: {}".format(print_result))
+            if args.eval_each_highway:
+                last_layer_results = print_result
+                each_layer_results = []
+                for i in range(model.num_layers):
+                    logger.info("\n")
+                    _result = evaluate(
+                        args, model, tokenizer, prefix=prefix, output_layer=i, eval_highway=args.eval_highway
+                    )
+                    if i + 1 < model.num_layers:
+                        each_layer_results.append(get_wanted_result(_result))
+                each_layer_results.append(last_layer_results)
+                save_fname = args.plot_data_dir + "/" + args.model_name_or_path[2:] + "/each_layer.npy"
+                if not os.path.exists(os.path.dirname(save_fname)):
+                    os.makedirs(os.path.dirname(save_fname))
+                np.save(save_fname, np.array(each_layer_results))
+                info_str = "Score of each layer:"
+                for i in range(model.num_layers):
+                    info_str += " {:.2f}".format(100 * each_layer_results[i])
+                logger.info(info_str)
+            result = dict((k + "_{}".format(global_step), v) for k, v in result.items())
+            results.update(result)
+
+    return results
+
+
+if __name__ == "__main__":
+    main()

examples/deebert/src/modeling_highway_bert.py

@@ -0,0 +1,396 @@
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss, MSELoss
+
+from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_callable
+from transformers.modeling_bert import (
+    BERT_INPUTS_DOCSTRING,
+    BERT_START_DOCSTRING,
+    BertEmbeddings,
+    BertLayer,
+    BertPooler,
+    BertPreTrainedModel,
+)
+
+
+def entropy(x):
+    """ Calculate entropy of a pre-softmax logit Tensor
+    """
+    exp_x = torch.exp(x)
+    A = torch.sum(exp_x, dim=1)  # sum of exp(x_i)
+    B = torch.sum(x * exp_x, dim=1)  # sum of x_i * exp(x_i)
+    return torch.log(A) - B / A
+
+
+class DeeBertEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.output_attentions = config.output_attentions
+        self.output_hidden_states = config.output_hidden_states
+        self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)])
+        self.highway = nn.ModuleList([BertHighway(config) for _ in range(config.num_hidden_layers)])
+
+        self.early_exit_entropy = [-1 for _ in range(config.num_hidden_layers)]
+
+    def set_early_exit_entropy(self, x):
+        if (type(x) is float) or (type(x) is int):
+            for i in range(len(self.early_exit_entropy)):
+                self.early_exit_entropy[i] = x
+        else:
+            self.early_exit_entropy = x
+
+    def init_highway_pooler(self, pooler):
+        loaded_model = pooler.state_dict()
+        for highway in self.highway:
+            for name, param in highway.pooler.state_dict().items():
+                param.copy_(loaded_model[name])
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+    ):
+        all_hidden_states = ()
+        all_attentions = ()
+        all_highway_exits = ()
+        for i, layer_module in enumerate(self.layer):
+            if self.output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            layer_outputs = layer_module(
+                hidden_states, attention_mask, head_mask[i], encoder_hidden_states, encoder_attention_mask
+            )
+            hidden_states = layer_outputs[0]
+
+            if self.output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+            current_outputs = (hidden_states,)
+            if self.output_hidden_states:
+                current_outputs = current_outputs + (all_hidden_states,)
+            if self.output_attentions:
+                current_outputs = current_outputs + (all_attentions,)
+
+            highway_exit = self.highway[i](current_outputs)
+            # logits, pooled_output
+
+            if not self.training:
+                highway_logits = highway_exit[0]
+                highway_entropy = entropy(highway_logits)
+                highway_exit = highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
+                all_highway_exits = all_highway_exits + (highway_exit,)
+
+                if highway_entropy < self.early_exit_entropy[i]:
+                    new_output = (highway_logits,) + current_outputs[1:] + (all_highway_exits,)
+                    raise HighwayException(new_output, i + 1)
+            else:
+                all_highway_exits = all_highway_exits + (highway_exit,)
+
+        # Add last layer
+        if self.output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        outputs = (hidden_states,)
+        if self.output_hidden_states:
+            outputs = outputs + (all_hidden_states,)
+        if self.output_attentions:
+            outputs = outputs + (all_attentions,)
+
+        outputs = outputs + (all_highway_exits,)
+        return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits
+
+
+@add_start_docstrings(
+    "The Bert Model transformer with early exiting (DeeBERT). ", BERT_START_DOCSTRING,
+)
+class DeeBertModel(BertPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+
+        self.embeddings = BertEmbeddings(config)
+        self.encoder = DeeBertEncoder(config)
+        self.pooler = BertPooler(config)
+
+        self.init_weights()
+
+    def init_highway_pooler(self):
+        self.encoder.init_highway_pooler(self.pooler)
+
+    def get_input_embeddings(self):
+        return self.embeddings.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.embeddings.word_embeddings = value
+
+    def _prune_heads(self, heads_to_prune):
+        """ Prunes heads of the model.
+            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
+            See base class PreTrainedModel
+        """
+        for layer, heads in heads_to_prune.items():
+            self.encoder.layer[layer].attention.prune_heads(heads)
+
+    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+    ):
+        r"""
+    Return:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
+        last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
+            Last layer hidden-state of the first token of the sequence (classification token)
+            further processed by a Linear layer and a Tanh activation function. The Linear
+            layer weights are trained from the next sentence prediction (classification)
+            objective during pre-training.
+
+            This output is usually *not* a good summary
+            of the semantic content of the input, you're often better with averaging or pooling
+            the sequence of hidden-states for the whole input sequence.
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        highway_exits (:obj:`tuple(tuple(torch.Tensor))`:
+            Tuple of each early exit's results (total length: number of layers)
+            Each tuple is again, a tuple of length 2 - the first entry is logits and the second entry is hidden states.
+        """
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+
+        if attention_mask is None:
+            attention_mask = torch.ones(input_shape, device=device)
+        if encoder_attention_mask is None:
+            encoder_attention_mask = torch.ones(input_shape, device=device)
+        if token_type_ids is None:
+            token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
+
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, device)
+
+        # If a 2D ou 3D attention mask is provided for the cross-attention
+        # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length]
+        if encoder_attention_mask.dim() == 3:
+            encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :]
+        if encoder_attention_mask.dim() == 2:
+            encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :]
+
+        encoder_extended_attention_mask = encoder_extended_attention_mask.to(
+            dtype=next(self.parameters()).dtype
+        )  # fp16 compatibility
+        encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0
+
+        # Prepare head mask if needed
+        # 1.0 in head_mask indicate we keep the head
+        # attention_probs has shape bsz x n_heads x N x N
+        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
+        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
+        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+
+        embedding_output = self.embeddings(
+            input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds
+        )
+        encoder_outputs = self.encoder(
+            embedding_output,
+            attention_mask=extended_attention_mask,
+            head_mask=head_mask,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_extended_attention_mask,
+        )
+        sequence_output = encoder_outputs[0]
+        pooled_output = self.pooler(sequence_output)
+
+        outputs = (sequence_output, pooled_output,) + encoder_outputs[
+            1:
+        ]  # add hidden_states and attentions if they are here
+        return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits
+
+
+class HighwayException(Exception):
+    def __init__(self, message, exit_layer):
+        self.message = message
+        self.exit_layer = exit_layer  # start from 1!
+
+
+class BertHighway(nn.Module):
+    """A module to provide a shortcut
+    from (the output of one non-final BertLayer in BertEncoder) to (cross-entropy computation in BertForSequenceClassification)
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        self.pooler = BertPooler(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+    def forward(self, encoder_outputs):
+        # Pooler
+        pooler_input = encoder_outputs[0]
+        pooler_output = self.pooler(pooler_input)
+        # "return" pooler_output
+
+        # BertModel
+        bmodel_output = (pooler_input, pooler_output) + encoder_outputs[1:]
+        # "return" bodel_output
+
+        # Dropout and classification
+        pooled_output = bmodel_output[1]
+
+        pooled_output = self.dropout(pooled_output)
+        logits = self.classifier(pooled_output)
+
+        return logits, pooled_output
+
+
+@add_start_docstrings(
+    """Bert Model (with early exiting - DeeBERT) with a classifier on top,
+    also takes care of multi-layer training. """,
+    BERT_START_DOCSTRING,
+)
+class DeeBertForSequenceClassification(BertPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.num_layers = config.num_hidden_layers
+
+        self.bert = DeeBertModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
+
+        self.init_weights()
+
+    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        output_layer=-1,
+        train_highway=False,
+    ):
+        r"""
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
+            If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
+            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
+            Classification (or regression if config.num_labels==1) loss.
+        logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        highway_exits (:obj:`tuple(tuple(torch.Tensor))`:
+            Tuple of each early exit's results (total length: number of layers)
+            Each tuple is again, a tuple of length 2 - the first entry is logits and the second entry is hidden states.
+        """
+
+        exit_layer = self.num_layers
+        try:
+            outputs = self.bert(
+                input_ids,
+                attention_mask=attention_mask,
+                token_type_ids=token_type_ids,
+                position_ids=position_ids,
+                head_mask=head_mask,
+                inputs_embeds=inputs_embeds,
+            )
+            # sequence_output, pooled_output, (hidden_states), (attentions), highway exits
+
+            pooled_output = outputs[1]
+
+            pooled_output = self.dropout(pooled_output)
+            logits = self.classifier(pooled_output)
+            outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+        except HighwayException as e:
+            outputs = e.message
+            exit_layer = e.exit_layer
+            logits = outputs[0]
+
+        if not self.training:
+            original_entropy = entropy(logits)
+            highway_entropy = []
+            highway_logits_all = []
+        if labels is not None:
+            if self.num_labels == 1:
+                #  We are doing regression
+                loss_fct = MSELoss()
+                loss = loss_fct(logits.view(-1), labels.view(-1))
+            else:
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+
+            # work with highway exits
+            highway_losses = []
+            for highway_exit in outputs[-1]:
+                highway_logits = highway_exit[0]
+                if not self.training:
+                    highway_logits_all.append(highway_logits)
+                    highway_entropy.append(highway_exit[2])
+                if self.num_labels == 1:
+                    #  We are doing regression
+                    loss_fct = MSELoss()
+                    highway_loss = loss_fct(highway_logits.view(-1), labels.view(-1))
+                else:
+                    loss_fct = CrossEntropyLoss()
+                    highway_loss = loss_fct(highway_logits.view(-1, self.num_labels), labels.view(-1))
+                highway_losses.append(highway_loss)
+
+            if train_highway:
+                outputs = (sum(highway_losses[:-1]),) + outputs
+                # exclude the final highway, of course
+            else:
+                outputs = (loss,) + outputs
+        if not self.training:
+            outputs = outputs + ((original_entropy, highway_entropy), exit_layer)
+            if output_layer >= 0:
+                outputs = (
+                    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
+                )  # use the highway of the last layer
+
+        return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)

examples/deebert/src/modeling_highway_roberta.py

@@ -0,0 +1,151 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import torch.nn as nn
+from torch.nn import CrossEntropyLoss, MSELoss
+
+from transformers.configuration_roberta import RobertaConfig
+from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_callable
+from transformers.modeling_roberta import ROBERTA_INPUTS_DOCSTRING, ROBERTA_START_DOCSTRING, RobertaEmbeddings
+
+from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy
+
+
+@add_start_docstrings(
+    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). ", ROBERTA_START_DOCSTRING,
+)
+class DeeRobertaModel(DeeBertModel):
+
+    config_class = RobertaConfig
+    base_model_prefix = "roberta"
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.embeddings = RobertaEmbeddings(config)
+        self.init_weights()
+
+
+@add_start_docstrings(
+    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
+    also takes care of multi-layer training. """,
+    ROBERTA_START_DOCSTRING,
+)
+class DeeRobertaForSequenceClassification(BertPreTrainedModel):
+
+    config_class = RobertaConfig
+    base_model_prefix = "roberta"
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.num_layers = config.num_hidden_layers
+
+        self.roberta = DeeRobertaModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
+
+    @add_start_docstrings_to_callable(ROBERTA_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        head_mask=None,
+        inputs_embeds=None,
+        labels=None,
+        output_layer=-1,
+        train_highway=False,
+    ):
+        r"""
+        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`, defaults to :obj:`None`):
+            Labels for computing the sequence classification/regression loss.
+            Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
+            If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
+            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+
+    Returns:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.RobertaConfig`) and inputs:
+        loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`label` is provided):
+            Classification (or regression if config.num_labels==1) loss.
+        logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.num_labels)`):
+            Classification (or regression if config.num_labels==1) scores (before SoftMax).
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        highway_exits (:obj:`tuple(tuple(torch.Tensor))`:
+            Tuple of each early exit's results (total length: number of layers)
+            Each tuple is again, a tuple of length 2 - the first entry is logits and the second entry is hidden states.
+        """
+
+        exit_layer = self.num_layers
+        try:
+            outputs = self.roberta(
+                input_ids,
+                attention_mask=attention_mask,
+                token_type_ids=token_type_ids,
+                position_ids=position_ids,
+                head_mask=head_mask,
+                inputs_embeds=inputs_embeds,
+            )
+
+            pooled_output = outputs[1]
+
+            pooled_output = self.dropout(pooled_output)
+            logits = self.classifier(pooled_output)
+            outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
+        except HighwayException as e:
+            outputs = e.message
+            exit_layer = e.exit_layer
+            logits = outputs[0]
+
+        if not self.training:
+            original_entropy = entropy(logits)
+            highway_entropy = []
+            highway_logits_all = []
+        if labels is not None:
+            if self.num_labels == 1:
+                #  We are doing regression
+                loss_fct = MSELoss()
+                loss = loss_fct(logits.view(-1), labels.view(-1))
+            else:
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+
+            # work with highway exits
+            highway_losses = []
+            for highway_exit in outputs[-1]:
+                highway_logits = highway_exit[0]
+                if not self.training:
+                    highway_logits_all.append(highway_logits)
+                    highway_entropy.append(highway_exit[2])
+                if self.num_labels == 1:
+                    #  We are doing regression
+                    loss_fct = MSELoss()
+                    highway_loss = loss_fct(highway_logits.view(-1), labels.view(-1))
+                else:
+                    loss_fct = CrossEntropyLoss()
+                    highway_loss = loss_fct(highway_logits.view(-1, self.num_labels), labels.view(-1))
+                highway_losses.append(highway_loss)
+
+            if train_highway:
+                outputs = (sum(highway_losses[:-1]),) + outputs
+                # exclude the final highway, of course
+            else:
+                outputs = (loss,) + outputs
+        if not self.training:
+            outputs = outputs + ((original_entropy, highway_entropy), exit_layer)
+            if output_layer >= 0:
+                outputs = (
+                    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
+                )  # use the highway of the last layer
+
+        return outputs  # (loss), logits, (hidden_states), (attentions), entropy

examples/deebert/test_glue_deebert.py

@@ -0,0 +1,97 @@
+import argparse
+import logging
+import sys
+import unittest
+from unittest.mock import patch
+
+import run_glue_deebert
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+logger = logging.getLogger()
+
+
+def get_setup_file():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f")
+    args = parser.parse_args()
+    return args.f
+
+
+class DeeBertTests(unittest.TestCase):
+    def test_glue_deebert(self):
+        stream_handler = logging.StreamHandler(sys.stdout)
+        logger.addHandler(stream_handler)
+
+        train_args = """
+            run_glue_deebert.py
+            --model_type roberta
+            --model_name_or_path roberta-base
+            --task_name MRPC
+            --do_train
+            --do_eval
+            --do_lower_case
+            --data_dir ./tests/fixtures/tests_samples/MRPC/
+            --max_seq_length 128
+            --per_gpu_eval_batch_size=1
+            --per_gpu_train_batch_size=8
+            --learning_rate 2e-4
+            --num_train_epochs 3
+            --overwrite_output_dir
+            --seed 42
+            --output_dir ./examples/deebert/saved_models/roberta-base/MRPC/two_stage
+            --plot_data_dir ./examples/deebert/results/
+            --save_steps 0
+            --overwrite_cache
+            --eval_after_first_stage
+            """.split()
+
+        eval_args = """
+            run_glue_deebert.py
+            --model_type roberta
+            --model_name_or_path ./examples/deebert/saved_models/roberta-base/MRPC/two_stage
+            --task_name MRPC
+            --do_eval
+            --do_lower_case
+            --data_dir ./tests/fixtures/tests_samples/MRPC/
+            --output_dir ./examples/deebert/saved_models/roberta-base/MRPC/two_stage
+            --plot_data_dir ./examples/deebert/results/
+            --max_seq_length 128
+            --eval_each_highway
+            --eval_highway
+            --overwrite_cache
+            --per_gpu_eval_batch_size=1
+            """.split()
+
+        entropy_eval_args = """
+            run_glue_deebert.py
+            --model_type roberta
+            --model_name_or_path ./examples/deebert/saved_models/roberta-base/MRPC/two_stage
+            --task_name MRPC
+            --do_eval
+            --do_lower_case
+            --data_dir ./tests/fixtures/tests_samples/MRPC/
+            --output_dir ./examples/deebert/saved_models/roberta-base/MRPC/two_stage
+            --plot_data_dir ./examples/deebert/results/
+            --max_seq_length 128
+            --early_exit_entropy 0.1
+            --eval_highway
+            --overwrite_cache
+            --per_gpu_eval_batch_size=1
+            """.split()
+
+        with patch.object(sys, "argv", train_args):
+            result = run_glue_deebert.main()
+            for value in result.values():
+                self.assertGreaterEqual(value, 0.75)
+
+        with patch.object(sys, "argv", eval_args):
+            result = run_glue_deebert.main()
+            for value in result.values():
+                self.assertGreaterEqual(value, 0.75)
+
+        with patch.object(sys, "argv", entropy_eval_args):
+            result = run_glue_deebert.main()
+            for value in result.values():
+                self.assertGreaterEqual(value, 0.75)

examples/deebert/train_deebert.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+export CUDA_VISIBLE_DEVICES=0
+
+PATH_TO_DATA=/h/xinji/projects/GLUE
+
+MODEL_TYPE=bert  # bert or roberta
+MODEL_SIZE=base  # base or large
+DATASET=MRPC  # SST-2, MRPC, RTE, QNLI, QQP, or MNLI
+
+MODEL_NAME=${MODEL_TYPE}-${MODEL_SIZE}
+EPOCHS=10
+if [ $MODEL_TYPE = 'bert' ]
+then
+  EPOCHS=3
+  MODEL_NAME=${MODEL_NAME}-uncased
+fi
+
+
+python -u run_glue_deebert.py \
+  --model_type $MODEL_TYPE \
+  --model_name_or_path $MODEL_NAME \
+  --task_name $DATASET \
+  --do_train \
+  --do_eval \
+  --do_lower_case \
+  --data_dir $PATH_TO_DATA/$DATASET \
+  --max_seq_length 128 \
+  --per_gpu_eval_batch_size=1 \
+  --per_gpu_train_batch_size=8 \
+  --learning_rate 2e-5 \
+  --num_train_epochs $EPOCHS \
+  --overwrite_output_dir \
+  --seed 42 \
+  --output_dir ./saved_models/${MODEL_TYPE}-${MODEL_SIZE}/$DATASET/two_stage \
+  --plot_data_dir ./results/ \
+  --save_steps 0 \
+  --overwrite_cache \
+  --eval_after_first_stage

examples/language-modeling/run_language_modeling.py

@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-Fine-tuning the library models for language modeling on a text file (GPT, GPT-2, BERT, RoBERTa).
-GPT and GPT-2 are fine-tuned using a causal language modeling (CLM) loss while BERT and RoBERTa are fine-tuned
-using a masked language modeling (MLM) loss.
+Fine-tuning the library models for language modeling on a text file (GPT, GPT-2, CTRL, BERT, RoBERTa, XLNet).
+GPT, GPT-2 and CTRL are fine-tuned using a causal language modeling (CLM) loss. BERT and RoBERTa are fine-tuned
+using a masked language modeling (MLM) loss. XLNet is fine-tuned using a permutation language modeling (PLM) loss.
 """
 
 
@@ -33,6 +33,7 @@ from transformers import (
     AutoModelWithLMHead,
     AutoTokenizer,
     DataCollatorForLanguageModeling,
+    DataCollatorForPermutationLanguageModeling,
     HfArgumentParser,
     LineByLineTextDataset,
     PreTrainedTokenizer,
@@ -101,6 +102,15 @@ class DataTrainingArguments:
     mlm_probability: float = field(
         default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
     )
+    plm_probability: float = field(
+        default=1 / 6,
+        metadata={
+            "help": "Ratio of length of a span of masked tokens to surrounding context length for permutation language modeling."
+        },
+    )
+    max_span_length: int = field(
+        default=5, metadata={"help": "Maximum length of a span of masked tokens for permutation language modeling."}
+    )
 
     block_size: int = field(
         default=-1,
@@ -207,8 +217,8 @@ def main():
 
     if config.model_type in ["bert", "roberta", "distilbert", "camembert"] and not data_args.mlm:
         raise ValueError(
-            "BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
-            "flag (masked language modeling)."
+            "BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the"
+            "--mlm flag (masked language modeling)."
         )
 
     if data_args.block_size <= 0:
@@ -221,9 +231,14 @@ def main():
 
     train_dataset = get_dataset(data_args, tokenizer=tokenizer) if training_args.do_train else None
     eval_dataset = get_dataset(data_args, tokenizer=tokenizer, evaluate=True) if training_args.do_eval else None
-    data_collator = DataCollatorForLanguageModeling(
-        tokenizer=tokenizer, mlm=data_args.mlm, mlm_probability=data_args.mlm_probability
-    )
+    if config.model_type == "xlnet":
+        data_collator = DataCollatorForPermutationLanguageModeling(
+            tokenizer=tokenizer, plm_probability=data_args.plm_probability, max_span_length=data_args.max_span_length,
+        )
+    else:
+        data_collator = DataCollatorForLanguageModeling(
+            tokenizer=tokenizer, mlm=data_args.mlm, mlm_probability=data_args.mlm_probability
+        )
 
     # Initialize our Trainer
     trainer = Trainer(

examples/longform-qa/eli5_app.py

@@ -1,10 +1,10 @@
 import faiss
 import nlp
 import numpy as np
+import streamlit as st
 import torch
 from elasticsearch import Elasticsearch
 
-import streamlit as st
 import transformers
 from eli5_utils import (
     embed_questions_for_retrieval,

examples/question-answering/README.md

@@ -77,7 +77,7 @@ exact_match = 86.91
 ```
 
 This fine-tuned model is available as a checkpoint under the reference
-`bert-large-uncased-whole-word-masking-finetuned-squad`.
+[`bert-large-uncased-whole-word-masking-finetuned-squad`](https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad).
 
 #### Fine-tuning XLNet on SQuAD
 
@@ -176,4 +176,5 @@ python run_tf_squad.py \
     --doc_stride 128    
 ```
 
-For the moment the evaluation is not available in the Tensorflow Trainer only the training.
+
+For the moment evaluation is not available in the Tensorflow Trainer only the training.

examples/question-answering/run_squad_trainer.py

@@ -0,0 +1,160 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Fine-tuning the library models for question-answering."""
+
+
+import logging
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Optional
+
+from transformers import AutoConfig, AutoModelForQuestionAnswering, AutoTokenizer, HfArgumentParser, SquadDataset
+from transformers import SquadDataTrainingArguments as DataTrainingArguments
+from transformers import Trainer, TrainingArguments
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
+    """
+
+    model_name_or_path: str = field(
+        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    use_fast: bool = field(default=False, metadata={"help": "Set this flag to use fast tokenization."})
+    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
+    # or just modify its tokenizer_config.json.
+    cache_dir: Optional[str] = field(
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
+    )
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    if (
+        os.path.exists(training_args.output_dir)
+        and os.listdir(training_args.output_dir)
+        and training_args.do_train
+        and not training_args.overwrite_output_dir
+    ):
+        raise ValueError(
+            f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
+        )
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,
+    )
+    logger.warning(
+        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+        training_args.local_rank,
+        training_args.device,
+        training_args.n_gpu,
+        bool(training_args.local_rank != -1),
+        training_args.fp16,
+    )
+    logger.info("Training/evaluation parameters %s", training_args)
+
+    # Prepare Question-Answering task
+    # Load pretrained model and tokenizer
+    #
+    # Distributed training:
+    # The .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+
+    config = AutoConfig.from_pretrained(
+        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+    )
+    model = AutoModelForQuestionAnswering.from_pretrained(
+        model_args.model_name_or_path,
+        from_tf=bool(".ckpt" in model_args.model_name_or_path),
+        config=config,
+        cache_dir=model_args.cache_dir,
+    )
+
+    # Get datasets
+    is_language_sensitive = hasattr(model.config, "lang2id")
+    train_dataset = (
+        SquadDataset(
+            data_args, tokenizer=tokenizer, is_language_sensitive=is_language_sensitive, cache_dir=model_args.cache_dir
+        )
+        if training_args.do_train
+        else None
+    )
+    eval_dataset = (
+        SquadDataset(
+            data_args,
+            tokenizer=tokenizer,
+            mode="dev",
+            is_language_sensitive=is_language_sensitive,
+            cache_dir=model_args.cache_dir,
+        )
+        if training_args.do_eval
+        else None
+    )
+
+    # Initialize our Trainer
+    trainer = Trainer(model=model, args=training_args, train_dataset=train_dataset, eval_dataset=eval_dataset,)
+
+    # Training
+    if training_args.do_train:
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
+        )
+        trainer.save_model()
+        # For convenience, we also re-save the tokenizer to the same directory,
+        # so that you can share your model easily on huggingface.co/models =)
+        if trainer.is_world_master():
+            tokenizer.save_pretrained(training_args.output_dir)
+
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+
+if __name__ == "__main__":
+    main()

examples/seq2seq/README.md

@@ -41,6 +41,28 @@ If you are using your own data, it must be formatted as one directory with 6 fil
 The `.source` files are the input, the `.target` files are the desired output.
 
 
+### Tips and Tricks
+
+General Tips:
+- since you need to run from `examples/seq2seq`, and likely need to modify code, the easiest workflow is fork transformers, clone your fork, and run `pip install -e .` before you get started.   
+- try `--freeze_encoder` or `--freeze_embeds` for faster training/larger batch size.  (3hr per epoch with bs=8, see the "xsum_shared_task" command below)
+- `fp16_opt_level=O1` (the default works best).
+- In addition to the pytorch-lightning .ckpt checkpoint, a transformers checkpoint will be saved.
+Load it with `BartForConditionalGeneration.from_pretrained(f'{output_dir}/best_tfmr)`.
+- At the moment, `--do_predict` does not work in a multi-gpu setting. You need to use `evaluate_checkpoint` or the `run_eval.py` code.
+- This warning can be safely ignored: 
+    > "Some weights of BartForConditionalGeneration were not initialized from the model checkpoint at facebook/bart-large-xsum and are newly initialized: ['final_logits_bias']"
+- Both finetuning and eval are 30% faster with `--fp16`. For that you need to [install apex](https://github.com/NVIDIA/apex#quick-start).
+- Read scripts before you run them! 
+
+Summarization Tips:
+- (summ) 1 epoch at batch size 1 for bart-large takes 24 hours and requires 13GB GPU RAM with fp16 on an NVIDIA-V100.
+- If you want to run experiments on improving the summarization finetuning process, try the XSUM Shared Task (below). It's faster to train than CNNDM because the summaries are shorter.
+- For CNN/DailyMail, the default `val_max_target_length` and `test_max_target_length` will truncate the ground truth labels, resulting in slightly higher rouge scores. To get accurate rouge scores, you should rerun calculate_rouge on the `{output_dir}/test_generations.txt` file saved by `trainer.test()`
+- `--max_target_length=60 --val_max_target_length=60 --test_max_target_length=100 ` is a reasonable setting for XSUM.
+- `wandb` can be used by specifying `--logger wandb`. It is useful for reproducibility. Specify the environment variable `WANDB_PROJECT='hf_xsum'` to do the XSUM shared task.
+- If you are finetuning on your own dataset, start from `distilbart-cnn-12-6` if you want long summaries and `distilbart-xsum-12-6` if you want short summaries.
+(It rarely makes sense to start from `bart-large` unless you are a researching finetuning methods). 
 
 ### Summarization Finetuning
 Run/modify `finetune.sh`
@@ -58,25 +80,20 @@ The following command should work on a 16GB GPU:
 
 *Note*: The following tips mostly apply to summarization finetuning.
 
-Tips:
-- 1 epoch at batch size 1 for bart-large takes 24 hours and requires 13GB GPU RAM with fp16 on an NVIDIA-V100. 
-- since you need to run from `examples/seq2seq`, and likely need to modify code, it is easiest to fork, then clone transformers and run `pip install -e .` before you get started.   
-- try `bart-base`, `--freeze_encoder` or `--freeze_embeds` for faster training/larger batch size.  (3hr/epoch with bs=8, see the "xsum_shared_task" command below)
-- `fp16_opt_level=O1` (the default works best).
-- If you are finetuning on your own dataset, start from `distilbart-cnn-12-6` if you want long summaries and `distilbart-xsum-12-6` if you want short summaries.
-(It rarely makes sense to start from `bart-large` unless you are a researching finetuning methods).
-- In addition to the pytorch-lightning .ckpt checkpoint, a transformers checkpoint will be saved.
-Load it with `BartForConditionalGeneration.from_pretrained(f'{output_dir}/best_tfmr)`.
-- At the moment, `--do_predict` does not work in a multi-gpu setting. You need to use `evaluate_checkpoint` or the `run_eval.py` code.
-- If you want to run experiments on improving the summarization finetuning process, try the XSUM Shared Task (below). It's faster to train than CNNDM because the summaries are shorter.
-- For CNN/DailyMail, the default `val_max_target_length` and `test_max_target_length` will truncate the ground truth labels, resulting in slightly higher rouge scores. To get accurate rouge scores, you should rerun calculate_rouge on the `{output_dir}/test_generations.txt` file saved by `trainer.test()`
-- `--max_target_length=60 --val_max_target_length=60 --test_max_target_length=100 ` is a reasonable setting for XSUM.
-- `wandb` can be used by specifying `--logger wandb`. It is useful for reproducibility. Specify the environment variable `WANDB_PROJECT='hf_xsum'` to do the XSUM shared task. 
-- This warning can be safely ignored: 
-    > "Some weights of BartForConditionalGeneration were not initialized from the model checkpoint at facebook/bart-large-xsum and are newly initialized: ['final_logits_bias']"
-- Both finetuning and eval are 30% faster with `--fp16`. For that you need to [install apex](https://github.com/NVIDIA/apex#quick-start).
+### Translation Finetuning
 
-#### Finetuning Outputs 
+First, follow the wmt_en_ro download instructions.
+Then you can finetune mbart_cc25 on english-romanian with the following command.
+**Recommendation:** Read and potentially modify the fairly opinionated defaults in `train_mbart_cc25_enro.sh` script before running it. 
+```bash
+export ENRO_DIR=${PWD}/wmt_en_ro   # may need to be fixed depending on where you downloaded
+export BS=4
+export GAS=8
+./train_mbart_cc25_enro.sh --output_dir cc25_v1_frozen/
+```
+
+
+### Finetuning Outputs 
 As you train, `output_dir` will be filled with files, that look kind of like this (comments are mine). 
 Some of them are metrics, some of them are checkpoints, some of them are metadata. Here is a quick tour:
 

examples/seq2seq/finetune.py

@@ -14,11 +14,12 @@ import torch
 from torch.utils.data import DataLoader
 
 from lightning_base import BaseTransformer, add_generic_args, generic_train
-from transformers import get_linear_schedule_with_warmup
+from transformers import MBartTokenizer, get_linear_schedule_with_warmup
 
 
 try:
     from .utils import (
+        assert_all_frozen,
         use_task_specific_params,
         SummarizationDataset,
         lmap,
@@ -47,6 +48,7 @@ except ImportError:
         get_git_info,
         ROUGE_KEYS,
         calculate_bleu_score,
+        assert_all_frozen,
     )
     from callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback
 
@@ -92,9 +94,12 @@ class SummarizationModule(BaseTransformer):
         if self.hparams.freeze_embeds:
             self.freeze_embeds()
         if self.hparams.freeze_encoder:
-            freeze_params(self.model.model.encoder)  # TODO: this will break for t5
+            freeze_params(self.model.get_encoder())
+            assert_all_frozen(self.model.get_encoder())
+
         self.hparams.git_sha = get_git_info()["repo_sha"]
         self.num_workers = hparams.num_workers
+        self.decoder_start_token_id = None
 
     def freeze_embeds(self):
         """Freeze token embeddings and positional embeddings for bart, just token embeddings for t5."""
@@ -160,7 +165,12 @@ class SummarizationModule(BaseTransformer):
         pad_token_id = self.tokenizer.pad_token_id
         source_ids, source_mask, y = SummarizationDataset.trim_seq2seq_batch(batch, pad_token_id)
         t0 = time.time()
-        generated_ids = self.model.generate(input_ids=source_ids, attention_mask=source_mask, use_cache=True,)
+        generated_ids = self.model.generate(
+            input_ids=source_ids,
+            attention_mask=source_mask,
+            use_cache=True,
+            decoder_start_token_id=self.decoder_start_token_id,
+        )
         gen_time = (time.time() - t0) / source_ids.shape[0]
         preds = self.ids_to_clean_text(generated_ids)
         target = self.ids_to_clean_text(y)
@@ -276,6 +286,9 @@ class SummarizationModule(BaseTransformer):
         parser.add_argument(
             "--task", type=str, default="summarization", required=False, help="# examples. -1 means use all."
         )
+        parser.add_argument("--src_lang", type=str, default="", required=False)
+        parser.add_argument("--tgt_lang", type=str, default="", required=False)
+
         return parser
 
 
@@ -285,6 +298,13 @@ class TranslationModule(SummarizationModule):
     metric_names = ["bleu"]
     val_metric = "bleu"
 
+    def __init__(self, hparams, **kwargs):
+        super().__init__(hparams, **kwargs)
+        self.dataset_kwargs["src_lang"] = hparams.src_lang
+        self.dataset_kwargs["tgt_lang"] = hparams.tgt_lang
+        if self.model.config.decoder_start_token_id is None and isinstance(self.tokenizer, MBartTokenizer):
+            self.decoder_start_token_id = self.tokenizer.lang_code_to_id[hparams.tgt_lang]
+
     def calc_generative_metrics(self, preds, target) -> dict:
         return calculate_bleu_score(preds, target)
 

examples/seq2seq/finetune_t5.sh

@@ -1,18 +1,13 @@
-export OUTPUT_DIR_NAME=t5
-export CURRENT_DIR=${PWD}
-export OUTPUT_DIR=${CURRENT_DIR}/${OUTPUT_DIR_NAME}
-
-# Make output directory if it doesn't exist
-mkdir -p $OUTPUT_DIR
-
 # Add parent directory to python path to access lightning_base.py
 export PYTHONPATH="../":"${PYTHONPATH}"
 
 python finetune.py \
---data_dir=./cnn-dailymail/cnn_dm \
---model_name_or_path=t5-large \
+--data_dir=$CNN_DIR \
 --learning_rate=3e-5 \
---train_batch_size=4 \
---eval_batch_size=4 \
+--train_batch_size=$BS \
+--eval_batch_size=$BS \
 --output_dir=$OUTPUT_DIR \
---do_train  $@
+--max_source_length=512 \
+--val_check_interval=0.1 --n_val=200 \
+--do_train --do_predict \
+ $@

examples/seq2seq/test_seq2seq_examples.py

@@ -223,10 +223,30 @@ def test_finetune(model):
         output_dir=output_dir,
         do_predict=True,
         task=task,
+        src_lang="en_XX",
+        tgt_lang="ro_RO",
+        freeze_encoder=True,
+        freeze_embeds=True,
     )
     assert "n_train" in args_d
     args = argparse.Namespace(**args_d)
-    main(args)
+    module = main(args)
+
+    input_embeds = module.model.get_input_embeddings()
+    assert not input_embeds.weight.requires_grad
+    if model == T5_TINY:
+        lm_head = module.model.lm_head
+        assert not lm_head.weight.requires_grad
+        assert (lm_head.weight == input_embeds.weight).all().item()
+
+    else:
+        bart = module.model.model
+        embed_pos = bart.decoder.embed_positions
+        assert not embed_pos.weight.requires_grad
+        assert not bart.shared.weight.requires_grad
+        # check that embeds are the same
+        assert bart.decoder.embed_tokens == bart.encoder.embed_tokens
+        assert bart.decoder.embed_tokens == bart.shared
 
 
 @pytest.mark.parametrize(
@@ -239,7 +259,12 @@ def test_dataset(tok):
     max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
     trunc_target = 4
     train_dataset = SummarizationDataset(
-        tokenizer, data_dir=tmp_dir, type_path="train", max_source_length=20, max_target_length=trunc_target,
+        tokenizer,
+        data_dir=tmp_dir,
+        type_path="train",
+        max_source_length=20,
+        max_target_length=trunc_target,
+        tgt_lang="ro_RO",
     )
     dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
     for batch in dataloader:

examples/seq2seq/train_mbart_cc25_enro.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+export PYTHONPATH="../":"${PYTHONPATH}"
+
+python finetune.py \
+    --learning_rate=3e-5 \
+    --fp16 \
+    --gpus 1 \
+    --do_train \
+    --do_predict \
+    --val_check_interval 0.1 \
+    --n_val 500 \
+    --adam_eps 1e-06 \
+    --num_train_epochs 3 --src_lang en_XX --tgt_lang ro_RO \
+    --freeze_encoder --freeze_embeds --data_dir $ENRO_DIR \
+    --max_source_length=300 --max_target_length 300 --val_max_target_length=300 --test_max_target_length 300 \
+    --train_batch_size=$BS --eval_batch_size=$BS --gradient_accumulation_steps=$GAS \
+    --model_name_or_path facebook/mbart-large-cc25 \
+    --task translation \
+    --warmup_steps 500 \
+    --logger wandb --sortish_sampler \
+    $@

examples/seq2seq/utils.py

@@ -14,6 +14,8 @@ from torch import nn
 from torch.utils.data import Dataset, Sampler
 from tqdm import tqdm
 
+from transformers import BartTokenizer
+
 
 def encode_file(
     tokenizer,
@@ -25,6 +27,7 @@ def encode_file(
     prefix="",
     tok_name="",
 ):
+    extra_kw = {"add_prefix_space": True} if isinstance(tokenizer, BartTokenizer) else {}
     cache_path = Path(f"{data_path}_{tok_name}{max_length}.pt")
     if not overwrite_cache and cache_path.exists():
         try:
@@ -46,8 +49,8 @@ def encode_file(
             max_length=max_length,
             padding="max_length" if pad_to_max_length else None,
             truncation=True,
-            add_prefix_space=True,
             return_tensors=return_tensors,
+            **extra_kw,
         )
         assert tokenized.input_ids.shape[1] == max_length
         examples.append(tokenized)
@@ -87,9 +90,14 @@ class SummarizationDataset(Dataset):
         n_obs=None,
         overwrite_cache=False,
         prefix="",
+        src_lang=None,
+        tgt_lang=None,
     ):
         super().__init__()
+        # FIXME: the rstrip logic strips all the chars, it seems.
         tok_name = tokenizer.__class__.__name__.lower().rstrip("tokenizer")
+        if hasattr(tokenizer, "set_lang") and src_lang is not None:
+            tokenizer.set_lang(src_lang)  # HACK: only applies to mbart
         self.source = encode_file(
             tokenizer,
             os.path.join(data_dir, type_path + ".source"),
@@ -100,7 +108,8 @@ class SummarizationDataset(Dataset):
         )
         tgt_path = os.path.join(data_dir, type_path + ".target")
         if hasattr(tokenizer, "set_lang"):
-            tokenizer.set_lang("ro_RO")  # HACK: only applies to mbart
+            assert tgt_lang is not None, "--tgt_lang must be passed to build a translation"
+            tokenizer.set_lang(tgt_lang)  # HACK: only applies to mbart
         self.target = encode_file(
             tokenizer, tgt_path, max_target_length, overwrite_cache=overwrite_cache, tok_name=tok_name
         )
@@ -224,8 +233,8 @@ def get_git_info():
 ROUGE_KEYS = ["rouge1", "rouge2", "rougeL"]
 
 
-def calculate_rouge(output_lns: List[str], reference_lns: List[str]) -> Dict:
-    scorer = rouge_scorer.RougeScorer(ROUGE_KEYS, use_stemmer=True)
+def calculate_rouge(output_lns: List[str], reference_lns: List[str], use_stemmer=True) -> Dict:
+    scorer = rouge_scorer.RougeScorer(ROUGE_KEYS, use_stemmer=use_stemmer)
     aggregator = scoring.BootstrapAggregator()
 
     for reference_ln, output_ln in zip(reference_lns, output_lns):

examples/test_xla_examples.py

@@ -0,0 +1,91 @@
+# coding=utf-8
+# Copyright 2018 HuggingFace Inc..
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import logging
+import sys
+import unittest
+from time import time
+from unittest.mock import patch
+
+from transformers.testing_utils import require_torch_tpu
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+logger = logging.getLogger()
+
+
+def get_setup_file():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f")
+    args = parser.parse_args()
+    return args.f
+
+
+@require_torch_tpu
+class TorchXLAExamplesTests(unittest.TestCase):
+    def test_run_glue(self):
+        import xla_spawn
+
+        stream_handler = logging.StreamHandler(sys.stdout)
+        logger.addHandler(stream_handler)
+
+        output_directory = "run_glue_output"
+
+        testargs = f"""
+            text-classification/run_glue.py
+            --num_cores=8
+            text-classification/run_glue.py
+            --do_train
+            --do_eval
+            --task_name=MRPC
+            --data_dir=../glue_data/MRPC
+            --cache_dir=./cache_dir
+            --num_train_epochs=1
+            --max_seq_length=128
+            --learning_rate=3e-5
+            --output_dir={output_directory}
+            --overwrite_output_dir
+            --logging_steps=5
+            --save_steps=5
+            --overwrite_cache
+            --tpu_metrics_debug
+            --model_name_or_path=bert-base-cased
+            --per_device_train_batch_size=64
+            --per_device_eval_batch_size=64
+            --evaluate_during_training
+            --overwrite_cache
+            """.split()
+        with patch.object(sys, "argv", testargs):
+            start = time()
+            xla_spawn.main()
+            end = time()
+
+            result = {}
+            with open(f"{output_directory}/eval_results_mrpc.txt") as f:
+                lines = f.readlines()
+                for line in lines:
+                    key, value = line.split(" = ")
+                    result[key] = float(value)
+
+            del result["eval_loss"]
+            for value in result.values():
+                # Assert that the model trains
+                self.assertGreaterEqual(value, 0.70)
+
+            # Assert that the script takes less than 100 seconds to make sure it doesn't hang.
+            self.assertLess(end - start, 100)

examples/text-generation/run_generation.py

@@ -214,8 +214,14 @@ def main():
     if requires_preprocessing:
         prepare_input = PREPROCESSING_FUNCTIONS.get(args.model_type)
         preprocessed_prompt_text = prepare_input(args, model, tokenizer, prompt_text)
+
+        if model.__class__.__name__ in ["TransfoXLLMHeadModel"]:
+            tokenizer_kwargs = {"add_space_before_punct_symbol": True}
+        else:
+            tokenizer_kwargs = {}
+
         encoded_prompt = tokenizer.encode(
-            preprocessed_prompt_text, add_special_tokens=False, return_tensors="pt", add_space_before_punct_symbol=True
+            preprocessed_prompt_text, add_special_tokens=False, return_tensors="pt", **tokenizer_kwargs
         )
     else:
         encoded_prompt = tokenizer.encode(prompt_text, add_special_tokens=False, return_tensors="pt")

examples/token-classification/run_tf_ner.py

@@ -17,6 +17,7 @@
 
 import logging
 import os
+import warnings
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Tuple
 
@@ -184,7 +185,12 @@ def main():
 
         for i in range(batch_size):
             for j in range(seq_len):
-                if label_ids[i, j] != -1:
+                if label_ids[i, j] == -1:
+                    label_ids[i, j] = -100
+                    warnings.warn(
+                        "Using `-1` to mask the loss for the token is depreciated. Please use `-100` instead."
+                    )
+                if label_ids[i, j] != -100:
                     out_label_list[i].append(label_map[label_ids[i][j]])
                     preds_list[i].append(label_map[preds[i][j]])
 

model_cards/MoseliMotsoehli/TswanaBert/README.md

@@ -3,12 +3,13 @@ language: setswana
 ---
 
 # TswanaBert
+Pretrained model on the Tswana language using a masked language modeling (MLM) objective.
 
 ## Model Description.
-TswanaBERT is a transformers model pretrained on a corpus of Setswana data in a self-supervised fashion by masking part of the input words and training to predict the masks.
+TswanaBERT is a transformer model pre-trained on a corpus of Setswana in a self-supervised fashion by masking part of the input words and training to predict the masks by using byte-level tokens.
 
 ## Intended uses & limitations
-The model can  be used for either masked language modeling or next word prediction. it can also be fine-tuned for a specifict application. 
+The model can  be used for either masked language modeling or next word prediction. It can also be fine-tuned on a specific down-stream NLP application. 
 
 #### How to use
 
@@ -44,13 +45,15 @@ The model can  be used for either masked language modeling or next word predicti
 ```
 
 #### Limitations and bias
-The model is trained on a fairly small collection of setwana, mostly from news articles and creative writtings, and so is not representative enough of the language as yet.
+The model is trained on a relatively small collection of setwana, mostly from news articles and creative writtings, and so is not representative enough of the language as yet.
 
 ## Training data
 
-The largest portion of this dataset (10k)  lines of text, comes from the [Leipzig Corpora Collection](https://wortschatz.uni-leipzig.de/en/download)
+1. The largest portion of this dataset (10k)  sentences of text, comes from the [Leipzig Corpora Collection](https://wortschatz.uni-leipzig.de/en/download)
 
-The I then added 200 more phrases and sentences by scrapping following sites. I continue to expand the dataset
+2. I Then added SABC news headlines collected by Marivate Vukosi, & Sefara Tshephisho, (2020)  that is generously made available on [zenoodo](http://doi.org/10.5281/zenodo.3668495 ). This added 185 tswana sentences to my corpus. 
+
+3. I went on to add 300 more sentences by scrapping following news sites and blogs that mosty originate in Botswana. I actively continue to expand the dataset.
 
 * http://setswana.blogspot.com/
 * https://omniglot.com/writing/tswana.php
@@ -58,10 +61,9 @@ The I then added 200 more phrases and sentences by scrapping following sites. I
 * http://www.mmegi.bw/index.php
 * https://tsena.co.bw
 * http://www.botswana.co.za/Cultural_Issues-travel/botswana-country-guide-en-route.html
-
-## Training procedure
-The model was trained on a google colab Tesla T4 GPU for 200 epochs with a batch size of 64, on 13446 learned tokens.
-Other model training configuration setting can be found [here](https://s3.amazonaws.com/models.huggingface.co/bert/MoseliMotsoehli/TswanaBert/config.json)
+* https://www.poemhunter.com/poem/2013-setswana/
+https://www.poemhunter.com/poem/ngwana-wa-mosetsana/
+ 
 
 ### BibTeX entry and citation info
 

model_cards/MoseliMotsoehli/zuBERTa/README.md

@@ -0,0 +1,56 @@
+---
+language: zulu
+---
+
+# zuBERTa
+zuBERTa is a RoBERTa style transformer language model trained on zulu text.
+
+## Intended uses & limitations
+The model can be used for getting embeddings to use on a down-stream task such as question answering.
+
+#### How to use
+
+```python
+>>> from transformers import pipeline
+>>> from transformers import AutoTokenizer, AutoModelWithLMHead
+
+>>> tokenizer = AutoTokenizer.from_pretrained("MoseliMotsoehli/zuBERTa")
+>>> model = AutoModelWithLMHead.from_pretrained("MoseliMotsoehli/zuBERTa")
+>>> unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer)
+>>> unmasker("Abafika eNkandla bafika sebeholwa <mask> uMpongo kaZingelwayo.")
+
+[
+  {
+    "sequence": "<s>Abafika eNkandla bafika sebeholwa khona uMpongo kaZingelwayo.</s>",
+    "score": 0.050459690392017365,
+    "token": 555,
+    "token_str": "Ġkhona"
+  },
+  {
+    "sequence": "<s>Abafika eNkandla bafika sebeholwa inkosi uMpongo kaZingelwayo.</s>",
+    "score": 0.03668094798922539,
+    "token": 2321,
+    "token_str": "Ġinkosi"
+  },
+  {
+    "sequence": "<s>Abafika eNkandla bafika sebeholwa ubukhosi uMpongo kaZingelwayo.</s>",
+    "score": 0.028774697333574295,
+    "token": 5101,
+    "token_str": "Ġubukhosi"
+  }
+]
+```
+
+## Training data
+
+1. 30k sentences of text, came from the [Leipzig Corpora Collection](https://wortschatz.uni-leipzig.de/en/download) of zulu 2018. These were collected from news articles and creative writtings. 
+2. ~7500 articles of human generated translations were scraped from the zulu [wikipedia](https://zu.wikipedia.org/wiki/Special:AllPages).
+
+### BibTeX entry and citation info
+
+```bibtex
+@inproceedings{author = {Moseli Motsoehli},
+  title = {Towards transformation of Southern African language models through transformers.},
+  year={2020}
+}
+```

model_cards/bashar-talafha/multi-dialect-bert-base-arabic/README.md

@@ -0,0 +1,69 @@
+---
+language: arabic
+thumbnail: https://raw.githubusercontent.com/mawdoo3/Multi-dialect-Arabic-BERT/master/multidialct_arabic_bert.png
+datasets:
+- nadi
+---
+# Multi-dialect-Arabic-BERT
+This is a repository of Multi-dialect Arabic BERT model.
+
+By [Mawdoo3-AI](https://ai.mawdoo3.com/). 
+
+<p align="center">
+    <br>
+    <img src="https://raw.githubusercontent.com/mawdoo3/Multi-dialect-Arabic-BERT/master/multidialct_arabic_bert.png" alt="Background reference: http://www.qfi.org/wp-content/uploads/2018/02/Qfi_Infographic_Mother-Language_Final.pdf" width="500"/>
+    <br>
+<p>
+
+
+
+### About our Multi-dialect-Arabic-BERT model
+Instead of training the Multi-dialect Arabic BERT model from scratch, we initialized the weights of the model using [Arabic-BERT](https://github.com/alisafaya/Arabic-BERT) and trained it on 10M arabic tweets from the unlabled data of [The Nuanced Arabic Dialect Identification (NADI) shared task](https://sites.google.com/view/nadi-shared-task).
+
+### To cite this work
+Please cite this paper for now:
+```
+@inproceedings{talafha-etal-2020-nadi,
+    title ={{Multi-dialect Arabic BERT for Country-level Dialect Identification}},
+    author = {Talafha, Bashar, Ali, Mohammad, Za'ter, Muhy Eddin, Seelawi, Haitham, Tuffaha, Ibraheem, Samir, Mostafa, Farhan, Wael and Al-Natsheh, Hussein},
+    booktitle ={{Proceedings of the Fifth Arabic Natural Language Processing Workshop (WANLP2020)}},
+    year = {2020},
+    address = {Barcelona, Spain}
+}
+```
+We will update the BibTeX once the paper published.
+
+### Usage
+The model weights can be loaded using `transformers` library by HuggingFace.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("bashar-talafha/multi-dialect-bert-base-arabic")
+model = AutoModel.from_pretrained("bashar-talafha/multi-dialect-bert-base-arabic")
+```
+
+Example using `pipeline`:
+
+```python
+from transformers import pipeline
+
+fill_mask = pipeline(
+    "fill-mask",
+    model="bashar-talafha/multi-dialect-bert-base-arabic ",
+    tokenizer="bashar-talafha/multi-dialect-bert-base-arabic "
+)
+
+fill_mask(" سافر الرحالة من مطار [MASK] ")
+```
+```
+[{'sequence': '[CLS] سافر الرحالة من مطار الكويت [SEP]', 'score': 0.08296813815832138, 'token': 3226},
+ {'sequence': '[CLS] سافر الرحالة من مطار دبي [SEP]', 'score': 0.05123933032155037, 'token': 4747},
+ {'sequence': '[CLS] سافر الرحالة من مطار مسقط [SEP]', 'score': 0.046838656067848206, 'token': 13205},
+ {'sequence': '[CLS] سافر الرحالة من مطار القاهرة [SEP]', 'score': 0.03234650194644928, 'token': 4003},
+ {'sequence': '[CLS] سافر الرحالة من مطار الرياض [SEP]', 'score': 0.02606341242790222, 'token': 2200}]
+```
+### Repository
+Please check the [original repository](https://github.com/mawdoo3/Multi-dialect-Arabic-BERT) for more information. 
+
+

model_cards/chrisliu298/arxiv_ai_gpt2/README.md

@@ -18,7 +18,7 @@ This GPT-2 (774M) model is capable of generating abstracts given paper titles. I
 
 #### How to use
 
-To generate paper abstracts, use the provided `generate.py` [here](https://gist.github.com/chrisliu298/ccb8144888eace069da64ad3e6472d64). This is very similar to the HuggingFace's `run_generation.py` [here](https://github.com/huggingface/transformers/tree/master/examples/text-generation). You can simply replace the text with with your own model path (line 89) and change the input string to your paper title (line 127).
+To generate paper abstracts, use the provided `generate.py` [here](https://gist.github.com/chrisliu298/ccb8144888eace069da64ad3e6472d64). This is very similar to the HuggingFace's `run_generation.py` [here](https://github.com/huggingface/transformers/tree/master/examples/text-generation). You can simply replace the text with with your own model path (line 89) and change the input string to your paper title (line 127). If you want to use your own script, make sure to prepend `<|startoftext|> ` at the front and append ` <|sep|>` at the end of the paper title.
 
 ## Training data
 I selected a subset of the [arXiv Archive](https://github.com/staeiou/arxiv_archive) dataset (Geiger, 2019) as the training and evaluation data to fine-tune GPT-2. The original arXiv Archive dataset contains a full archive of metadata about papers on arxiv.org, from the start of the site in 1993 to the end of 2019. Our subset includes all the paper titles (query) and abstracts (context) under the Artificial Intelligence (cs.AI), Machine Learning (cs.LG), Computation and Language (cs.CL), and Computer Vision and Pattern Recognition (cs.CV) categories. I provide the information  of the sub-dataset and the distribution of the training and evaluation dataset as follows.

model_cards/codegram/calbert-base-uncased/README.md

@@ -1,25 +1,91 @@
 ---
-language: catalan
+language: "ca"
+tags:
+  - lm-head
+  - masked-lm
+  - catalan
+  - exbert
+license: mit
 ---
 
-# CALBERT: a Catalan Language Model
+# Calbert: a Catalan Language Model
 
 ## Introduction
 
-CALBERT is an open-source language model for Catalan based on the ALBERT architecture. 
+CALBERT is an open-source language model for Catalan pretrained on the ALBERT architecture.
 
-It is now available on Hugging Face in its `base-uncased` version, and was pretrained on the [OSCAR dataset](https://traces1.inria.fr/oscar/).
+It is now available on Hugging Face in its `tiny-uncased` version and `base-uncased` (the one you're looking at) as well, and was pretrained on the [OSCAR dataset](https://traces1.inria.fr/oscar/).
 
 For further information or requests, please go to the [GitHub repository](https://github.com/codegram/calbert)
 
 ## Pre-trained models
 
-| Model                               |  Arch.           | Training data                     |
-|-------------------------------------|------------------|-----------------------------------|
-| `codegram` / `calbert-base-uncased` |  Base (uncased)  | OSCAR (4.3 GB of text)            |
+| Model                               | Arch.          | Training data          |
+| ----------------------------------- | -------------- | ---------------------- |
+| `codegram` / `calbert-tiny-uncased` | Tiny (uncased) | OSCAR (4.3 GB of text) |
+| `codegram` / `calbert-base-uncased` | Base (uncased) | OSCAR (4.3 GB of text) |
 
+## How to use Calbert with HuggingFace
 
-## Authors 
+#### Load Calbert and its tokenizer:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("codegram/calbert-base-uncased")
+model = AutoModel.from_pretrained("codegram/calbert-base-uncased")
+
+model.eval() # disable dropout (or leave in train mode to finetune
+```
+
+#### Filling masks using pipeline
+
+```python
+from transformers import pipeline
+
+calbert_fill_mask  = pipeline("fill-mask", model="codegram/calbert-base-uncased", tokenizer="codegram/calbert-base-uncased")
+results = calbert_fill_mask("M'agrada [MASK] això")
+# results
+# [{'sequence': "[CLS] m'agrada molt aixo[SEP]", 'score': 0.614592969417572, 'token': 61},
+#  {'sequence': "[CLS] m'agrada moltíssim aixo[SEP]", 'score': 0.06058056280016899, 'token': 4867},
+#  {'sequence': "[CLS] m'agrada més aixo[SEP]", 'score': 0.017195818945765495, 'token': 43},
+#  {'sequence': "[CLS] m'agrada llegir aixo[SEP]", 'score': 0.016321714967489243, 'token': 684},
+#  {'sequence': "[CLS] m'agrada escriure aixo[SEP]", 'score': 0.012185849249362946, 'token': 1306}]
+
+```
+
+#### Extract contextual embedding features from Calbert output
+
+```python
+import torch
+# Tokenize in sub-words with SentencePiece
+tokenized_sentence = tokenizer.tokenize("M'és una mica igual")
+# ['▁m', "'", 'es', '▁una', '▁mica', '▁igual']
+
+# 1-hot encode and add special starting and end tokens
+encoded_sentence = tokenizer.encode(tokenized_sentence)
+# [2, 109, 7, 71, 36, 371, 1103, 3]
+# NB: Can be done in one step : tokenize.encode("M'és una mica igual")
+
+# Feed tokens to Calbert as a torch tensor (batch dim 1)
+encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0)
+embeddings, _ = model(encoded_sentence)
+embeddings.size()
+# torch.Size([1, 8, 768])
+embeddings.detach()
+# tensor([[[-0.0261,  0.1166, -0.1075,  ..., -0.0368,  0.0193,  0.0017],
+#          [ 0.1289, -0.2252,  0.9881,  ..., -0.1353,  0.3534,  0.0734],
+#          [-0.0328, -1.2364,  0.9466,  ...,  0.3455,  0.7010, -0.2085],
+#          ...,
+#          [ 0.0397, -1.0228, -0.2239,  ...,  0.2932,  0.1248,  0.0813],
+#          [-0.0261,  0.1165, -0.1074,  ..., -0.0368,  0.0193,  0.0017],
+#          [-0.1934, -0.2357, -0.2554,  ...,  0.1831,  0.6085,  0.1421]]])
+```
+
+## Authors
 
 CALBERT was trained and evaluated by [Txus Bach](https://twitter.com/txustice), as part of [Codegram](https://www.codegram.com)'s applied research.
 
+<a href="https://huggingface.co/exbert/?model=codegram/calbert-base-uncased&modelKind=bidirectional&sentence=M%27agradaria%20força%20saber-ne%20més">
+	<img width="300px" src="https://hf-dinosaur.huggingface.co/exbert/button.png">
+</a>

model_cards/codegram/calbert-tiny-uncased/README.md

@@ -0,0 +1,91 @@
+---
+language: "ca"
+tags:
+  - lm-head
+  - masked-lm
+  - catalan
+  - exbert
+license: mit
+---
+
+# Calbert: a Catalan Language Model
+
+## Introduction
+
+CALBERT is an open-source language model for Catalan pretrained on the ALBERT architecture.
+
+It is now available on Hugging Face in its `tiny-uncased` version (the one you're looking at) and `base-uncased` as well, and was pretrained on the [OSCAR dataset](https://traces1.inria.fr/oscar/).
+
+For further information or requests, please go to the [GitHub repository](https://github.com/codegram/calbert)
+
+## Pre-trained models
+
+| Model                               | Arch.          | Training data          |
+| ----------------------------------- | -------------- | ---------------------- |
+| `codegram` / `calbert-tiny-uncased` | Tiny (uncased) | OSCAR (4.3 GB of text) |
+| `codegram` / `calbert-base-uncased` | Base (uncased) | OSCAR (4.3 GB of text) |
+
+## How to use Calbert with HuggingFace
+
+#### Load Calbert and its tokenizer:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("codegram/calbert-tiny-uncased")
+model = AutoModel.from_pretrained("codegram/calbert-tiny-uncased")
+
+model.eval() # disable dropout (or leave in train mode to finetune
+```
+
+#### Filling masks using pipeline
+
+```python
+from transformers import pipeline
+
+calbert_fill_mask  = pipeline("fill-mask", model="codegram/calbert-tiny-uncased", tokenizer="codegram/calbert-tiny-uncased")
+results = calbert_fill_mask("M'agrada [MASK] això")
+# results
+# [{'sequence': "[CLS] m'agrada molt aixo[SEP]", 'score': 0.4403671622276306, 'token': 61},
+#  {'sequence': "[CLS] m'agrada més aixo[SEP]", 'score': 0.050061386078596115, 'token': 43},
+#  {'sequence': "[CLS] m'agrada veure aixo[SEP]", 'score': 0.026286985725164413, 'token': 157},
+#  {'sequence': "[CLS] m'agrada bastant aixo[SEP]", 'score': 0.022483550012111664, 'token': 2143},
+#  {'sequence': "[CLS] m'agrada moltíssim aixo[SEP]", 'score': 0.014491282403469086, 'token': 4867}]
+
+```
+
+#### Extract contextual embedding features from Calbert output
+
+```python
+import torch
+# Tokenize in sub-words with SentencePiece
+tokenized_sentence = tokenizer.tokenize("M'és una mica igual")
+# ['▁m', "'", 'es', '▁una', '▁mica', '▁igual']
+
+# 1-hot encode and add special starting and end tokens
+encoded_sentence = tokenizer.encode(tokenized_sentence)
+# [2, 109, 7, 71, 36, 371, 1103, 3]
+# NB: Can be done in one step : tokenize.encode("M'és una mica igual")
+
+# Feed tokens to Calbert as a torch tensor (batch dim 1)
+encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0)
+embeddings, _ = model(encoded_sentence)
+embeddings.size()
+# torch.Size([1, 8, 312])
+embeddings.detach()
+# tensor([[[-0.2726, -0.9855,  0.9643,  ...,  0.3511,  0.3499, -0.1984],
+#         [-0.2824, -1.1693, -0.2365,  ..., -3.1866, -0.9386, -1.3718],
+#         [-2.3645, -2.2477, -1.6985,  ..., -1.4606, -2.7294,  0.2495],
+#         ...,
+#         [ 0.8800, -0.0244, -3.0446,  ...,  0.5148, -3.0903,  1.1879],
+#         [ 1.1300,  0.2425,  0.2162,  ..., -0.5722, -2.2004,  0.4045],
+#         [ 0.4549, -0.2378, -0.2290,  ..., -2.1247, -2.2769, -0.0820]]])
+```
+
+## Authors
+
+CALBERT was trained and evaluated by [Txus Bach](https://twitter.com/txustice), as part of [Codegram](https://www.codegram.com)'s applied research.
+
+<a href="https://huggingface.co/exbert/?model=codegram/calbert-tiny-uncased&modelKind=bidirectional&sentence=M%27agradaria%20força%20saber-ne%20més">
+	<img width="300px" src="https://hf-dinosaur.huggingface.co/exbert/button.png">
+</a>

model_cards/deepset/electra-base-squad2/README.md

@@ -0,0 +1,117 @@
+---
+datasets:
+- squad_v2
+---
+
+# electra-base for QA
+
+## Overview
+**Language model:** electra-base  
+**Language:** English  
+**Downstream-task:** Extractive QA  
+**Training data:** SQuAD 2.0  
+**Eval data:** SQuAD 2.0  
+**Code:**  See [example](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) in [FARM](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py)  
+**Infrastructure**: 1x Tesla v100
+
+## Hyperparameters
+
+```
+seed=42
+batch_size = 32
+n_epochs = 5
+base_LM_model = "google/electra-base-discriminator"
+max_seq_len = 384
+learning_rate = 1e-4
+lr_schedule = LinearWarmup
+warmup_proportion = 0.1
+doc_stride=128
+max_query_length=64
+```
+
+## Performance
+Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/).
+```
+"exact": 77.30144024256717,
+ "f1": 81.35438272008543,
+ "total": 11873,
+ "HasAns_exact": 74.34210526315789,
+ "HasAns_f1": 82.45961302894314,
+ "HasAns_total": 5928,
+ "NoAns_exact": 80.25231286795626,
+ "NoAns_f1": 80.25231286795626,
+ "NoAns_total": 5945
+```
+
+## Usage
+
+### In Transformers
+```python
+from transformers.pipelines import pipeline
+from transformers.modeling_auto import AutoModelForQuestionAnswering
+from transformers.tokenization_auto import AutoTokenizer
+
+model_name = "deepset/electra-base-squad2"
+
+# a) Get predictions
+nlp = pipeline('question-answering', model=model_name, tokenizer=model_name)
+QA_input = {
+    'question': 'Why is model conversion important?',
+    'context': 'The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks.'
+}
+res = nlp(QA_input)
+
+# b) Load model & tokenizer
+model = AutoModelForQuestionAnswering.from_pretrained(model_name)
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+```
+
+### In FARM
+
+```python
+from farm.modeling.adaptive_model import AdaptiveModel
+from farm.modeling.tokenization import Tokenizer
+from farm.infer import Inferencer
+
+model_name = "deepset/electra-base-squad2"
+
+# a) Get predictions
+nlp = Inferencer.load(model_name, task_type="question_answering")
+QA_input = [{"questions": ["Why is model conversion important?"],
+             "text": "The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."}]
+res = nlp.inference_from_dicts(dicts=QA_input)
+
+# b) Load model & tokenizer
+model = AdaptiveModel.convert_from_transformers(model_name, device="cpu", task_type="question_answering")
+tokenizer = Tokenizer.load(model_name)
+```
+
+### In haystack
+For doing QA at scale (i.e. many docs instead of single paragraph), you can load the model also in [haystack](https://github.com/deepset-ai/haystack/):
+```python
+reader = FARMReader(model_name_or_path="deepset/electra-base-squad2")
+# or
+reader = TransformersReader(model="deepset/electra-base-squad2",tokenizer="deepset/electra-base-squad2")
+```
+
+
+## Authors
+Vaishali Pal `vaishali.pal [at] deepset.ai`
+Branden Chan: `branden.chan [at] deepset.ai`
+Timo Möller: `timo.moeller [at] deepset.ai`
+Malte Pietsch: `malte.pietsch [at] deepset.ai`
+Tanay Soni: `tanay.soni [at] deepset.ai`
+
+## About us
+![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png)
+
+We bring NLP to the industry via open source!  
+Our focus: Industry specific language models & large scale QA systems.  
+
+Some of our work:
+- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert)
+- [FARM](https://github.com/deepset-ai/FARM)
+- [Haystack](https://github.com/deepset-ai/haystack/)
+
+Get in touch:
+[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai)

model_cards/gilf/french-postag-model/README.md

@@ -0,0 +1,90 @@
+## About
+
+The  *french-postag-model* is a part of speech tagging model for French that was trained on the *free-french-treebank* dataset available on 
+[github](https://github.com/nicolashernandez/free-french-treebank). The base tokenizer and model used for training is *'bert-base-multilingual-cased'*.
+
+## Supported Tags
+
+It uses the following tags:
+
+| Tag      |          Category              |  Extra Info |
+|----------|:------------------------------:|------------:|
+| ADJ      |           adjectif             |             |
+| ADJWH    |           adjectif             |             |
+| ADV      |           adverbe              |             |
+| ADVWH    |           adverbe              |             |
+| CC       |  conjonction de coordination   |             |
+| CLO      |            pronom              |     obj     |
+| CLR      |            pronom              |     refl    |
+| CLS      |            pronom              |     suj     |
+| CS       |  conjonction de subordination  |             |
+| DET      |          déterminant           |             |
+| DETWH    |          déterminant           |             |
+| ET       |          mot étranger          |             |
+| I        |          interjection          |             |
+| NC       |          nom commun            |             |
+| NPP      |          nom propre            |             |
+| P        |          préposition           |             |
+| P+D      |   préposition + déterminant    |             |
+| PONCT    |      signe de ponctuation      |             |
+| PREF     |            préfixe             |             |
+| PRO      |        autres pronoms          |             |
+| PROREL   |        autres pronoms          |     rel     |
+| PROWH    |        autres pronoms          |     int     |
+| U        |               ?                |             |
+| V        |             verbe              |             |
+| VIMP     |        verbe imperatif         |             |
+| VINF     |        verbe infinitif         |             |
+| VPP      |        participe passé         |             |
+| VPR      |        participe présent       |             |
+| VS       |        subjonctif              |             |
+
+More information on the tags can be found here:
+
+http://alpage.inria.fr/statgram/frdep/Publications/crabbecandi-taln2008-final.pdf
+
+## Usage
+
+The usage of this model follows the common transformers patterns. Here is a short example of its usage:
+
+```python
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+
+tokenizer = AutoTokenizer.from_pretrained("gilf/french-postag-model")
+model = AutoModelForTokenClassification.from_pretrained("gilf/french-postag-model")
+
+from transformers import pipeline
+
+nlp_token_class = pipeline('ner', model=model, tokenizer=tokenizer, grouped_entities=True)
+
+nlp_token_class('Face à un choc inédit, les mesures mises en place par le gouvernement ont permis une protection forte et efficace des ménages')
+```
+
+The lines above would display something like this on a Jupyter notebook:
+
+```
+[{'entity_group': 'PONCT', 'score': 0.0742340236902237, 'word': '[CLS]'},
+ {'entity_group': 'U', 'score': 0.9995399713516235, 'word': 'Face'},
+ {'entity_group': 'P', 'score': 0.9999609589576721, 'word': 'à'},
+ {'entity_group': 'DET', 'score': 0.9999597072601318, 'word': 'un'},
+ {'entity_group': 'NC', 'score': 0.9998948276042938, 'word': 'choc'},
+ {'entity_group': 'ADJ', 'score': 0.995318204164505, 'word': 'inédit'},
+ {'entity_group': 'PONCT', 'score': 0.9999793171882629, 'word': ','},
+ {'entity_group': 'DET', 'score': 0.999964714050293, 'word': 'les'},
+ {'entity_group': 'NC', 'score': 0.999936580657959, 'word': 'mesures'},
+ {'entity_group': 'VPP', 'score': 0.9995776414871216, 'word': 'mises'},
+ {'entity_group': 'P', 'score': 0.99996417760849, 'word': 'en'},
+ {'entity_group': 'NC', 'score': 0.999882161617279, 'word': 'place'},
+ {'entity_group': 'P', 'score': 0.9999671578407288, 'word': 'par'},
+ {'entity_group': 'DET', 'score': 0.9999637603759766, 'word': 'le'},
+ {'entity_group': 'NC', 'score': 0.9999350309371948, 'word': 'gouvernement'},
+ {'entity_group': 'V', 'score': 0.9999298453330994, 'word': 'ont'},
+ {'entity_group': 'VPP', 'score': 0.9998740553855896, 'word': 'permis'},
+ {'entity_group': 'DET', 'score': 0.9999625086784363, 'word': 'une'},
+ {'entity_group': 'NC', 'score': 0.9999420046806335, 'word': 'protection'},
+ {'entity_group': 'ADJ', 'score': 0.9998913407325745, 'word': 'forte'},
+ {'entity_group': 'CC', 'score': 0.9998615980148315, 'word': 'et'},
+ {'entity_group': 'ADJ', 'score': 0.9998483657836914, 'word': 'efficace'},
+ {'entity_group': 'P+D', 'score': 0.9987645149230957, 'word': 'des'},
+ {'entity_group': 'NC', 'score': 0.8720395267009735, 'word': 'ménages [SEP]'}]
+```

model_cards/microsoft/codebert-base-mlm/README.md

@@ -0,0 +1,46 @@
+## CodeBERT-base-mlm
+Pretrained weights for [CodeBERT: A Pre-Trained Model for Programming and Natural Languages](https://arxiv.org/abs/2002.08155).
+
+### Training Data
+The model is trained on the code corpus of [CodeSearchNet](https://github.com/github/CodeSearchNet)
+
+### Training Objective
+This model is initialized with Roberta-base and trained with a simple MLM (Masked Language Model) objective.
+
+### Usage
+```python
+from transformers import RobertaTokenizer, RobertaForMaskedLM, pipeline
+
+model = RobertaForMaskedLM.from_pretrained('microsoft/codebert-base-mlm')
+tokenizer = RobertaTokenizer.from_pretrained('microsoft/codebert-base-mlm')
+
+code_example = "if (x is not None) <mask> (x>1)"
+fill_mask = pipeline('fill-mask', model=model, tokenizer=tokenizer)
+
+outputs = fill_mask(code_example)
+print(outputs)
+```
+Expected results:
+```
+{'sequence': '<s> if (x is not None) and (x>1)</s>', 'score': 0.6049249172210693, 'token': 8}
+{'sequence': '<s> if (x is not None) or (x>1)</s>', 'score': 0.30680200457572937, 'token': 50}
+{'sequence': '<s> if (x is not None) if (x>1)</s>', 'score': 0.02133703976869583, 'token': 114}
+{'sequence': '<s> if (x is not None) then (x>1)</s>', 'score': 0.018607674166560173, 'token': 172}
+{'sequence': '<s> if (x is not None) AND (x>1)</s>', 'score': 0.007619690150022507, 'token': 4248}
+```
+
+### Reference
+1. [Bimodal CodeBERT trained with MLM+RTD objective](https://huggingface.co/microsoft/codebert-base) (suitable for code search and document generation)
+2. 🤗 [Hugging Face's CodeBERTa](https://huggingface.co/huggingface/CodeBERTa-small-v1) (small size, 6 layers)
+
+### Citation
+```bibtex
+@misc{feng2020codebert,
+    title={CodeBERT: A Pre-Trained Model for Programming and Natural Languages},
+    author={Zhangyin Feng and Daya Guo and Duyu Tang and Nan Duan and Xiaocheng Feng and Ming Gong and Linjun Shou and Bing Qin and Ting Liu and Daxin Jiang and Ming Zhou},
+    year={2020},
+    eprint={2002.08155},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+```

model_cards/microsoft/codebert-base/README.md

@@ -0,0 +1,27 @@
+## CodeBERT-base
+Pretrained weights for [CodeBERT: A Pre-Trained Model for Programming and Natural Languages](https://arxiv.org/abs/2002.08155).
+
+### Training Data
+The model is trained on bi-modal data (documents & code) of [CodeSearchNet](https://github.com/github/CodeSearchNet)
+
+### Training Objective
+This model is initialized with Roberta-base and trained with MLM+RTD objective (cf. the paper).
+
+### Usage
+Please see [the official repository](https://github.com/microsoft/CodeBERT) for scripts that support "code search" and "code-to-document generation".
+
+### Reference
+1. [CodeBERT trained with Masked LM objective](https://huggingface.co/microsoft/codebert-base-mlm) (suitable for code completion)
+2. 🤗 [Hugging Face's CodeBERTa](https://huggingface.co/huggingface/CodeBERTa-small-v1) (small size, 6 layers)
+
+### Citation
+```bibtex
+@misc{feng2020codebert,
+    title={CodeBERT: A Pre-Trained Model for Programming and Natural Languages},
+    author={Zhangyin Feng and Daya Guo and Duyu Tang and Nan Duan and Xiaocheng Feng and Ming Gong and Linjun Shou and Bing Qin and Ting Liu and Daxin Jiang and Ming Zhou},
+    year={2020},
+    eprint={2002.08155},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+}
+```

model_cards/mrm8488/electra-small-finetuned-squadv1/README.md

@@ -0,0 +1,87 @@
+---
+language: english
+---
+
+# Electra small ⚡ + SQuAD v1 ❓
+
+[Electra-small-discriminator](https://huggingface.co/google/electra-small-discriminator) fine-tuned on [SQUAD v1.1 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) for **Q&A** downstream task.
+
+## Details of the downstream task (Q&A) - Model 🧠
+
+**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset.
+
+
+## Details of the downstream task (Q&A) - Dataset 📚
+
+**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.
+SQuAD v1.1 contains **100,000+** question-answer pairs on **500+** articles.
+
+## Model training 🏋️‍
+
+The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
+
+```bash
+python transformers/examples/question-answering/run_squad.py \
+  --model_type electra \
+  --model_name_or_path 'google/electra-small-discriminator' \
+  --do_eval \
+  --do_train \
+  --do_lower_case \
+  --train_file '/content/dataset/train-v1.1.json' \
+  --predict_file '/content/dataset/dev-v1.1.json' \
+  --per_gpu_train_batch_size 16 \
+  --learning_rate 3e-5 \
+  --num_train_epochs 10 \
+  --max_seq_length 384 \
+  --doc_stride 128 \
+  --output_dir '/content/output' \
+  --overwrite_output_dir \
+  --save_steps 1000
+```
+
+## Test set Results 🧾
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **77.70** |
+| **F1** | **85.74** |
+| **Size**| **50 MB** |
+
+Very good metrics for such a "small" model!
+
+```json
+
+{
+'exact': 77.70104068117313,
+'f1': 85.73991234187997,
+'total': 10570,
+'HasAns_exact': 77.70104068117313,
+'HasAns_f1': 85.73991234187997,
+'HasAns_total': 10570,
+'best_exact': 77.70104068117313,
+'best_exact_thresh': 0.0,
+'best_f1': 85.73991234187997,
+'best_f1_thresh': 0.0
+}
+```
+
+### Model in action 🚀
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import pipeline
+
+QnA_pipeline = pipeline('question-answering', model='mrm8488/electra-small-finetuned-squadv1')
+QnA_pipeline({
+    'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.',
+    'question': 'What has been discovered by scientists from China ?'
+})
+
+# Output:
+{'answer': 'A new strain of flu', 'end': 19, 'score': 0.7950334108113424, 'start': 0}
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/electra-small-finetuned-squadv2/README.md

@@ -0,0 +1,86 @@
+---
+language: english
+---
+
+# Electra small ⚡ + SQuAD v2 ❓
+
+[Electra-small-discriminator](https://huggingface.co/google/electra-small-discriminator) fine-tuned on [SQUAD v2.0 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/) for **Q&A** downstream task.
+
+## Details of the downstream task (Q&A) - Model 🧠
+
+**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset.
+
+
+## Details of the downstream task (Q&A) - Dataset 📚
+
+**SQuAD2.0** combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering.
+
+## Model training 🏋️‍
+
+The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
+
+```bash
+python transformers/examples/question-answering/run_squad.py \
+  --model_type electra \
+  --model_name_or_path 'google/electra-small-discriminator' \
+  --do_eval \
+  --do_train \
+  --do_lower_case \
+  --train_file '/content/dataset/train-v2.0.json' \
+  --predict_file '/content/dataset/dev-v2.0.json' \
+  --per_gpu_train_batch_size 16 \
+  --learning_rate 3e-5 \
+  --num_train_epochs 10 \
+  --max_seq_length 384 \
+  --doc_stride 128 \
+  --output_dir '/content/output' \
+  --overwrite_output_dir \
+  --save_steps 1000 \
+  --version_2_with_negative
+```
+
+## Test set Results 🧾
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **69.71** |
+| **F1** | **73.44** |
+| **Size**| **50 MB** |
+
+
+```json
+{
+'exact': 69.71279373368147,
+'f1': 73.4439546123672,
+'total': 11873,
+'HasAns_exact': 69.92240215924427,
+'HasAns_f1': 77.39542393937836,
+'HasAns_total': 5928,
+'NoAns_exact': 69.50378469301934,
+'NoAns_f1': 69.50378469301934,
+'NoAns_total': 5945,
+'best_exact': 69.71279373368147,
+'best_exact_thresh': 0.0,
+'best_f1': 73.44395461236732,
+'best_f1_thresh': 0.0
+}
+```
+
+### Model in action 🚀
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import pipeline
+QnA_pipeline = pipeline('question-answering', model='mrm8488/electra-base-finetuned-squadv2')
+QnA_pipeline({
+    'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.',
+    'question': 'What has been discovered by scientists from China ?'
+})
+# Output:
+{'answer': 'A new strain of flu', 'end': 19, 'score': 0.8650811568752914, 'start': 0}
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/electricidad-small-finetuned-squadv1-es/README.md

@@ -0,0 +1,102 @@
+---
+language: spanish
+thumbnail: https://imgur.com/uxAvBfh
+---
+
+# Electricidad small + Spanish SQuAD v1 ⚡❓
+
+[Electricidad-small-discriminator](https://huggingface.co/mrm8488/electricidad-small-discriminator) fine-tuned on [Spanish SQUAD v1.1 dataset](https://github.com/ccasimiro88/TranslateAlignRetrieve/tree/master/SQuAD-es-v1.1) for **Q&A** downstream task.
+
+## Details of the downstream task (Q&A) - Dataset 📚
+
+[SQuAD-es-v1.1](https://github.com/ccasimiro88/TranslateAlignRetrieve/tree/master/SQuAD-es-v1.1)
+
+| Dataset split | # Samples |
+| ------------- | --------- |
+| Train         | 130 K     |
+| Test          | 11 K      |
+
+## Model training 🏋️‍
+
+The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
+
+```bash
+python /content/transformers/examples/question-answering/run_squad.py \
+  --model_type electra \
+  --model_name_or_path 'mrm8488/electricidad-small-discriminator' \
+  --do_eval \
+  --do_train \
+  --do_lower_case \
+  --train_file '/content/dataset/train-v1.1-es.json' \
+  --predict_file '/content/dataset/dev-v1.1-es.json' \
+  --per_gpu_train_batch_size 16 \
+  --learning_rate 3e-5 \
+  --num_train_epochs 10 \
+  --max_seq_length 384 \
+  --doc_stride 128 \
+  --output_dir '/content/electricidad-small-finetuned-squadv1-es' \
+  --overwrite_output_dir \
+  --save_steps 1000
+```
+
+## Test set Results 🧾
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **46.82** |
+| **F1** | **64.79** |
+
+```json
+{
+'exact': 46.82119205298013,
+'f1': 64.79435260021918,
+'total': 10570,
+'HasAns_exact': 46.82119205298013,
+HasAns_f1': 64.79435260021918,
+'HasAns_total': 10570,
+'best_exact': 46.82119205298013,
+'best_exact_thresh': 0.0,
+'best_f1': 64.79435260021918,
+'best_f1_thresh': 0.0
+}
+```
+
+### Model in action 🚀
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import pipeline
+
+qa_pipeline = pipeline(
+    "question-answering",
+    model="mrm8488/electricidad-small-finetuned-squadv1-es",
+    tokenizer="mrm8488/electricidad-small-finetuned-squadv1-es"
+)
+
+context = "Manuel ha creado una versión del modelo Electra small en español que alcanza una puntuación F1 de 65 en el dataset SQUAD-es y sólo pesa 50 MB"
+
+q1 = "Cuál es su marcador F1?"
+q2 = "¿Cuál es el tamaño del modelo?"
+q3 = "¿Quién lo ha creado?"
+q4 = "¿Que es lo que ha hecho Manuel?"
+
+
+questions = [q1, q2, q3, q4]
+
+for question in questions:
+  result = qa_pipeline({
+    'context': context,
+    'question': question})
+  print(result)
+
+# Output:
+{'score': 0.14836778166355025, 'start': 98, 'end': 100, 'answer': '65'}
+{'score': 0.32219420810758237, 'start': 136, 'end': 140, 'answer': '50 MB'}
+{'score': 0.9672326951118713, 'start': 0, 'end': 6, 'answer': 'Manuel'}
+{'score': 0.23552458113848118, 'start': 10, 'end': 53, 'answer': 'creado una versión del modelo Electra small'}
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/roberta-base-1B-1-finetuned-squadv1/README.md

@@ -0,0 +1,88 @@
+---
+language: english
+---
+
+# RoBERTa-base (1B-1) + SQuAD v1 ❓
+
+[roberta-base-1B-1](https://huggingface.co/nyu-mll/roberta-base-1B-1) fine-tuned on [SQUAD v1.1 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) for **Q&A** downstream task.
+
+## Details of the downstream task (Q&A) - Model 🧠
+
+RoBERTa Pretrained on Smaller Datasets
+
+[NYU Machine Learning for Language](https://huggingface.co/nyu-mll) pretrained RoBERTa on smaller datasets (1M, 10M, 100M, 1B tokens). They released 3 models with lowest perplexities for each pretraining data size out of 25 runs (or 10 in the case of 1B tokens). The pretraining data reproduces that of BERT: They combine English Wikipedia and a reproduction of BookCorpus using texts from smashwords in a ratio of approximately 3:1.
+
+
+## Details of the downstream task (Q&A) - Dataset 📚
+
+**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.
+SQuAD v1.1 contains **100,000+** question-answer pairs on **500+** articles.
+
+## Model training 🏋️‍
+
+The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
+
+```bash
+python transformers/examples/question-answering/run_squad.py \
+  --model_type roberta \
+  --model_name_or_path 'nyu-mll/roberta-base-1B-1' \
+  --do_eval \
+  --do_train \
+  --do_lower_case \
+  --train_file /content/dataset/train-v1.1.json \
+  --predict_file /content/dataset/dev-v1.1.json \
+  --per_gpu_train_batch_size 16 \
+  --learning_rate 3e-5 \
+  --num_train_epochs 10 \
+  --max_seq_length 384 \
+  --doc_stride 128 \
+  --output_dir /content/output \
+  --overwrite_output_dir \
+  --save_steps 1000
+```
+
+## Test set Results 🧾
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **72.62** |
+| **F1** | **82.19** |
+
+
+
+```json
+{
+'exact': 72.62062440870388,
+'f1': 82.19430877136834,
+'total': 10570,
+'HasAns_exact': 72.62062440870388,
+'HasAns_f1': 82.19430877136834,
+'HasAns_total': 10570,
+'best_exact': 72.62062440870388,
+'best_exact_thresh': 0.0,
+'best_f1': 82.19430877136834,
+'best_f1_thresh': 0.0
+}
+
+```
+
+### Model in action 🚀
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import pipeline
+
+QnA_pipeline = pipeline('question-answering', model='mrm8488/roberta-base-1B-1-finetuned-squadv1')
+
+QnA_pipeline({
+    'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.',
+    'question': 'What has been discovered by scientists from China ?'
+})
+# Output:
+
+{'answer': 'A new strain of flu', 'end': 19, 'score': 0.04702283976040074, 'start': 0}
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/roberta-base-1B-1-finetuned-squadv2/README.md

@@ -0,0 +1,92 @@
+---
+language: english
+---
+
+# RoBERTa-base (1B-1) + SQuAD v2 ❓
+
+[roberta-base-1B-1](https://huggingface.co/nyu-mll/roberta-base-1B-1) fine-tuned on [SQUAD v2 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/) for **Q&A** downstream task.
+
+## Details of the downstream task (Q&A) - Model 🧠
+
+RoBERTa Pretrained on Smaller Datasets
+
+[NYU Machine Learning for Language](https://huggingface.co/nyu-mll) pretrained RoBERTa on smaller datasets (1M, 10M, 100M, 1B tokens). They released 3 models with lowest perplexities for each pretraining data size out of 25 runs (or 10 in the case of 1B tokens). The pretraining data reproduces that of BERT: They combine English Wikipedia and a reproduction of BookCorpus using texts from smashwords in a ratio of approximately 3:1.
+
+
+## Details of the downstream task (Q&A) - Dataset 📚
+
+**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.
+
+ **SQuAD2.0** combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering.
+
+## Model training 🏋️‍
+
+The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
+
+```bash
+python transformers/examples/question-answering/run_squad.py \
+  --model_type roberta \
+  --model_name_or_path 'nyu-mll/roberta-base-1B-1' \
+  --do_eval \
+  --do_train \
+  --do_lower_case \
+  --train_file /content/dataset/train-v2.0.json \
+  --predict_file /content/dataset/dev-v2.0.json \
+  --per_gpu_train_batch_size 16 \
+  --learning_rate 3e-5 \
+  --num_train_epochs 10 \
+  --max_seq_length 384 \
+  --doc_stride 128 \
+  --output_dir /content/output \
+  --overwrite_output_dir \
+  --save_steps 1000 \
+  --version_2_with_negative
+```
+
+## Test set Results 🧾
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **64.86** |
+| **F1** | **68.99** |
+
+
+
+```json
+{
+'exact': 64.86145034953255, 
+'f1': 68.9902640378272,
+'total': 11873,
+'HasAns_exact': 64.03508771929825,
+'HasAns_f1': 72.3045554860189,
+'HasAns_total': 5928,
+'NoAns_exact': 65.68544995794785,
+'NoAns_f1': 65.68544995794785,
+'NoAns_total': 5945,
+'best_exact': 64.86987282068559,
+'best_exact_thresh': 0.0,
+'best_f1': 68.99868650898054,
+'best_f1_thresh': 0.0
+}
+```
+
+### Model in action 🚀
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import pipeline
+
+QnA_pipeline = pipeline('question-answering', model='mrm8488/roberta-base-1B-1-finetuned-squadv2')
+
+QnA_pipeline({
+    'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.',
+    'question': 'What has been discovered by scientists from China ?'
+})
+# Output:
+
+{'answer': 'A new strain of flu', 'end': 19, 'score': 0.7145650685380576,'start': 0}
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/t5-small-finetuned-squadv1/README.md

@@ -0,0 +1,77 @@
+---
+language: english
+datasets:
+- squad
+---
+
+# T5-small fine-tuned on SQuAD
+
+[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) [(small)](https://huggingface.co/t5-small) fine-tuned on [SQuAD v1.1](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task.
+
+## Details of T5
+
+The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract:
+
+Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
+
+![model image](https://i.imgur.com/jVFMMWR.png)
+
+
+## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓
+
+Dataset ID: ```squad``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
+
+| Dataset  | Split | # samples |
+| -------- | ----- | --------- |
+| squad | train | 87599    |
+| squad | valid  | 10570    |
+
+How to load it from [nlp](https://github.com/huggingface/nlp)
+
+```python
+train_dataset  = nlp.load_dataset('squad, split=nlp.Split.TRAIN)
+valid_dataset = nlp.load_dataset('squad', split=nlp.Split.VALIDATION)
+```
+Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/)
+
+
+## Model fine-tuning 🏋️‍
+
+The training script is a slightly modified version of [this  awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28)
+
+## Results 📝
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **76.95** |
+| **F1** | **85.71** |
+
+
+## Model in Action 🚀
+
+```python
+from transformers import AutoModelWithLMHead, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-small-finetuned-squadv1")
+model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-small-finetuned-squadv1")
+
+def get_answer(question, context):
+  input_text = "question: %s  context: %s </s>" % (question, context)
+  features = tokenizer([input_text], return_tensors='pt')
+
+  output = model.generate(input_ids=features['input_ids'], 
+               attention_mask=features['attention_mask'])
+
+  return tokenizer.decode(output[0])
+
+context = "Manuel have created RuPERTa-base (a Spanish RoBERTa) with the support of HF-Transformers and Google"
+question = "Who has supported Manuel?"
+
+get_answer(question, context)
+
+# output: 'HF-Transformers and Google'
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/t5-small-finetuned-squadv2/README.md

@@ -0,0 +1,78 @@
+---
+language: english
+datasets:
+- squad_v2
+---
+
+# T5-small fine-tuned on SQuAD v2
+
+[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) [(small)](https://huggingface.co/t5-small) fine-tuned on [SQuAD v2](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task.
+
+## Details of T5
+
+The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract:
+
+Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
+
+![model image](https://i.imgur.com/jVFMMWR.png)
+
+
+## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓
+
+Dataset ID: ```squad_v2``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
+
+| Dataset  | Split | # samples |
+| -------- | ----- | --------- |
+| squad_v2 | train | 130319    |
+| squad_v2 | valid  | 11873    |
+
+How to load it from [nlp](https://github.com/huggingface/nlp)
+
+```python
+train_dataset  = nlp.load_dataset('squad_v2, split=nlp.Split.TRAIN)
+valid_dataset = nlp.load_dataset('squad_v2', split=nlp.Split.VALIDATION)
+```
+Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/)
+
+
+## Model fine-tuning 🏋️‍
+
+The training script is a slightly modified version of [this  awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28)
+
+## Results 📝
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **69.46** |
+| **F1** | **73.01** |
+
+
+
+## Model in Action 🚀
+
+```python
+from transformers import AutoModelWithLMHead, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-small-finetuned-squadv2")
+model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-small-finetuned-squadv2")
+
+def get_answer(question, context):
+  input_text = "question: %s  context: %s </s>" % (question, context)
+  features = tokenizer([input_text], return_tensors='pt')
+
+  output = model.generate(input_ids=features['input_ids'], 
+               attention_mask=features['attention_mask'])
+
+  return tokenizer.decode(output[0])
+
+context = "Manuel have created RuPERTa-base (a Spanish RoBERTa) with the support of HF-Transformers and Google"
+question = "Who has supported Manuel?"
+
+get_answer(question, context)
+
+# output: 'HF-Transformers and Google'
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/umberto-wikipedia-uncased-v1-finetuned-squadv1-it/README.md

@@ -0,0 +1,96 @@
+---
+language: italian
+---
+
+# UmBERTo Wikipedia Uncased +  italian SQuAD v1 📚 🧐 ❓
+
+[UmBERTo-Wikipedia-Uncased](https://huggingface.co/Musixmatch/umberto-wikipedia-uncased-v1) fine-tuned on [Italian SQUAD v1 dataset](https://github.com/crux82/squad-it) for **Q&A** downstream task.
+
+## Details of the downstream task (Q&A) - Model 🧠
+
+[UmBERTo](https://github.com/musixmatchresearch/umberto) is a Roberta-based Language Model trained on large Italian Corpora and uses two innovative approaches: SentencePiece and Whole Word Masking.
+UmBERTo-Wikipedia-Uncased Training is trained on a relative small corpus (~7GB) extracted from Wikipedia-ITA.
+
+
+## Details of the downstream task (Q&A) - Dataset 📚
+
+[SQuAD](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) [Rajpurkar et al. 2016] is a large scale dataset for training of question answering systems on factoid questions. It contains more than 100,000 question-answer pairs about passages from 536 articles chosen from various domains of Wikipedia.
+
+**SQuAD-it** is derived from the SQuAD dataset and it is obtained through semi-automatic translation of the SQuAD dataset into Italian. It represents a large-scale dataset for open question answering processes on factoid questions in Italian. The dataset contains more than 60,000 question/answer pairs derived from the original English dataset. 
+
+## Model training 🏋️‍
+
+The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
+
+```bash
+python transformers/examples/question-answering/run_squad.py \
+  --model_type bert \
+  --model_name_or_path 'Musixmatch/umberto-wikipedia-uncased-v1' \
+  --do_eval \
+  --do_train \
+  --do_lower_case \
+  --train_file '/content/dataset/SQuAD_it-train.json' \
+  --predict_file '/content/dataset/SQuAD_it-test.json' \
+  --per_gpu_train_batch_size 16 \
+  --learning_rate 3e-5 \
+  --num_train_epochs 10 \
+  --max_seq_length 384 \
+  --doc_stride 128 \
+  --output_dir /content/drive/My\ Drive/umberto-uncased-finetuned-squadv1-it \
+  --overwrite_output_dir \
+  --save_steps 1000
+```
+With 10 epochs the model overfits the train dataset so I evaluated the different checkpoints created during training (every 1000 steps) and chose the best (In this case the one created at 17000 steps).
+
+## Test set Results 🧾
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **60.50** |
+| **F1** | **72.41** |
+
+
+
+```json
+{
+'exact': 60.50729399395453,
+'f1': 72.4141113348361,
+'total': 7609,
+'HasAns_exact': 60.50729399395453,
+'HasAns_f1': 72.4141113348361,
+'HasAns_total': 7609,
+'best_exact': 60.50729399395453,
+'best_exact_thresh': 0.0,
+'best_f1': 72.4141113348361,
+'best_f1_thresh': 0.0
+}
+```
+
+## Comparison ⚖️
+
+| Model                                                                                                                            | EM        | F1 score  |
+| -------------------------------------------------------------------------------------------------------------------------------- | --------- | --------- |
+| [DrQA-it trained on SQuAD-it ](https://github.com/crux82/squad-it/blob/master/README.md#evaluating-a-neural-model-over-squad-it) | 56.1      | 65.9      |
+| This one                                                                                                                         |60.50      |72.41      |
+| [bert-italian-finedtuned-squadv1-it-alfa](https://huggingface.co/mrm8488/bert-italian-finedtuned-squadv1-it-alfa)                |**62.51**  |**74.16**  |                                                                                                                  | **62.51** | **74.16** |
+
+
+### Model in action 🚀
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import pipeline
+
+QnA_pipeline = pipeline('question-answering', model='mrm8488/umberto-wikipedia-uncased-v1-finetuned-squadv1-it')
+
+QnA_pipeline({
+    'context': 'Marco Aurelio era un imperatore romano che praticava lo stoicismo come filosofia di vita .',
+    'question': 'Quale filosofia seguì Marco Aurelio ?'
+})
+# Output:
+{'answer': 'stoicismo', 'end': 65, 'score': 0.9477770241566028, 'start': 56}
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/savasy/bert-base-turkish-ner-cased/README.md

@@ -1,3 +1,6 @@
+---
+language: turkish
+---
 
 # For Turkish language, here is an easy-to-use NER application. 
  ** Türkçe için kolay bir python  NER (Bert + Transfer Learning)  (İsim Varlık Tanıma) modeli... 

model_cards/savasy/bert-base-turkish-sentiment-cased/README.md

@@ -1,3 +1,6 @@
+---
+language: turkish
+---
 # Bert-base Turkish Sentiment Model
 
 https://huggingface.co/savasy/bert-base-turkish-sentiment-cased

model_cards/savasy/bert-turkish-text-classification/README.md

@@ -0,0 +1,102 @@
+---
+language: turkish
+---
+
+# Turkish Text Classification
+
+This model is a fine-tune model of https://github.com/stefan-it/turkish-bert by using text classification data where there are 7 categories as follows
+
+```
+code_to_label={
+ 'LABEL_0': 'dunya ',
+ 'LABEL_1': 'ekonomi ',
+ 'LABEL_2': 'kultur ',
+ 'LABEL_3': 'saglik ',
+ 'LABEL_4': 'siyaset ',
+ 'LABEL_5': 'spor ',
+ 'LABEL_6': 'teknoloji '}
+ 
+ ```
+
+
+## Data 
+The following Turkish benchmark dataset is used for fine-tuning
+
+https://www.kaggle.com/savasy/ttc4900
+
+## Quick Start
+
+Bewgin with installing transformers as follows
+> pip install transformers
+
+```
+# Code:
+# import libraries
+from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer, AutoModelForSequenceClassification
+tokenizer= AutoTokenizer.from_pretrained("savasy/bert-turkish-text-classification")
+
+# build and load model, it take time depending on your internet connection
+model= AutoModelForSequenceClassification.from_pretrained("savasy/bert-turkish-text-classification")
+
+# make pipeline
+nlp=pipeline("sentiment-analysis", model=model, tokenizer=tokenizer)
+
+# apply model
+nlp("bla bla")
+# [{'label': 'LABEL_2', 'score': 0.4753005802631378}]
+
+code_to_label={
+ 'LABEL_0': 'dunya ',
+ 'LABEL_1': 'ekonomi ',
+ 'LABEL_2': 'kultur ',
+ 'LABEL_3': 'saglik ',
+ 'LABEL_4': 'siyaset ',
+ 'LABEL_5': 'spor ',
+ 'LABEL_6': 'teknoloji '}
+ 
+code_to_label[nlp("bla bla")[0]['label']]
+# > 'kultur '
+```
+
+## How the model was trained
+
+```
+
+## loading data for Turkish text classification
+import pandas as pd
+# https://www.kaggle.com/savasy/ttc4900
+df=pd.read_csv("7allV03.csv")
+df.columns=["labels","text"]
+df.labels=pd.Categorical(df.labels)
+
+traind_df=...
+eval_df=...
+
+# model
+from simpletransformers.classification import ClassificationModel
+import torch,sklearn
+
+model_args = {
+    "use_early_stopping": True,
+    "early_stopping_delta": 0.01,
+    "early_stopping_metric": "mcc",
+    "early_stopping_metric_minimize": False,
+    "early_stopping_patience": 5,
+    "evaluate_during_training_steps": 1000,
+    "fp16": False,
+    "num_train_epochs":3
+}
+
+model = ClassificationModel(
+    "bert", 
+    "dbmdz/bert-base-turkish-cased",
+     use_cuda=cuda_available, 
+     args=model_args, 
+     num_labels=7
+)
+model.train_model(train_df, acc=sklearn.metrics.accuracy_score)
+```
+For other training models please check https://simpletransformers.ai/
+
+
+For the detailed usage of Turkish Text Classification please check [python notebook](https://github.com/savasy/TurkishTextClassification/blob/master/Bert_base_Text_Classification_for_Turkish.ipynb)

model_cards/sentence-transformers/bert-base-nli-cls-token/README.md

@@ -0,0 +1,76 @@
+---
+language: english
+tags:
+- exbert
+license: apache-2.0
+datasets:
+- SNLI
+- MultiNLI
+---
+
+# BERT base model (uncased) for Sentence Embeddings
+This is the `bert-base-nli-cls-token` model from the [sentence-transformers](https://github.com/UKPLab/sentence-transformers)-repository. The sentence-transformers repository allows to train and use Transformer models for generating sentence and text embeddings. 
+The model is described in  the paper  [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084)
+
+## Usage (HuggingFace Models Repository)
+
+You can use the model directly from the model repository to compute sentence embeddings. The CLS token of each input represents the sentence embedding:
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Sentences we want sentence embeddings for
+sentences = ['This framework generates embeddings for each input sentence',
+             'Sentences are passed as a list of string.',
+             'The quick brown fox jumps over the lazy dog.']
+
+#Load AutoModel from huggingface model repository
+tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/bert-base-nli-cls-token")
+model = AutoModel.from_pretrained("sentence-transformers/bert-base-nli-cls-token")
+
+#Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=128, return_tensors='pt')
+
+#Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+    sentence_embeddings = model_output[0][:,0] #Take the first token ([CLS]) from each sentence 
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+## Usage (Sentence-Transformers)
+Using this model becomes more convenient when you have [sentence-transformers](https://github.com/UKPLab/sentence-transformers) installed:
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+```python
+from sentence_transformers import SentenceTransformer
+model = SentenceTransformer('bert-base-nli-cls-token')
+sentences = ['This framework generates embeddings for each input sentence',
+    'Sentences are passed as a list of string.', 
+    'The quick brown fox jumps over the lazy dog.']
+sentence_embeddings = model.encode(sentences)
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+## Citing & Authors
+If you find this model helpful, feel free to cite our publication [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084):
+``` 
+@inproceedings{reimers-2019-sentence-bert,
+    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
+    author = "Reimers, Nils and Gurevych, Iryna",
+    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
+    month = "11",
+    year = "2019",
+    publisher = "Association for Computational Linguistics",
+    url = "http://arxiv.org/abs/1908.10084",
+}
+```

model_cards/sentence-transformers/bert-base-nli-max-tokens/README.md

@@ -0,0 +1,88 @@
+---
+language: english
+tags:
+- exbert
+license: apache-2.0
+datasets:
+- SNLI
+- MultiNLI
+---
+
+# BERT base model (uncased) for Sentence Embeddings
+This is the `bert-base-nli-max-tokens` model from the [sentence-transformers](https://github.com/UKPLab/sentence-transformers)-repository. The sentence-transformers repository allows to train and use Transformer models for generating sentence and text embeddings. 
+The model is described in  the paper  [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084)
+
+## Usage (HuggingFace Models Repository)
+
+You can use the model directly from the model repository to compute sentence embeddings. It uses max pooling to generate a fixed sized sentence embedding:
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Max Pooling - Take the max value over time for every dimension
+def max_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    token_embeddings[input_mask_expanded == 0] = -1e9  # Set padding tokens to large negative value
+    max_over_time = torch.max(token_embeddings, 1)[0]
+    return max_over_time
+
+
+#Sentences we want sentence embeddings for
+sentences = ['This framework generates embeddings for each input sentence',
+             'Sentences are passed as a list of string.',
+             'The quick brown fox jumps over the lazy dog.']
+
+#Load AutoModel from huggingface model repository
+tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/bert-base-nli-max-tokens")
+model = AutoModel.from_pretrained("sentence-transformers/bert-base-nli-max-tokens")
+
+#Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=128, return_tensors='pt')
+
+#Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+#Perform pooling. In this case, max pooling
+sentence_embeddings = max_pooling(model_output, encoded_input['attention_mask'])
+
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+## Usage (Sentence-Transformers)
+Using this model becomes more convenient when you have [sentence-transformers](https://github.com/UKPLab/sentence-transformers) installed:
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+```python
+from sentence_transformers import SentenceTransformer
+model = SentenceTransformer('bert-base-nli-max-tokens')
+sentences = ['This framework generates embeddings for each input sentence',
+    'Sentences are passed as a list of string.', 
+    'The quick brown fox jumps over the lazy dog.']
+sentence_embeddings = model.encode(sentences)
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+## Citing & Authors
+If you find this model helpful, feel free to cite our publication [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084):
+``` 
+@inproceedings{reimers-2019-sentence-bert,
+    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
+    author = "Reimers, Nils and Gurevych, Iryna",
+    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
+    month = "11",
+    year = "2019",
+    publisher = "Association for Computational Linguistics",
+    url = "http://arxiv.org/abs/1908.10084",
+}
+```

model_cards/sentence-transformers/bert-base-nli-mean-tokens/README.md

@@ -0,0 +1,85 @@
+---
+language: english
+tags:
+- exbert
+license: apache-2.0
+datasets:
+- SNLI
+- MultiNLI
+---
+
+# BERT base model (uncased) for Sentence Embeddings
+This is the `bert-base-nli-mean-tokens` model from the [sentence-transformers](https://github.com/UKPLab/sentence-transformers)-repository. The sentence-transformers repository allows to train and use Transformer models for generating sentence and text embeddings. 
+The model is described in  the paper  [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084)
+
+## Usage (HuggingFace Models Repository)
+
+You can use the model directly from the model repository to compute sentence embeddings:
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
+    sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+    return sum_embeddings / sum_mask
+
+
+
+#Sentences we want sentence embeddings for
+sentences = ['This framework generates embeddings for each input sentence',
+             'Sentences are passed as a list of string.',
+             'The quick brown fox jumps over the lazy dog.']
+
+#Load AutoModel from huggingface model repository
+tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/bert-base-nli-mean-tokens")
+model = AutoModel.from_pretrained("sentence-transformers/bert-base-nli-mean-tokens")
+
+#Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=128, return_tensors='pt')
+
+#Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+#Perform pooling. In this case, mean pooling
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+```
+
+## Usage (Sentence-Transformers)
+Using this model becomes more convenient when you have [sentence-transformers](https://github.com/UKPLab/sentence-transformers) installed:
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+```python
+from sentence_transformers import SentenceTransformer
+model = SentenceTransformer('bert-base-nli-mean-tokens')
+sentences = ['This framework generates embeddings for each input sentence',
+    'Sentences are passed as a list of string.', 
+    'The quick brown fox jumps over the lazy dog.']
+sentence_embeddings = model.encode(sentences)
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+## Citing & Authors
+If you find this model helpful, feel free to cite our publication [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084):
+``` 
+@inproceedings{reimers-2019-sentence-bert,
+    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
+    author = "Reimers, Nils and Gurevych, Iryna",
+    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
+    month = "11",
+    year = "2019",
+    publisher = "Association for Computational Linguistics",
+    url = "http://arxiv.org/abs/1908.10084",
+}
+```

model_cards/surajp/RoBERTa-hindi-guj-san/README.md

@@ -0,0 +1,108 @@
+---
+language: "multilingual"
+tags:
+- Hindi
+- Sanskrit
+- Gujarati
+- Indic
+- roberta
+license: "MIT"
+datasets:
+- Wikipedia (Hindi, Sanskrit, Gujarati)
+metrics:
+- perplexity
+---
+
+# RoBERTa-hindi-guj-san
+
+## Model description
+
+Multillingual RoBERTa like model trained on Wikipedia articles of Hindi, Sanskrit, Gujarati languages. The tokenizer was trained on combined text. 
+However, Hindi text was used to pre-train the model and then it was fine-tuned on Sanskrit and Gujarati Text combined hoping that pre-training with Hindi 
+will help the model learn similar languages.
+
+### Configuration
+
+| Parameter | Value |
+|---|---|
+| `hidden_size` | 768 |
+| `num_attention_heads` | 12 |
+| `num_hidden_layers` | 6 |
+| `vocab_size` | 30522 |
+|`model_type`|`roberta`|
+
+## Intended uses & limitations
+
+#### How to use
+
+```python
+# Example usage
+from transformers import AutoTokenizer, AutoModelWithLMHead, pipeline
+
+tokenizer = AutoTokenizer.from_pretrained("surajp/RoBERTa-hindi-guj-san")
+model = AutoModelWithLMHead.from_pretrained("surajp/RoBERTa-hindi-guj-san")
+
+fill_mask = pipeline(
+    "fill-mask",
+    model=model,
+    tokenizer=tokenizer
+)
+
+# Sanskrit: इयं भाषा न केवलं भारतस्य अपि तु विश्वस्य प्राचीनतमा भाषा इति मन्यते।
+# Hindi:  अगर आप अब अभ्यास नहीं करते हो तो आप अपने परीक्षा में मूर्खतापूर्ण गलतियाँ करोगे।
+# Gujarati: ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો <mask> હતો.
+fill_mask("ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો <mask> હતો.")
+
+'''
+Output:
+--------
+[
+{'score': 0.07849744707345963, 'sequence': '<s> ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો જ હતો.</s>', 'token': 390},
+{'score': 0.06273336708545685, 'sequence': '<s> ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો ન હતો.</s>', 'token': 478},
+{'score': 0.05160355195403099, 'sequence': '<s> ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો થઇ હતો.</s>', 'token': 2075},
+{'score': 0.04751499369740486, 'sequence': '<s> ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો એક હતો.</s>', 'token': 600},
+{'score': 0.03788900747895241, 'sequence': '<s> ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો પણ હતો.</s>', 'token': 840}
+]
+
+```
+
+## Training data
+
+Cleaned wikipedia articles in Hindi, Sanskrit and Gujarati on Kaggle. It contains training as well as evaluation text. 
+Used in [iNLTK](https://github.com/goru001/inltk)
+
+- [Hindi](https://www.kaggle.com/disisbig/hindi-wikipedia-articles-172k)
+- [Gujarati](https://www.kaggle.com/disisbig/gujarati-wikipedia-articles)
+- [Sanskrit](https://www.kaggle.com/disisbig/sanskrit-wikipedia-articles)
+
+## Training procedure
+
+- On TPU (using `xla_spawn.py`)
+- For language modelling
+- Iteratively increasing `--block_size` from 128 to 256 over epochs
+- Tokenizer trained on combined text
+- Pre-training with Hindi and fine-tuning on Sanskrit and Gujarati texts
+
+```
+--model_type distillroberta-base \
+--model_name_or_path "/content/SanHiGujBERTa" \
+--mlm_probability 0.20 \
+--line_by_line \
+--save_total_limit 2 \
+--per_device_train_batch_size 128 \
+--per_device_eval_batch_size 128 \
+--num_train_epochs 5 \
+--block_size 256 \
+--seed 108 \
+--overwrite_output_dir \
+```
+
+## Eval results
+
+perplexity = 2.920005983224673
+
+
+
+> Created by [Suraj Parmar/@parmarsuraj99](https://twitter.com/parmarsuraj99) | [LinkedIn](https://www.linkedin.com/in/parmarsuraj99/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in India

model_cards/valhalla/t5-base-e2e-qg/README.md

@@ -0,0 +1,38 @@
+---
+datasets:
+- squad
+tags:
+- question-generation
+widget:
+- text: "Python is a programming language. It is developed by Guido Van Rossum and released in 1991. </s>"
+license: "MIT"
+---
+
+## T5 for question-generation
+This is [t5-base](https://arxiv.org/abs/1910.10683) model trained for end-to-end question generation task. Simply input the text and the model will generate multile questions. 
+
+You can play with the model using the inference API, just put the text and see the results!
+
+For more deatils see [this](https://github.com/patil-suraj/question_generation) repo.
+
+### Model in action 🚀
+
+You'll need to clone the [repo](https://github.com/patil-suraj/question_generation).
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb)
+
+```python3
+from pipelines import pipeline
+
+text = "Python is an interpreted, high-level, general-purpose programming language. Created by Guido van Rossum \
+and first released in 1991, Python's design philosophy emphasizes code \
+readability with its notable use of significant whitespace."
+
+nlp = pipeline("e2e-qg", model="valhalla/t5-base-e2e-qg")
+nlp(text)
+=> [
+ 'Who created Python?',
+ 'When was Python first released?',
+ "What is Python's design philosophy?"
+]
+```

model_cards/valhalla/t5-base-qa-qg-hl/README.md

@@ -0,0 +1,50 @@
+---
+datasets:
+- squad
+tags:
+- question-generation
+widget:
+- text: "generate question: <hl> 42 <hl> is the answer to life, the universe and everything. </s>"
+- text: "question: What is 42 context: 42 is the answer to life, the universe and everything. </s>"
+license: "MIT"
+---
+
+## T5 for multi-task QA and QG
+This is multi-task [t5-base](https://arxiv.org/abs/1910.10683) model trained for question answering and answer aware question generation tasks. 
+
+For question generation the answer spans are highlighted within the text with special highlight tokens (`<hl>`) and prefixed with 'generate question: '. For QA the input is processed like this `question: question_text context: context_text </s>` 
+
+You can play with the model using the inference API. Here's how you can use it
+
+For QG
+
+`generate question: <hl> 42 <hl> is the answer to life, the universe and everything. </s>`
+
+For QA
+
+`question: What is 42 context: 42 is the answer to life, the universe and everything. </s>`
+
+For more deatils see [this](https://github.com/patil-suraj/question_generation) repo.
+
+
+### Model in action 🚀
+
+You'll need to clone the [repo](https://github.com/patil-suraj/question_generation).
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb)
+
+```python3
+from pipelines import pipeline
+nlp = pipeline("multitask-qa-qg", model="valhalla/t5-base-qa-qg-hl")
+
+# to generate questions simply pass the text
+nlp("42 is the answer to life, the universe and everything.")
+=> [{'answer': '42', 'question': 'What is the answer to life, the universe and everything?'}]
+
+# for qa pass a dict with "question" and "context"
+nlp({
+    "question": "What is 42 ?",
+    "context": "42 is the answer to life, the universe and everything."
+})
+=> 'the answer to life, the universe and everything'
+```

model_cards/valhalla/t5-base-qg-hl/README.md

@@ -0,0 +1,33 @@
+---
+datasets:
+- squad
+tags:
+- question-generation
+widget:
+- text: "<hl> 42 <hl> is the answer to life, the universe and everything. </s>"
+- text: "Python is a programming language. It is developed by <hl> Guido Van Rossum <hl>. </s>"
+- text: "Although <hl> practicality <hl> beats purity </s>"
+license: "MIT"
+---
+
+## T5 for question-generation
+This is [t5-base](https://arxiv.org/abs/1910.10683) model trained for answer aware question generation task. The answer spans are highlighted within the text with special highlight tokens. 
+
+You can play with the model using the inference API, just highlight the answer spans with `<hl>` tokens and end the text with `</s>`. For example
+
+`<hl> 42 <hl> is the answer to life, the universe and everything. </s>`
+
+For more deatils see [this](https://github.com/patil-suraj/question_generation) repo.
+
+### Model in action 🚀
+
+You'll need to clone the [repo](https://github.com/patil-suraj/question_generation).
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb)
+
+```python3
+from pipelines import pipeline
+nlp = pipeline("question-generation", model="valhalla/t5-base-qg-hl")
+nlp("42 is the answer to life, universe and everything.")
+=> [{'answer': '42', 'question': 'What is the answer to life, universe and everything?'}]
+```

model_cards/valhalla/t5-samll-qg-prepend/README.md

@@ -0,0 +1,36 @@
+---
+datasets:
+- squad
+tags:
+- question-generation
+widget:
+- text: "answer: 42  context: 42 is the answer to life, the universe and everything. </s>"
+- text: "answer: Guido Van Rossum context: Python is a programming language. It is developed by Guido Van Rossum. </s>"
+- text: "answer: Explicit context: Explicit is better than implicit </s>"
+license: "MIT"
+---
+
+## T5 for question-generation
+This is [t5-small](https://arxiv.org/abs/1910.10683) model trained for answer aware question generation task. The answer text is prepended before the context text. 
+
+You can play with the model using the inference API, just get the input text in this format and see the results!
+`answer: answer_text context: context_text </s>`
+
+For example
+
+`answer: 42  context: 42 is the answer to life, the universe and everything. </s>`
+
+For more deatils see [this](https://github.com/patil-suraj/question_generation) repo.
+
+### Model in action 🚀
+
+You'll need to clone the [repo](https://github.com/patil-suraj/question_generation).
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb)
+
+```python3
+from pipelines import pipeline
+nlp = pipeline("question-generation", qg_format="prepend")
+nlp("42 is the answer to life, universe and everything.")
+=> [{'answer': '42', 'question': 'What is the answer to life, universe and everything?'}]
+```

model_cards/valhalla/t5-small-e2e-qg/README.md

@@ -0,0 +1,38 @@
+---
+datasets:
+- squad
+tags:
+- question-generation
+widget:
+- text: "Python is developed by Guido Van Rossum and released in 1991. </s>"
+license: "MIT"
+---
+
+## T5 for question-generation
+This is [t5-small](https://arxiv.org/abs/1910.10683) model trained for end-to-end question generation task. Simply input the text and the model will generate multile questions. 
+
+You can play with the model using the inference API, just put the text and see the results!
+
+For more deatils see [this](https://github.com/patil-suraj/question_generation) repo.
+
+### Model in action 🚀
+
+You'll need to clone the [repo](https://github.com/patil-suraj/question_generation).
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb)
+
+```python3
+from pipelines import pipeline
+
+text = "Python is an interpreted, high-level, general-purpose programming language. Created by Guido van Rossum \
+and first released in 1991, Python's design philosophy emphasizes code \
+readability with its notable use of significant whitespace."
+
+nlp = pipeline("e2e-qg")
+nlp(text)
+=> [
+ 'Who created Python?',
+ 'When was Python first released?',
+ "What is Python's design philosophy?"
+]
+```

model_cards/valhalla/t5-small-qa-qg-hl/README.md

@@ -0,0 +1,49 @@
+---
+datasets:
+- squad
+tags:
+- question-generation
+widget:
+- text: "generate question: <hl> 42 <hl> is the answer to life, the universe and everything. </s>"
+- text: "question: What is 42 context: 42 is the answer to life, the universe and everything. </s>"
+license: "MIT"
+---
+
+## T5 for multi-task QA and QG
+This is multi-task [t5-small](https://arxiv.org/abs/1910.10683) model trained for question answering and answer aware question generation tasks. 
+
+For question generation the answer spans are highlighted within the text with special highlight tokens (`<hl>`) and prefixed with 'generate question: '. For QA the input is processed like this `question: question_text context: context_text </s>` 
+
+You can play with the model using the inference API. Here's how you can use it
+
+For QG
+
+`generate question: <hl> 42 <hl> is the answer to life, the universe and everything. </s>`
+
+For QA
+
+`question: What is 42 context: 42 is the answer to life, the universe and everything. </s>`
+
+For more deatils see [this](https://github.com/patil-suraj/question_generation) repo.
+
+### Model in action 🚀
+
+You'll need to clone the [repo](https://github.com/patil-suraj/question_generation).
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb)
+
+```python3
+from pipelines import pipeline
+nlp = pipeline("multitask-qa-qg")
+
+# to generate questions simply pass the text
+nlp("42 is the answer to life, the universe and everything.")
+=> [{'answer': '42', 'question': 'What is the answer to life, the universe and everything?'}]
+
+# for qa pass a dict with "question" and "context"
+nlp({
+    "question": "What is 42 ?",
+    "context": "42 is the answer to life, the universe and everything."
+})
+=> 'the answer to life, the universe and everything'
+```

model_cards/valhalla/t5-small-qg-hl/README.md

@@ -0,0 +1,33 @@
+---
+datasets:
+- squad
+tags:
+- question-generation
+widget:
+- text: "<hl> 42 <hl> is the answer to life, the universe and everything. </s>"
+- text: "Python is a programming language. It is developed by <hl> Guido Van Rossum <hl>. </s>"
+- text: "Simple is better than <hl> complex <hl>. </s>"
+license: "MIT"
+---
+
+## T5 for question-generation
+This is [t5-small](https://arxiv.org/abs/1910.10683) model trained for answer aware question generation task. The answer spans are highlighted within the text with special highlight tokens. 
+
+You can play with the model using the inference API, just highlight the answer spans with `<hl>` tokens and end the text with `</s>`. For example
+
+`<hl> 42 <hl> is the answer to life, the universe and everything. </s>`
+
+For more deatils see [this](https://github.com/patil-suraj/question_generation) repo.
+
+### Model in action 🚀
+
+You'll need to clone the [repo](https://github.com/patil-suraj/question_generation).
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb)
+
+```python3
+from pipelines import pipeline
+nlp = pipeline("question-generation")
+nlp("42 is the answer to life, universe and everything.")
+=> [{'answer': '42', 'question': 'What is the answer to life, universe and everything?'}]
+```

model_cards/xlm-roberta-large-finetuned-conll03-german-README.md

@@ -0,0 +1,5 @@
+---
+language: german
+---
+
+## xlm-roberta-large-finetuned-conll03-german

model_cards/yjernite/bart_eli5/README.md

@@ -0,0 +1,10 @@
+---
+language: english
+license: apache-2.0
+datasets:
+- eli5
+---
+
+## BART ELI5
+
+Read the article at https://yjernite.github.io/lfqa.html and try the demo at https://huggingface.co/qa/

model_cards/youscan/ukr-roberta-base/README.md

@@ -0,0 +1,26 @@
+---
+language:
+- ukrainian
+---
+
+# ukr-roberta-base
+
+## Pre-training corpora
+Below is the list of corpora used along with the output of wc command (counting lines, words and characters). These corpora were concatenated and tokenized with HuggingFace Roberta Tokenizer.
+
+| Tables        | Lines           | Words  | Characters  |
+| ------------- |--------------:| -----:| -----:|
+| [Ukrainian Wikipedia - May 2020](https://dumps.wikimedia.org/ukwiki/latest/ukwiki-latest-pages-articles.xml.bz2)      | 18 001 466| 201 207 739 | 2 647 891 947 |
+| [Ukrainian OSCAR deduplicated dataset](https://oscar-public.huma-num.fr/shuffled/uk_dedup.txt.gz) | 56 560 011      |    2 250 210 650 | 29 705 050 592 |
+| Sampled mentions from social networks | 11 245 710      |    128 461 796 | 1 632 567 763 |
+| Total | 85 807 187      |    2 579 880 185 | 33 985 510 302 |
+
+## Pre-training details
+
+* Ukrainian Roberta was trained with code provided in [HuggingFace tutorial](https://huggingface.co/blog/how-to-train)
+* Currently released model follows roberta-base-cased model architecture (12-layer, 768-hidden, 12-heads, 125M parameters)
+* The model was trained on 4xV100 (85 hours)
+* Training configuration you can find in the [original repository](https://github.com/youscan/language-models)
+
+## Author
+Vitalii Radchenko - contact me on Twitter [@vitaliradchenko](https://twitter.com/vitaliradchenko)

notebooks/05-benchmark.ipynb

@@ -6,7 +6,7 @@
       "name": "05-benchmark",
       "provenance": [],
       "collapsed_sections": [],
-      "authorship_tag": "ABX9TyNQ2BQG0erOGhTFF/2Mdn5a",
+      "authorship_tag": "ABX9TyOAUMA92fdE4FM6A349/FWI",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -272,7 +272,7 @@
         "colab_type": "text"
       },
       "source": [
-        "<a href=\"https://colab.research.google.com/github/huggingface/transformers/blob/add_benchmark_notebook/05_benchmark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+        "<a href=\"https://colab.research.google.com/github/huggingface/transformers/blob/update_notebook/notebooks/05_benchmark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
       ]
     },
     {
@@ -312,8 +312,8 @@
         ":-- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n",
         "**Speed - Inference** | ✔ | ✔ | ✔ | ✔ | ✔ | ✘ | ✔ |\n",
         "**Memory - Inference** | ✔ | ✔ | ✔ | ✔ | ✔ | ✘ | ✘ |\n",
-        "**Speed - Train** | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ |\n",
-        "**Memory - Train** | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ |\n",
+        "**Speed - Train** | ✔ | ✘ | ✔ | ✘ | ✘ | ✘ | ✔ |\n",
+        "**Memory - Train** | ✔ | ✘ | ✔ | ✘ | ✘ | ✘ | ✘ |\n",
         "\n",
         "*   *eager execution* means that the function is run in the eager execution environment of TensorFlow 2, see [here](https://www.tensorflow.org/guide/eager).\n",
         "\n",
@@ -321,7 +321,7 @@
         "\n",
         "*   *FP16* stands for TensorFlow's mixed-precision package and is analogous to PyTorch's FP16 feature, see [here](https://www.tensorflow.org/guide/mixed_precision).\n",
         "\n",
-        "***Note***: In ~1,2 weeks it will also be possible to benchmark training in TensorFlow.\n",
+        "***Note***: Benchmark training in TensorFlow is not included in v3.0.2, but available in master.\n",
         "\n",
         "\n",
         "This notebook will show the user how to use `PyTorchBenchmark` and `TensorFlowBenchmark` for two different scenarios:\n",
@@ -407,7 +407,7 @@
         " print(\"GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB\".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))\n",
         "printm()"
       ],
-      "execution_count": 1,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -431,7 +431,7 @@
         "# If GPU RAM Util > 0% => crash notebook on purpose\n",
         "# !kill -9 -1"
       ],
-      "execution_count": 2,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -499,7 +499,7 @@
       "source": [
         "!python run_benchmark.py --help"
       ],
-      "execution_count": 4,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -615,7 +615,7 @@
         "# create plots folder in content\n",
         "!mkdir -p plots_pt"
       ],
-      "execution_count": 5,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -638,7 +638,7 @@
         "                                --inference_memory_csv_file plots_pt/required_memory.csv \\\n",
         "                                --env_info_csv_file plots_pt/env.csv >/dev/null 2>&1  # redirect all prints"
       ],
-      "execution_count": 6,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -666,7 +666,7 @@
         "df = pd.read_csv('plots_pt/required_memory.csv')\n",
         "df"
       ],
-      "execution_count": 7,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -901,7 +901,7 @@
         "df = pd.read_csv('plots_pt/env.csv')\n",
         "df"
       ],
-      "execution_count": 8,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "execute_result",
@@ -1086,7 +1086,7 @@
         "colab_type": "code",
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 514
+          "height": 534
         },
         "outputId": "22499f33-bafc-42b3-f1b7-fcb202df9cd2"
       },
@@ -1098,7 +1098,7 @@
         "from IPython.display import Image\n",
         "Image('plots_pt/required_memory_plot.png')"
       ],
-      "execution_count": 9,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1133,7 +1133,7 @@
         "In short, all memory that is allocated for a given *model identifier*, *batch size* and *sequence length* is measured in a separate process. This way it can be ensured that there is no previously unreleased memory falsely included in the measurement. One should also note that the measured memory even includes the memory allocated by the CUDA driver to load PyTorch and TensorFlow and is, therefore, higher than library-specific memory measurement function, *e.g.* this one for [PyTorch](https://pytorch.org/docs/stable/cuda.html#torch.cuda.max_memory_allocated).\n",
         "\n",
         "Alright, let's analyze the results. It can be noted that the models `aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2` and `deepset/roberta-base-squad2` require significantly less memory than the other three models. Besides `mrm8488/longformer-base-4096-finetuned-squadv2` all models more or less follow the same memory consumption pattern with `aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2` seemingly being able to better scale to larger sequence lengths. \n",
-        "`mrm8488/longformer-base-4096-finetuned-squadv2` is a *Longformer* model, which makes use of *LocalAttention* (check this blog post to learn more about local attention) so that the model scales much better to longer input sequences.\n",
+        "`mrm8488/longformer-base-4096-finetuned-squadv2` is a *Longformer* model, which makes use of *LocalAttention* (check [this](https://huggingface.co/blog/reformer) blog post to learn more about local attention) so that the model scales much better to longer input sequences.\n",
         "\n",
         "For the sake of this notebook, we assume that the longest required input will be less than 512 tokens so that we settle on the models `aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2` and `deepset/roberta-base-squad2`. \n",
         "\n",
@@ -1161,7 +1161,7 @@
         "                                --batch_sizes 64 128 256 512\\\n",
         "                                --no_env_print"
       ],
-      "execution_count": 10,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1207,7 +1207,7 @@
         "colab_type": "code",
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 514
+          "height": 534
         },
         "outputId": "092c4dac-5002-4603-8eba-cd4bca727744"
       },
@@ -1223,7 +1223,7 @@
         "from IPython.display import Image\n",
         "Image('plots_pt/required_memory_plot_2.png')"
       ],
-      "execution_count": 11,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1283,7 +1283,7 @@
         "                                --batch_sizes 64 128 256 512 \\\n",
         "                                --no_env_print \\"
       ],
-      "execution_count": 12,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1351,7 +1351,7 @@
         "colab_type": "code",
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 514
+          "height": 534
         },
         "outputId": "3947ccf0-b91c-43bf-8569-d6afe0232185"
       },
@@ -1363,7 +1363,7 @@
         "from IPython.display import Image\n",
         "Image('plots_tf/required_memory_plot_2.png')"
       ],
-      "execution_count": 13,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1420,7 +1420,7 @@
         "                                --batch_sizes 256 \\\n",
         "                                --no_env_print \\"
       ],
-      "execution_count": 14,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1454,7 +1454,7 @@
         "colab_type": "code",
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 514
+          "height": 534
         },
         "outputId": "152f14c7-288a-4471-9cc0-5108cb24804c"
       },
@@ -1466,7 +1466,7 @@
         "from IPython.display import Image\n",
         "Image('plots_tf/time_plot_2.png')"
       ],
-      "execution_count": 15,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1523,7 +1523,7 @@
         "                                --no_env_print \\\n",
         "                                --use_xla"
       ],
-      "execution_count": 16,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1584,7 +1584,7 @@
         "# Imports\n",
         "from transformers import BartConfig, PyTorchBenchmark, PyTorchBenchmarkArguments"
       ],
-      "execution_count": 17,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1622,7 +1622,7 @@
       "source": [
         "BartConfig.from_pretrained(\"facebook/bart-large-mnli\").to_diff_dict()"
       ],
-      "execution_count": 18,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "display_data",
@@ -1720,7 +1720,7 @@
         "config_10000_vocab = BartConfig.from_pretrained(\"facebook/bart-large-mnli\", vocab_size=10000)\n",
         "config_8_layers = BartConfig.from_pretrained(\"facebook/bart-large-mnli\", encoder_layers=8, decoder_layers=8)"
       ],
-      "execution_count": 19,
+      "execution_count": null,
       "outputs": []
     },
     {
@@ -1770,7 +1770,7 @@
         "# run benchmark\n",
         "result = benchmark.run()"
       ],
-      "execution_count": 20,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1829,7 +1829,7 @@
         "colab_type": "code",
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 514
+          "height": 534
         },
         "outputId": "5dbeb7f7-c996-4db2-a560-735354a5b76f"
       },
@@ -1841,7 +1841,7 @@
         "from IPython.display import Image\n",
         "Image('plots_pt/training_mem_fp16.png')"
       ],
-      "execution_count": 21,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1916,7 +1916,7 @@
         "# run benchmark\n",
         "result = benchmark.run()"
       ],
-      "execution_count": 22,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -1961,7 +1961,7 @@
         "colab_type": "code",
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 514
+          "height": 534
         },
         "outputId": "8a4b4db7-abed-47c4-da61-c3b1ccae66f1"
       },
@@ -1973,7 +1973,7 @@
         "from IPython.display import Image\n",
         "Image('plots_pt/training_speed_fp16.png')"
       ],
-      "execution_count": 23,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
@@ -2017,7 +2017,7 @@
       "source": [
         "Alright, that's it! Now you should be able to benchmark your favorite models on your favorite configurations. \n",
         "\n",
-        "Transparency for the computational cost of a model is becoming more and more important. Feel free to share your results with the community on a shared spreadsheet or by tweeting us @huggingface 🤗."
+        "Feel free to share your results with the community [here](https://github.com/huggingface/transformers/blob/master/examples/benchmarking/README.md) or by tweeting us https://twitter.com/HuggingFace 🤗."
       ]
     }
   ]

notebooks/README.md

@@ -18,6 +18,7 @@ Pull Request so it can be included under the Community notebooks.
 | [How to generate text](https://github.com/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)| How to use different decoding methods for language generation with transformers | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)|
 | [How to export model to ONNX](https://github.com/huggingface/transformers/blob/master/notebooks/04-onnx-export.ipynb) | Highlight how to export and run inference workloads through ONNX |
 | [How to use Benchmarks](https://github.com/huggingface/transformers/blob/master/notebooks/05-benchmark.ipynb) | How to benchmark models with transformers | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/transformers/blob/master/notebooks/05-benchmark.ipynb)|
+| [Reformer](https://github.com/huggingface/blog/blob/master/notebooks/03_reformer.ipynb) | How Reformer pushes the limits of language modeling | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/blog/blob/master/notebooks/03_reformer.ipynb)|
 
 
 ## Community notebooks:

setup.cfg

@@ -26,6 +26,7 @@ known_third_party =
     sacrebleu
     seqeval
     sklearn
+    streamlit
     tensorboardX
     tensorflow
     tensorflow_datasets

setup.py

@@ -71,11 +71,15 @@ extras["sklearn"] = ["scikit-learn"]
 # keras2onnx and onnxconverter-common version is specific through a commit until 1.7.0 lands on pypi
 extras["tf"] = [
     "tensorflow",
+    # "onnxconverter-common",
+    # "keras2onnx"
     "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
     "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx"
 ]
 extras["tf-cpu"] = [
     "tensorflow-cpu",
+    # "onnxconverter-common",
+    # "keras2onnx"
     "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
     "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx"
 ]
@@ -89,6 +93,7 @@ extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "psutil"]
 extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
 extras["quality"] = [
     "black",
+    # "isort",
     "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
     "flake8",
 ]
@@ -96,7 +101,7 @@ extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3<1", "sci
 
 setup(
     name="transformers",
-    version="3.0.0",
+    version="3.0.2",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
@@ -109,7 +114,7 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.8.0-rc4",
+        "tokenizers == 0.8.1.rc1",
         # dataclasses for Python versions that don't have it
         "dataclasses;python_version<'3.7'",
         # utilities from PyPA to e.g. compare versions

src/transformers/__init__.py

@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-__version__ = "3.0.0"
+__version__ = "3.0.2"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -27,6 +27,7 @@ from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig
 from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
 from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
 from .configuration_distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig
+from .configuration_dpr import DPR_PRETRAINED_CONFIG_ARCHIVE_MAP, DPRConfig
 from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig
 from .configuration_encoder_decoder import EncoderDecoderConfig
 from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig
@@ -129,6 +130,14 @@ from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenize
 from .tokenization_camembert import CamembertTokenizer
 from .tokenization_ctrl import CTRLTokenizer
 from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFast
+from .tokenization_dpr import (
+    DPRContextEncoderTokenizer,
+    DPRContextEncoderTokenizerFast,
+    DPRQuestionEncoderTokenizer,
+    DPRQuestionEncoderTokenizerFast,
+    DPRReaderTokenizer,
+    DPRReaderTokenizerFast,
+)
 from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast
 from .tokenization_flaubert import FlaubertTokenizer
 from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
@@ -344,6 +353,7 @@ if is_torch_available():
         FlaubertModel,
         FlaubertWithLMHeadModel,
         FlaubertForSequenceClassification,
+        FlaubertForTokenClassification,
         FlaubertForQuestionAnswering,
         FlaubertForQuestionAnsweringSimple,
         FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
@@ -382,6 +392,14 @@ if is_torch_available():
         LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
     )
 
+    from .modeling_dpr import (
+        DPRPretrainedContextEncoder,
+        DPRPretrainedQuestionEncoder,
+        DPRPretrainedReader,
+        DPRContextEncoder,
+        DPRQuestionEncoder,
+        DPRReader,
+    )
     from .modeling_retribert import (
         RetriBertPreTrainedModel,
         RetriBertModel,
@@ -399,9 +417,21 @@ if is_torch_available():
     )
 
     # Trainer
-    from .trainer import Trainer, torch_distributed_zero_first
-    from .data.data_collator import default_data_collator, DataCollator, DataCollatorForLanguageModeling
-    from .data.datasets import GlueDataset, TextDataset, LineByLineTextDataset, GlueDataTrainingArguments
+    from .trainer import Trainer, set_seed, torch_distributed_zero_first, EvalPrediction
+    from .data.data_collator import (
+        default_data_collator,
+        DataCollator,
+        DataCollatorForLanguageModeling,
+        DataCollatorForPermutationLanguageModeling,
+    )
+    from .data.datasets import (
+        GlueDataset,
+        TextDataset,
+        LineByLineTextDataset,
+        GlueDataTrainingArguments,
+        SquadDataset,
+        SquadDataTrainingArguments,
+    )
 
     # Benchmarks
     from .benchmark.benchmark import PyTorchBenchmark
@@ -424,6 +454,9 @@ if is_tf_available():
         TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
         TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
         TF_MODEL_WITH_LM_HEAD_MAPPING,
+        TF_MODEL_FOR_CAUSAL_LM_MAPPING,
+        TF_MODEL_FOR_MASKED_LM_MAPPING,
+        TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
         TFAutoModel,
         TFAutoModelForMultipleChoice,
         TFAutoModelForPreTraining,
@@ -431,6 +464,9 @@ if is_tf_available():
         TFAutoModelForSequenceClassification,
         TFAutoModelForTokenClassification,
         TFAutoModelWithLMHead,
+        TFAutoModelForCausalLM,
+        TFAutoModelForMaskedLM,
+        TFAutoModelForSeq2SeqLM,
     )
 
     from .modeling_tf_albert import (
@@ -449,6 +485,7 @@ if is_tf_available():
     from .modeling_tf_bert import (
         TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFBertEmbeddings,
+        TFBertLMHeadModel,
         TFBertForMaskedLM,
         TFBertForMultipleChoice,
         TFBertForNextSentencePrediction,

src/transformers/benchmark/benchmark.py

@@ -157,7 +157,7 @@ class PyTorchBenchmark(Benchmark):
         else:
             train_model = model
 
-        model.eval()
+        model.train()
         model.to(self.args.device)
 
         # encoder-decoder has vocab size saved differently
@@ -175,12 +175,12 @@ class PyTorchBenchmark(Benchmark):
         def compute_loss_and_backprob_encoder():
             loss = train_model(input_ids, labels=input_ids)[0]
             loss.backward()
-            train_model.zero_grad()
+            return loss
 
         def compute_loss_and_backprob_encoder_decoder():
             loss = train_model(input_ids, decoder_input_ids=input_ids, labels=input_ids)[0]
             loss.backward()
-            train_model.zero_grad()
+            return loss
 
         _train = (
             compute_loss_and_backprob_encoder_decoder

src/transformers/benchmark/benchmark_tf.py

@@ -24,7 +24,13 @@ import timeit
 from functools import wraps
 from typing import Callable, Optional
 
-from transformers import TF_MODEL_MAPPING, PretrainedConfig, is_py3nvml_available, is_tf_available
+from transformers import (
+    TF_MODEL_MAPPING,
+    TF_MODEL_WITH_LM_HEAD_MAPPING,
+    PretrainedConfig,
+    is_py3nvml_available,
+    is_tf_available,
+)
 
 from .benchmark_utils import (
     Benchmark,
@@ -92,10 +98,11 @@ class TensorFlowBenchmark(Benchmark):
         _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
         return self._measure_speed(_inference)
 
-    def _train_speed(self, model_name, batch_size, sequence_length):
-        raise NotImplementedError(
-            "Training is currently not really implemented." "Wait for TFTrainer to support CLM and MLM."
-        )
+    def _train_speed(self, model_name: str, batch_size: int, sequence_length: int) -> float:
+        strategy = self.args.strategy
+        assert strategy is not None, "A device strategy has to be initialized before using TensorFlow."
+        _train = self._prepare_train_func(model_name, batch_size, sequence_length)
+        return self._measure_speed(_train)
 
     def _inference_memory(
         self, model_name: str, batch_size: int, sequence_length: int
@@ -108,10 +115,16 @@ class TensorFlowBenchmark(Benchmark):
         _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
         return self._measure_memory(_inference)
 
-    def _train_memory(self, model_name, batch_size, sequence_length):
-        raise NotImplementedError(
-            "Training is currently not really implemented. Wait for TFTrainer to support CLM and MLM."
-        )
+    def _train_memory(
+        self, model_name: str, batch_size: int, sequence_length: int
+    ) -> [Memory, Optional[MemorySummary]]:
+        if self.args.is_gpu:
+            tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx], True)
+        strategy = self.args.strategy
+        assert strategy is not None, "A device strategy has to be initialized before using TensorFlow."
+
+        _train = self._prepare_train_func(model_name, batch_size, sequence_length)
+        return self._measure_memory(_train)
 
     def _prepare_inference_func(self, model_name: str, batch_size: int, sequence_length: int) -> Callable[[], None]:
         config = self.config_dict[model_name]
@@ -149,6 +162,50 @@ class TensorFlowBenchmark(Benchmark):
 
         return _inference
 
+    def _prepare_train_func(self, model_name: str, batch_size: int, sequence_length: int) -> Callable[[], None]:
+        config = self.config_dict[model_name]
+
+        assert (
+            self.args.eager_mode is False
+        ), "Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."
+
+        if self.args.fp16:
+            raise NotImplementedError("Mixed precision is currently not supported.")
+
+        has_model_class_in_config = hasattr(config, "architecture") and len(config.architectures) > 1
+        if not self.args.only_pretrain_model and has_model_class_in_config:
+            try:
+                model_class = "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
+                transformers_module = __import__("transformers", fromlist=[model_class])
+                model_cls = getattr(transformers_module, model_class)
+                model = model_cls(config)
+            except ImportError:
+                raise ImportError(
+                    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to set `--only_pretrain_model` or `args.only_pretrain_model=True`."
+                )
+        else:
+            model = TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](config)
+
+        # encoder-decoder has vocab size saved differently
+        vocab_size = config.vocab_size if hasattr(config, "vocab_size") else config.encoder.vocab_size
+        input_ids = random_input_ids(batch_size, sequence_length, vocab_size)
+
+        @run_with_tf_optimizations(self.args.eager_mode, self.args.use_xla)
+        def encoder_decoder_train():
+            loss = model(input_ids, decoder_input_ids=input_ids, labels=input_ids, training=True)[0]
+            gradients = tf.gradients(loss, model.trainable_variables)
+            return gradients
+
+        @run_with_tf_optimizations(self.args.eager_mode, self.args.use_xla)
+        def encoder_train():
+            loss = model(input_ids, labels=input_ids, training=True)[0]
+            gradients = tf.gradients(loss, model.trainable_variables)
+            return gradients
+
+        _train = encoder_decoder_train if config.is_encoder_decoder else encoder_train
+
+        return _train
+
     def _measure_speed(self, func) -> float:
         with self.args.strategy.scope():
             try:

src/transformers/benchmark/benchmark_utils.py

@@ -386,6 +386,9 @@ def start_memory_tracing(
             elif isinstance(events_to_trace, (list, tuple)) and event not in events_to_trace:
                 return traceit
 
+        if "__name__" not in frame.f_globals:
+            return traceit
+
         # Filter modules
         name = frame.f_globals["__name__"]
         if not isinstance(name, str):

src/transformers/configuration_dpr.py

@@ -0,0 +1,49 @@
+# coding=utf-8
+# Copyright 2010, DPR authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" DPR model configuration """
+
+
+import logging
+
+from .configuration_bert import BertConfig
+
+
+logger = logging.getLogger(__name__)
+
+DPR_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "facebook/dpr-ctx_encoder-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-ctx_encoder-single-nq-base/config.json",
+    "facebook/dpr-question_encoder-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-question_encoder-single-nq-base/config.json",
+    "facebook/dpr-reader-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-reader-single-nq-base/config.json",
+}
+
+
+class DPRConfig(BertConfig):
+    r"""
+        :class:`~transformers.DPRConfig` is the configuration class to store the configuration of a
+        `DPRModel`.
+
+        This is the configuration class to store the configuration of a `DPRContextEncoder`, `DPRQuestionEncoder`, or a `DPRReader`.
+        It is used to instantiate the components of the DPR model.
+
+        Args:
+            projection_dim (:obj:`int`, optional, defaults to 0):
+                Dimension of the projection for the context and question encoders.
+                If it is set to zero (default), then no projection is done.
+    """
+    model_type = "dpr"
+
+    def __init__(self, projection_dim: int = 0, **kwargs):  # projection of the encoders, 0 for no projection
+        super().__init__(**kwargs)
+        self.projection_dim = projection_dim

src/transformers/configuration_utils.py

@@ -20,7 +20,7 @@ import copy
 import json
 import logging
 import os
-from typing import Dict, Tuple
+from typing import Any, Dict, Tuple
 
 from .file_utils import CONFIG_NAME, cached_path, hf_bucket_url, is_remote_url
 
@@ -30,31 +30,110 @@ logger = logging.getLogger(__name__)
 
 class PretrainedConfig(object):
     r""" Base class for all configuration classes.
-        Handles a few parameters common to all models' configurations as well as methods for loading/downloading/saving configurations.
+        Handles a few parameters common to all models' configurations as well as methods for loading/downloading/saving
+        configurations.
 
         Note:
-            A configuration file can be loaded and saved to disk. Loading the configuration file and using this file to initialize a model does **not** load the model weights.
+            A configuration file can be loaded and saved to disk. Loading the configuration file and using this file to
+            initialize a model does **not** load the model weights.
             It only affects the model's configuration.
 
-        Class attributes (overridden by derived classes):
-            - ``model_type``: a string that identifies the model type, that we serialize into the JSON file, and that we use to recreate the correct object in :class:`~transformers.AutoConfig`.
+        Class attributes (overridden by derived classes)
+            - **model_type** (:obj:`str`): An identifier for the model type, serialized into the JSON file, and used to
+              recreate the correct object in :class:`~transformers.AutoConfig`.
 
         Args:
-            finetuning_task (:obj:`string` or :obj:`None`, `optional`, defaults to :obj:`None`):
-                Name of the task used to fine-tune the model. This can be used when converting from an original (TensorFlow or PyTorch) checkpoint.
-            num_labels (:obj:`int`, `optional`, defaults to `2`):
-                Number of classes to use when the model is a classification model (sequences/tokens)
             output_hidden_states (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Should the model returns all hidden-states.
+                Whether or not the model should return all hidden-states.
             output_attentions (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Should the model returns all attentions.
-            torchscript (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Is the model used with Torchscript (for PyTorch models).
+                Whether or not the model should returns all attentions.
+            use_cache (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                Whether or not the model should return the last key/values attentions (not used by all models).
+            return_tuple (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                Whether or not the model should return tuples instead of :obj:`ModelOutput` objects.
+            is_encoder_decoder (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                Whether the model is used as an encoder/decoder or not.
+            is_decoder (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                Whether the model is used as decoder or not (in which case it's used as an encoder).
+            prune_heads (:obj:`Dict[int, List[int]]`, `optional`, defaults to :obj:`{}`):
+                Pruned heads of the model. The keys are the selected layer indices and the associated values, the list
+                of heads to prune in said layer.
+
+                For instance ``{1: [0, 2], 2: [2, 3]}`` will prune heads 0 and 2 on layer 1 and heads 2 and 3 on layer
+                2.
+            xla_device (:obj:`bool`, `optional`):
+                A flag to indicate if TPU are available or not.
+
+        Parameters for sequence generation
+            - **max_length** (:obj:`int`, `optional`, defaults to 20) -- Maximum length that will be used by
+              default in the :obj:`generate` method of the model.
+            - **min_length** (:obj:`int`, `optional`, defaults to 10) -- Minimum length that will be used by
+              default in the :obj:`generate` method of the model.
+            - **do_sample** (:obj:`bool`, `optional`, defaults to :obj:`False`) -- Flag that will be used by default in
+              the :obj:`generate` method of the model. Whether or not to use sampling ; use greedy decoding otherwise.
+            - **early_stopping** (:obj:`bool`, `optional`, defaults to :obj:`False`) -- Flag that will be used by
+              default in the :obj:`generate` method of the model. Whether to stop the beam search when at least
+              ``num_beams`` sentences are finished per batch or not.
+            - **num_beams** (:obj:`int`, `optional`, defaults to 1) -- Number of beams for beam search that will be
+              used by default in the :obj:`generate` method of the model. 1 means no beam search.
+            - **temperature** (:obj:`float`, `optional`, defaults to 1) -- The value used to module the next token
+              probabilities that will be used by default in the :obj:`generate` method of the model. Must be strictly
+              positive.
+            - **top_k** (:obj:`int`, `optional`, defaults to 50) -- Number of highest probability vocabulary tokens to
+              keep for top-k-filtering that will be used by default in the :obj:`generate` method of the model.
+            - **top_p** (:obj:`float`, `optional`, defaults to 1) --  Value that will be used by default in the
+              :obj:`generate` method of the model for ``top_p``. If set to float < 1, only the most probable tokens
+              with probabilities that add up to ``top_p`` or highest are kept for generation.
+            - **repetition_penalty** (:obj:`float`, `optional`, defaults to 1) -- Parameter for repetition penalty
+              that will be used by default in the :obj:`generate` method of the model. 1.0 means no penalty.
+            - **length_penalty** (:obj:`float`, `optional`, defaults to 1) -- Exponential penalty to the length that
+              will be used by default in the :obj:`generate` method of the model.
+            - **no_repeat_ngram_size** (:obj:`int`, `optional`, defaults to 0) -- Value that will be used by default
+              in the :obj:`generate` method of the model for ``no_repeat_ngram_size``. If set to int > 0, all ngrams of
+              that size can only occur once.
+            - **bad_words_ids** (:obj:`List[int]`, `optional`) -- List of token ids that are not allowed to be
+              generated that will be used by default in the :obj:`generate` method of the model. In order to get the
+              tokens of the words that should not appear in the generated text, use
+              :obj:`tokenizer.encode(bad_word, add_prefix_space=True)`.
+            - **num_return_sequences** (:obj:`int`, `optional`, defaults to 1) -- Number of independently computed
+              returned sequences for each element in the batch that will be used by default in the :obj:`generate`
+              method of the model.
+
+        Parameters for fine-tuning tasks
+            - **architectures** (:obj:List[`str`], `optional`) -- Model architectures that can be used with the
+              model pretrained weights.
+            - **finetuning_task** (:obj:`str`, `optional`) -- Name of the task used to fine-tune the model. This can be
+              used when converting from an original (TensorFlow or PyTorch) checkpoint.
+            - **id2label** (:obj:`List[str]`, `optional`) -- A map from index (for instance prediction index, or target
+              index) to label.
+            - **label2id** (:obj:`Dict[str, int]`, `optional`) -- A map from label to index for the model.
+            - **num_labels** (:obj:`int`, `optional`) -- Number of labels to use in the last layer added to the model,
+              typically for a classification task.
+            - **task_specific_params** (:obj:`Dict[str, Any]`, `optional`) -- Additional keyword arguments to store for
+              the current task.
+
+        Parameters linked to the tokenizer
+            - **prefix** (:obj:`str`, `optional`) -- A specific prompt that should be added at the beginning of each
+              text before calling the model.
+            - **bos_token_id** (:obj:`int`, `optional`)) -- The id of the `beginning-of-stream` token.
+            - **pad_token_id** (:obj:`int`, `optional`)) -- The id of the `padding` token.
+            - **eos_token_id** (:obj:`int`, `optional`)) -- The id of the `end-of-stream` token.
+            - **decoder_start_token_id** (:obj:`int`, `optional`)) -- If an encoder-decoder model starts decoding with
+              a different token than `bos`, the id of that token.
+
+        PyTorch specific parameters
+            - **torchscript** (:obj:`bool`, `optional`, defaults to :obj:`False`) -- Whether or not the model should be
+              used with Torchscript.
+
+        TensorFlow specific parameters
+            - **use_bfloat16** (:obj:`bool`, `optional`, defaults to :obj:`False`) -- Whether or not the model should
+              use BFloat16 scalars (only used by some TensorFlow models).
     """
     model_type: str = ""
 
     def __init__(self, **kwargs):
         # Attributes with defaults
+        self.return_tuple = kwargs.pop("return_tuple", False)
         self.output_hidden_states = kwargs.pop("output_hidden_states", False)
         self.output_attentions = kwargs.pop("output_attentions", False)
         self.use_cache = kwargs.pop("use_cache", True)  # Not used by all models
@@ -115,22 +194,27 @@ class PretrainedConfig(object):
                 raise err
 
     @property
-    def num_labels(self):
+    def use_return_tuple(self):
+        # If torchscript is set, force return_tuple to avoid jit errors
+        return self.return_tuple or self.torchscript
+
+    @property
+    def num_labels(self) -> int:
         return len(self.id2label)
 
     @num_labels.setter
-    def num_labels(self, num_labels):
+    def num_labels(self, num_labels: int):
         self.id2label = {i: "LABEL_{}".format(i) for i in range(num_labels)}
         self.label2id = dict(zip(self.id2label.values(), self.id2label.keys()))
 
-    def save_pretrained(self, save_directory):
+    def save_pretrained(self, save_directory: str):
         """
-        Save a configuration object to the directory `save_directory`, so that it
-        can be re-loaded using the :func:`~transformers.PretrainedConfig.from_pretrained` class method.
+        Save a configuration object to the directory ``save_directory``, so that it can be re-loaded using the
+        :func:`~transformers.PretrainedConfig.from_pretrained` class method.
 
         Args:
-            save_directory (:obj:`string`):
-                Directory where the configuration JSON file will be saved.
+            save_directory (:obj:`str`):
+                Directory where the configuration JSON file will be saved (will be created if it does not exist).
         """
         if os.path.isfile(save_directory):
             raise AssertionError("Provided path ({}) should be a directory, not a file".format(save_directory))
@@ -142,45 +226,49 @@ class PretrainedConfig(object):
         logger.info("Configuration saved in {}".format(output_config_file))
 
     @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs) -> "PretrainedConfig":
+    def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs) -> "PretrainedConfig":
         r"""
-
-        Instantiate a :class:`~transformers.PretrainedConfig` (or a derived class) from a pre-trained model configuration.
+        Instantiate a :class:`~transformers.PretrainedConfig` (or a derived class) from a pretrained model
+        configuration.
 
         Args:
-            pretrained_model_name_or_path (:obj:`string`):
-                either:
-                  - a string with the `shortcut name` of a pre-trained model configuration to load from cache or
-                    download, e.g.: ``bert-base-uncased``.
-                  - a string with the `identifier name` of a pre-trained model configuration that was user-uploaded to
-                    our S3, e.g.: ``dbmdz/bert-base-german-cased``.
-                  - a path to a `directory` containing a configuration file saved using the
-                    :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``.
-                  - a path or url to a saved configuration JSON `file`, e.g.:
-                    ``./my_model_directory/configuration.json``.
-            cache_dir (:obj:`string`, `optional`):
-                Path to a directory in which a downloaded pre-trained model
-                configuration should be cached if the standard cache should not be used.
-            kwargs (:obj:`Dict[str, any]`, `optional`):
-                The values in kwargs of any keys which are configuration attributes will be used to override the loaded
-                values. Behavior concerning key/value pairs whose keys are *not* configuration attributes is
-                controlled by the `return_unused_kwargs` keyword parameter.
+            pretrained_model_name_or_path (:obj:`str`):
+                This can be either:
+
+                - the `shortcut name` of a pretrained model configuration to load from cache or download, e.g.,
+                  ``bert-base-uncased``.
+                - the `identifier name` of a pretrained model configuration that was uploaded to our S3 by any user,
+                  e.g., ``dbmdz/bert-base-german-cased``.
+                - a path to a `directory` containing a configuration file saved using the
+                  :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g., ``./my_model_directory/``.
+                - a path or url to a saved configuration JSON `file`, e.g.,
+                  ``./my_model_directory/configuration.json``.
+            cache_dir (:obj:`str`, `optional`):
+                Path to a directory in which a downloaded pretrained model configuration should be cached if the
+                standard cache should not be used.
             force_download (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Force to (re-)download the model weights and configuration files and override the cached versions if they exist.
+                Wheter or not to force to (re-)download the configuration files and override the cached versions if they
+                exist.
             resume_download (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Do not delete incompletely recieved file. Attempt to resume the download if such a file exists.
-            proxies (:obj:`Dict`, `optional`):
-                A dictionary of proxy servers to use by protocol or endpoint, e.g.:
+                Whether or not to delete incompletely received file. Attempts to resume the download if such a file
+                exists.
+            proxies (:obj:`Dict[str, str]`, `optional`):
+                A dictionary of proxy servers to use by protocol or endpoint, e.g.,
                 :obj:`{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.`
                 The proxies are used on each request.
-            return_unused_kwargs: (`optional`) bool:
-                If False, then this function returns just the final configuration object.
-                If True, then this functions returns a :obj:`Tuple(config, unused_kwargs)` where `unused_kwargs` is a
-                dictionary consisting of the key/value pairs whose keys are not configuration attributes: ie the part
-                of kwargs which has not been used to update `config` and is otherwise ignored.
+            return_unused_kwargs (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                If :obj:`False`, then this function returns just the final configuration object.
+
+                If :obj:`True`, then this functions returns a :obj:`Tuple(config, unused_kwargs)` where `unused_kwargs`
+                is a dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e.,
+                the part of ``kwargs`` which has not been used to update ``config`` and is otherwise ignored.
+            kwargs (:obj:`Dict[str, Any]`, `optional`):
+                The values in kwargs of any keys which are configuration attributes will be used to override the loaded
+                values. Behavior concerning key/value pairs whose keys are *not* configuration attributes is
+                controlled by the ``return_unused_kwargs`` keyword parameter.
 
         Returns:
-            :class:`PretrainedConfig`: An instance of a configuration object
+            :class:`PretrainedConfig`: The configuration object instantiated from this pretrained model.
 
         Examples::
 
@@ -201,17 +289,17 @@ class PretrainedConfig(object):
         return cls.from_dict(config_dict, **kwargs)
 
     @classmethod
-    def get_config_dict(cls, pretrained_model_name_or_path: str, **kwargs) -> Tuple[Dict, Dict]:
+    def get_config_dict(cls, pretrained_model_name_or_path: str, **kwargs) -> Tuple[Dict[str, Any], Dict[str, Any]]:
         """
-        From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used
-        for instantiating a Config using `from_dict`.
+        From a ``pretrained_model_name_or_path``, resolve to a dictionary of parameters, to be used
+        for instantiating a :class:`~transformers.PretrainedConfig` using ``from_dict``.
 
         Parameters:
-            pretrained_model_name_or_path (:obj:`string`):
+            pretrained_model_name_or_path (:obj:`str`):
                 The identifier of the pre-trained checkpoint from which we want the dictionary of parameters.
 
         Returns:
-            :obj:`Tuple[Dict, Dict]`: The dictionary that will be used to instantiate the configuration object.
+            :obj:`Tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the configuration object.
 
         """
         cache_dir = kwargs.pop("cache_dir", None)
@@ -266,20 +354,20 @@ class PretrainedConfig(object):
         return config_dict, kwargs
 
     @classmethod
-    def from_dict(cls, config_dict: Dict, **kwargs) -> "PretrainedConfig":
+    def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "PretrainedConfig":
         """
-        Constructs a `Config` from a Python dictionary of parameters.
+        Instantiates a :class:`~transformers.PretrainedConfig` from a Python dictionary of parameters.
 
         Args:
-            config_dict (:obj:`Dict[str, any]`):
-                Dictionary that will be used to instantiate the configuration object. Such a dictionary can be retrieved
-                from a pre-trained checkpoint by leveraging the :func:`~transformers.PretrainedConfig.get_config_dict`
-                method.
-            kwargs (:obj:`Dict[str, any]`):
+            config_dict (:obj:`Dict[str, Any]`):
+                Dictionary that will be used to instantiate the configuration object. Such a dictionary can be
+                retrieved from a pretrained checkpoint by leveraging the
+                :func:`~transformers.PretrainedConfig.get_config_dict` method.
+            kwargs (:obj:`Dict[str, Any]`):
                 Additional parameters from which to initialize the configuration object.
 
         Returns:
-            :class:`PretrainedConfig`: An instance of a configuration object
+            :class:`PretrainedConfig`: The configuration object instantiated from those parameters.
         """
         return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)
 
@@ -306,14 +394,14 @@ class PretrainedConfig(object):
     @classmethod
     def from_json_file(cls, json_file: str) -> "PretrainedConfig":
         """
-        Constructs a `Config` from the path to a json file of parameters.
+        Instantiates a :class:`~transformers.PretrainedConfig` from the path to a JSON file of parameters.
 
         Args:
-            json_file (:obj:`string`):
+            json_file (:obj:`str`):
                 Path to the JSON file containing the parameters.
 
         Returns:
-            :class:`PretrainedConfig`: An instance of a configuration object
+            :class:`PretrainedConfig`: The configuration object instantiated from that JSON file.
 
         """
         config_dict = cls._dict_from_json_file(json_file)
@@ -331,14 +419,14 @@ class PretrainedConfig(object):
     def __repr__(self):
         return "{} {}".format(self.__class__.__name__, self.to_json_string())
 
-    def to_diff_dict(self):
+    def to_diff_dict(self) -> Dict[str, Any]:
         """
         Removes all attributes from config which correspond to the default
         config attributes for better readability and serializes to a Python
         dictionary.
 
         Returns:
-            :obj:`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+            :obj:`Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,
         """
         config_dict = self.to_dict()
 
@@ -354,28 +442,29 @@ class PretrainedConfig(object):
 
         return serializable_config_dict
 
-    def to_dict(self):
+    def to_dict(self) -> Dict[str, Any]:
         """
         Serializes this instance to a Python dictionary.
 
         Returns:
-            :obj:`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
+            :obj:`Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
         """
         output = copy.deepcopy(self.__dict__)
         if hasattr(self.__class__, "model_type"):
             output["model_type"] = self.__class__.model_type
         return output
 
-    def to_json_string(self, use_diff=True):
+    def to_json_string(self, use_diff: bool = True) -> str:
         """
         Serializes this instance to a JSON string.
 
         Args:
-            use_diff (:obj:`bool`):
-                If set to True, only the difference between the config instance and the default PretrainedConfig() is serialized to JSON string.
+            use_diff (:obj:`bool`, `optional`, defaults to :obj:`True`):
+                If set to ``True``, only the difference between the config instance and the default
+                ``PretrainedConfig()`` is serialized to JSON string.
 
         Returns:
-            :obj:`string`: String containing all the attributes that make up this configuration instance in JSON format.
+            :obj:`str`: String containing all the attributes that make up this configuration instance in JSON format.
         """
         if use_diff is True:
             config_dict = self.to_diff_dict()
@@ -383,26 +472,26 @@ class PretrainedConfig(object):
             config_dict = self.to_dict()
         return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"
 
-    def to_json_file(self, json_file_path, use_diff=True):
+    def to_json_file(self, json_file_path: str, use_diff: bool = True):
         """
-        Save this instance to a json file.
+        Save this instance to a JSON file.
 
         Args:
-            json_file_path (:obj:`string`):
+            json_file_path (:obj:`str`):
                 Path to the JSON file in which this configuration instance's parameters will be saved.
-            use_diff (:obj:`bool`):
-                If set to True, only the difference between the config instance and the default PretrainedConfig() is serialized to JSON file.
+            use_diff (:obj:`bool`, `optional`, defaults to :obj:`True`):
+                If set to ``True``, only the difference between the config instance and the default
+                ``PretrainedConfig()`` is serialized to JSON file.
         """
         with open(json_file_path, "w", encoding="utf-8") as writer:
             writer.write(self.to_json_string(use_diff=use_diff))
 
-    def update(self, config_dict: Dict):
+    def update(self, config_dict: Dict[str, Any]):
         """
-        Updates attributes of this class
-        with attributes from `config_dict`.
+        Updates attributes of this class with attributes from ``config_dict``.
 
         Args:
-            :obj:`Dict[str, any]`: Dictionary of attributes that shall be updated for this class.
+            config_dict (:obj:`Dict[str, Any]`): Dictionary of attributes that shall be updated for this class.
         """
         for key, value in config_dict.items():
             setattr(self, key, value)

src/transformers/configuration_xlnet.py

@@ -15,8 +15,8 @@
 # limitations under the License.
 """ XLNet configuration """
 
-
 import logging
+import warnings
 
 from .configuration_utils import PretrainedConfig
 
@@ -195,6 +195,17 @@ class XLNetConfig(PretrainedConfig):
         self.pad_token_id = pad_token_id
         self.eos_token_id = eos_token_id
 
+        if mem_len is None or mem_len == 0:
+            warnings.warn(
+                "This config doesn't use attention memories, a core feature of XLNet."
+                " Consider setting `men_len` to a non-zero value, for example "
+                "`xlnet = XLNetLMHeadModel.from_pretrained('xlnet-base-cased'', mem_len=1024)`,"
+                " for accurate training performance as well as an order of magnitude faster inference."
+                " Starting from version 3.5.0, the default parameter will be 1024, following"
+                " the implementation in https://arxiv.org/abs/1906.08237",
+                FutureWarning,
+            )
+
     @property
     def max_position_embeddings(self):
         return -1