Truncate when overflowing

Fix the masked LM model
Test the Masked LM model
2020-12-10 17:13:02 -05:00 · 2020-12-10 15:25:33 -05:00 · 2020-12-10 15:18:02 -05:00 · 2020-12-10 14:55:17 -05:00 · 2020-12-09 16:31:08 -05:00 · 2020-12-09 21:18:21 +01:00
1169 changed files with 52092 additions and 20174 deletions
@@ -3,6 +3,28 @@ orbs:
    gcp-gke: circleci/gcp-gke@1.0.4
    go: circleci/go@1.3.0

+commands:
+  skip-job-on-doc-only-changes:
+    description: "Do not continue this job and exit with success for PRs with only doc changes"
+    steps:
+
+      - run:
+          name: docs-only changes skip check
+          command: |
+            # pipeline.git.base_revision is not always defined, so only proceed if all external vars are defined
+            if test -n "<< pipeline.git.base_revision >>" && test -n "<< pipeline.git.revision >>" && test -n "$(git diff --name-only << pipeline.git.base_revision >>...<< pipeline.git.revision >>)"
+            then
+                if git diff --name-only << pipeline.git.base_revision >>...<< pipeline.git.revision >> | egrep -qv '\.(md|rst)$'
+                then
+                    echo "Non-docs were modified in this PR, proceeding normally"
+                else
+                    echo "Only docs were modified in this PR, quitting this job"
+                    # disable skipping for now, as circleCI's base_revision is inconsistent leading to invalid ranges
+                    # circleci step halt
+                fi
+            else
+                echo "Can't perform skipping check w/o base_revision defined, continuing the job"
+            fi

 # TPU REFERENCES
 references:
@@ -73,6 +95,7 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch_and_tf-{{ checksum "setup.py" }}
@@ -100,6 +123,7 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
@@ -127,6 +151,7 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-tf-{{ checksum "setup.py" }}
@@ -153,6 +178,7 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                keys:
                    - v0.4-flax-{{ checksum "setup.py" }}
@@ -179,6 +205,7 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
@@ -206,6 +233,7 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-tf-{{ checksum "setup.py" }}
@@ -230,6 +258,7 @@ jobs:
            RUN_CUSTOM_TOKENIZERS: yes
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
@@ -257,13 +286,14 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch_examples-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,torch,sentencepiece,testing]
-            - run: pip install -r examples/_tests_requirements.txt
+            - run: pip install -r examples/requirements.txt
            - save_cache:
                  key: v0.4-torch_examples-{{ checksum "setup.py" }}
                  paths:
@@ -53,5 +53,4 @@ deploy_doc "3ebb1b3" v3.2.0
 deploy_doc "0613f05" v3.3.1
 deploy_doc "eb0e0ce" v3.4.0
 deploy_doc "818878d" v3.5.1
-deploy_doc "c781171" v4.0.0
-deploy_doc "bfa4ccf" # v4.1.1 Latest stable release
+deploy_doc "c781171" # v4.0.0 Latest stable release
@@ -40,8 +40,6 @@ jobs:
          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-seq-2-seq-bart-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-seq-2-seq-bart-tokenizer.json --path=templates/adding_a_new_model
          make style
          python utils/check_table.py --fix_and_overwrite
          python utils/check_dummies.py --fix_and_overwrite
@@ -50,7 +50,6 @@ jobs:
          pip install --upgrade pip
          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
          pip install git+https://github.com/huggingface/datasets
-          pip install pandas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cu102.html

      - name: Are GPUs recognized by our DL frameworks
        run: |
@@ -188,7 +187,6 @@ jobs:
          pip install --upgrade pip
          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
          pip install git+https://github.com/huggingface/datasets
-          pip install pandas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cu102.html

      - name: Are GPUs recognized by our DL frameworks
        run: |
@@ -222,7 +222,7 @@ Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
 ultilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT.
 1. **[SqueezeBert](https://huggingface.co/transformers/model_doc/squeezebert.html)** released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://arxiv.org/abs/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
 1. **[T5](https://huggingface.co/transformers/model_doc/t5.html)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
-1. **[TAPAS](https://huggingface.co/transformers/master/model_doc/tapas.html)** (from Google AI) released with the paper [TAPAS: Weakly Supervised Table Parsing via Pre-training](https://arxiv.org/abs/2004.02349) by Jonathan Herzig, Paweł Krzysztof Nowak, Thomas Müller, Francesco Piccinno and Julian Martin Eisenschlos.
+1. **[TAPAS](https://huggingface.co/transformers/master/model_doc/tapas.html)** released with the paper [TAPAS: Weakly Supervised Table Parsing via Pre-training](https://arxiv.org/abs/2004.02349) by Jonathan Herzig, Paweł Krzysztof Nowak, Thomas Müller, Francesco Piccinno and Julian Martin Eisenschlos.
 1. **[Transformer-XL](https://huggingface.co/transformers/model_doc/transformerxl.html)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
 1. **[XLM](https://huggingface.co/transformers/model_doc/xlm.html)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau.
 1. **[XLM-ProphetNet](https://huggingface.co/transformers/model_doc/xlmprophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
@@ -1,11 +1,10 @@
 // These two things need to be updated at each release for the version selector.
 // Last stable version
-const stableVersion = "v4.1.1"
+const stableVersion = "v4.0.0"
 // Dictionary doc folder to label. The last stable version should have an empty key.
 const versionMapping = {
    "master": "master",
-    "": "v4.1.1 (stable)",
-    "v4.0.1": "v4.0.0/v4.0.1",
+    "": "v4.0.0 (stable)",
    "v3.5.1": "v3.5.0/v3.5.1",
    "v3.4.0": "v3.4.0",
    "v3.3.1": "v3.3.0/v3.3.1",
@@ -34,5 +34,5 @@ help people access the inner representations, mainly adapted from the great work
  in https://arxiv.org/abs/1905.10650.

 To help you understand and use these features, we have added a specific example script: `bertology.py
-<https://github.com/huggingface/transformers/blob/master/examples/research_projects/bertology/run_bertology.py>`_ while
-extract information and prune a model pre-trained on GLUE.
+<https://github.com/huggingface/transformers/blob/master/examples/bertology/run_bertology.py>`_ while extract
+information and prune a model pre-trained on GLUE.
@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'4.1.1'
+release = u'4.0.0'


 # -- General configuration ---------------------------------------------------
@@ -176,9 +176,9 @@ and conversion utilities for the following models:
 30. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
    Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by Colin Raffel and Noam Shazeer and Adam
    Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
-31. `TAPAS <https://huggingface.co/transformers/master/model_doc/tapas.html>`__ (from Google AI) released with the
-    paper `TAPAS: Weakly Supervised Table Parsing via Pre-training <https://arxiv.org/abs/2004.02349>`__ by Jonathan
-    Herzig, Paweł Krzysztof Nowak, Thomas Müller, Francesco Piccinno and Julian Martin Eisenschlos.
+31. `TAPAS <https://huggingface.co/transformers/master/model_doc/tapas.html>`__ released with the paper `TAPAS: Weakly
+    Supervised Table Parsing via Pre-training <https://arxiv.org/abs/2004.02349>`__ by Jonathan Herzig, Paweł Krzysztof
+    Nowak, Thomas Müller, Francesco Piccinno and Julian Martin Eisenschlos.
 32. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
    Attentive Language Models Beyond a Fixed-Length Context <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*,
    Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
@@ -32,9 +32,6 @@ generation.
 .. autoclass:: transformers.LogitsProcessorList
    :members: __call__

-.. autoclass:: transformers.LogitsWarper
-    :members: __call__
-
 .. autoclass:: transformers.MinLengthLogitsProcessor
    :members: __call__

@@ -70,10 +67,3 @@ BeamSearch

 .. autoclass:: transformers.BeamSearchScorer
    :members: process, finalize
-
-Utilities
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autofunction:: transformers.top_k_top_p_filtering
-
-.. autofunction:: transformers.tf_top_k_top_p_filtering
@@ -91,6 +91,8 @@ TensorFlow loss functions
 TensorFlow Helper Functions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

+.. autofunction:: transformers.modeling_tf_utils.cast_bool_to_primitive
+
 .. autofunction:: transformers.modeling_tf_utils.get_initializer

 .. autofunction:: transformers.modeling_tf_utils.keras_serializable
@@ -22,8 +22,6 @@ Utilities

 .. autoclass:: transformers.EvalPrediction

-.. autoclass:: transformers.EvaluationStrategy
-
 .. autofunction:: transformers.set_seed

 .. autofunction:: transformers.torch_distributed_zero_first
@@ -34,15 +32,8 @@ Callbacks internals

 .. autoclass:: transformers.trainer_callback.CallbackHandler

-
 Distributed Evaluation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.trainer_pt_utils.DistributedTensorGatherer
    :members:
-
-
-Distributed Evaluation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.HfArgumentParser
@@ -13,10 +13,9 @@
 Models
 -----------------------------------------------------------------------------------------------------------------------

-The base classes :class:`~transformers.PreTrainedModel`, :class:`~transformers.TFPreTrainedModel`, and
-:class:`~transformers.FlaxPreTrainedModel` implement the common methods for loading/saving a model either from a local
-file or directory, or from a pretrained model configuration provided by the library (downloaded from HuggingFace's AWS
-S3 repository).
+The base classes :class:`~transformers.PreTrainedModel` and :class:`~transformers.TFPreTrainedModel` implement the
+common methods for loading/saving a model either from a local file or directory, or from a pretrained model
+configuration provided by the library (downloaded from HuggingFace's AWS S3 repository).

 :class:`~transformers.PreTrainedModel` and :class:`~transformers.TFPreTrainedModel` also implement a few methods which
 are common among all the models to:
@@ -58,13 +57,6 @@ TFModelUtilsMixin
    :members:


-FlaxPreTrainedModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxPreTrainedModel
-    :members:
-
-
 Generation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -43,10 +43,6 @@ Schedules
 Learning Rate Schedules (Pytorch)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-.. autoclass:: transformers.SchedulerType
-
-.. autofunction:: transformers.get_scheduler
-
 .. autofunction:: transformers.get_constant_schedule


@@ -78,10 +74,6 @@ Learning Rate Schedules (Pytorch)
    :target: /imgs/warmup_linear_schedule.png
    :alt:

-
-.. autofunction:: transformers.get_polynomial_decay_schedule_with_warmup
-
-
 Warmup (TensorFlow)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

@@ -34,7 +34,6 @@ There are two categories of pipeline abstractions to be aware about:
    - :class:`~transformers.TranslationPipeline`
    - :class:`~transformers.ZeroShotClassificationPipeline`
    - :class:`~transformers.Text2TextGenerationPipeline`
-    - :class:`~transformers.TableQuestionAnsweringPipeline`

 The pipeline abstraction
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -74,9 +73,8 @@ FillMaskPipeline
 NerPipeline
 =======================================================================================================================

-.. autoclass:: transformers.NerPipeline
-
-See :class:`~transformers.TokenClassificationPipeline` for all details.
+This class is an alias of the :class:`~transformers.TokenClassificationPipeline` defined below. Please refer to that
+pipeline for documentation and usage examples.

 QuestionAnsweringPipeline
 =======================================================================================================================
@@ -92,13 +90,6 @@ SummarizationPipeline
    :special-members: __call__
    :members:

-TableQuestionAnsweringPipeline
-=======================================================================================================================
-
-.. autoclass:: transformers.TableQuestionAnsweringPipeline
-    :special-members: __call__
-
-
 TextClassificationPipeline
 =======================================================================================================================

@@ -127,13 +118,6 @@ TokenClassificationPipeline
    :special-members: __call__
    :members:

-TranslationPipeline
-=======================================================================================================================
-
-.. autoclass:: transformers.TranslationPipeline
-    :special-members: __call__
-    :members:
-
 ZeroShotClassificationPipeline
 =======================================================================================================================

@@ -63,13 +63,6 @@ Trainer
    :members:


-Seq2SeqTrainer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.Seq2SeqTrainer
-    :members: evaluate, predict
-
-
 TFTrainer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -84,13 +77,6 @@ TrainingArguments
    :members:


-Seq2SeqTrainingArguments
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.Seq2SeqTrainingArguments
-    :members:
-
-
 TFTrainingArguments
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -60,13 +60,6 @@ AlbertTokenizer
        create_token_type_ids_from_sequences, save_vocabulary


-AlbertTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.AlbertTokenizerFast
-    :members:
-
-
 Albert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -114,13 +114,6 @@ AutoModelForQuestionAnswering
    :members:


-AutoModelForTableQuestionAnswering
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.AutoModelForTableQuestionAnswering
-    :members:
-
-
 TFAutoModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -182,10 +175,3 @@ TFAutoModelForQuestionAnswering

 .. autoclass:: transformers.TFAutoModelForQuestionAnswering
    :members:
-
-
-FlaxAutoModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxAutoModel
-    :members:
@@ -98,12 +98,6 @@ BartTokenizer
    :members:


-BartTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.BartTokenizerFast
-    :members:
-

 BartModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -51,9 +51,3 @@ BarthezTokenizer
 .. autoclass:: transformers.BarthezTokenizer
    :members:

-
-BarthezTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.BarthezTokenizerFast
-    :members:
@@ -207,10 +207,3 @@ FlaxBertModel

 .. autoclass:: transformers.FlaxBertModel
    :members: __call__
-
-
-FlaxBertForMaskedLM
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxBertForMaskedLM
-    :members: __call__
@@ -100,15 +100,6 @@ BlenderbotSmallTokenizer
    :members:


-BlenderbotModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-See :obj:`transformers.BartModel` for arguments to `forward` and `generate`
-
-.. autoclass:: transformers.BlenderbotModel
-    :members:
-
-
 BlenderbotForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -54,13 +54,6 @@ CamembertTokenizer
        create_token_type_ids_from_sequences, save_vocabulary


-CamembertTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.CamembertTokenizerFast
-    :members:
-
-
 CamembertModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -97,8 +97,3 @@ TFCTRLLMHeadModel
 .. autoclass:: transformers.TFCTRLLMHeadModel
    :members: call

-TFCTRLForSequenceClassification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFCTRLForSequenceClassification
-    :members: call
@@ -138,9 +138,3 @@ TFOpenAIGPTDoubleHeadsModel

 .. autoclass:: transformers.TFOpenAIGPTDoubleHeadsModel
    :members: call
-
-TFOpenAIGPTForSequenceClassification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFOpenAIGPTForSequenceClassification
-    :members: call
@@ -57,13 +57,6 @@ LayoutLMTokenizer
    :members:


-LayoutLMTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.LayoutLMTokenizerFast
-    :members:
-
-
 LayoutLMModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -34,12 +34,6 @@ contrast to most prior work, we also pretrain Longformer and finetune it on a va
 pretrained Longformer consistently outperforms RoBERTa on long document tasks and sets new state-of-the-art results on
 WikiHop and TriviaQA.*

-Tips:
-
- Since the Longformer is based on RoBERTa, it doesn't have :obj:`token_type_ids`. You don't need to indicate which
-  token belongs to which segment. Just separate your segments with the separation token :obj:`tokenizer.sep_token` (or
-  :obj:`</s>`).
-
 The Authors' code can be found `here <https://github.com/allenai/longformer>`__.

 Longformer Self Attention
@@ -90,20 +90,6 @@ MBartTokenizer
    :members: build_inputs_with_special_tokens, prepare_seq2seq_batch


-MBartTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MBartTokenizerFast
-    :members:
-
-
-MBartModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MBartModel
-    :members:
-
-
 MBartForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -1,15 +1,3 @@
-.. 
-    Copyright 2020 The HuggingFace Team. All rights reserved.
-
-    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-    the License. You may obtain a copy of the License at
-
-        http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
-    an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-    specific language governing permissions and limitations under the License.
-
 MPNet
 -----------------------------------------------------------------------------------------------------------------------

@@ -37,22 +37,6 @@ MT5Config
    :members:


-MT5Tokenizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5Tokenizer
-
-See :class:`~transformers.T5Tokenizer` for all details.
-
-
-MT5TokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5TokenizerFast
-
-See :class:`~transformers.T5TokenizerFast` for all details.
-
-
 MT5Model
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -112,19 +112,6 @@ warning: ``add_tokens`` does not work at the moment.
    :members: __call__, prepare_seq2seq_batch


-PegasusTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.PegasusTokenizerFast
-    :members:
-
-
-PegasusModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.PegasusModel
-
-
 PegasusForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -163,13 +163,6 @@ ReformerTokenizer
    :members: save_vocabulary


-ReformerTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ReformerTokenizerFast
-    :members:
-
-
 ReformerModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -44,9 +44,9 @@ Tips:

  For more information about which prefix to use, it is easiest to look into Appendix D of the `paper
  <https://arxiv.org/pdf/1910.10683.pdf>`__. - For sequence-to-sequence generation, it is recommended to use
-  :obj:`T5ForConditionalGeneration.generate()`. This method takes care of feeding the encoded input via cross-attention
-  layers to the decoder and auto-regressively generates the decoder output. - T5 uses relative scalar embeddings.
-  Encoder input padding can be done on the left and on the right.
+  :obj:`T5ForConditionalGeneration.generate()``. This method takes care of feeding the encoded input via
+  cross-attention layers to the decoder and auto-regressively generates the decoder output. - T5 uses relative scalar
+  embeddings. Encoder input padding can be done on the left and on the right.

 The original code can be found `here <https://github.com/google-research/text-to-text-transfer-transformer>`__.

@@ -55,7 +55,7 @@ Training

 T5 is an encoder-decoder model and converts all NLP problems into a text-to-text format. It is trained using teacher
 forcing. This means that for training we always need an input sequence and a target sequence. The input sequence is fed
-to the model using :obj:`input_ids`. The target sequence is shifted to the right, i.e., prepended by a start-sequence
+to the model using :obj:`input_ids``. The target sequence is shifted to the right, i.e., prepended by a start-sequence
 token and fed to the decoder using the :obj:`decoder_input_ids`. In teacher-forcing style, the target sequence is then
 appended by the EOS token and corresponds to the :obj:`labels`. The PAD token is hereby used as the start-sequence
 token. T5 can be trained / fine-tuned both in a supervised and unsupervised fashion.
@@ -107,13 +107,6 @@ T5Tokenizer
        create_token_type_ids_from_sequences, prepare_seq2seq_batch, save_vocabulary


-T5TokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.T5TokenizerFast
-    :members:
-
-
 T5Model
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -131,7 +124,7 @@ T5EncoderModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.T5EncoderModel
-    :members: forward, parallelize, deparallelize
+    :members: forward

 TFT5Model
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1,13 +1,6 @@
 TAPAS
 -----------------------------------------------------------------------------------------------------------------------

-.. note::
-
-    This is a recently introduced model so the API hasn't been tested extensively. There may be some bugs or slight
-    breaking changes to fix them in the future.
-
-
-
 Overview
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -18,11 +11,9 @@ answering questions about tabular data. Compared to BERT, TAPAS uses relative po
 that encode tabular structure. TAPAS is pre-trained on the masked language modeling (MLM) objective on a large dataset
 comprising millions of tables from English Wikipedia and corresponding texts. For question answering, TAPAS has 2 heads
 on top: a cell selection head and an aggregation head, for (optionally) performing aggregations (such as counting or
-summing) among selected cells. TAPAS has been fine-tuned on several datasets: `SQA
-<https://www.microsoft.com/en-us/download/details.aspx?id=54253>`__ (Sequential Question Answering by Microsoft), `WTQ
-<https://github.com/ppasupat/WikiTableQuestions>`__ (Wiki Table Questions by Stanford University) and `WikiSQL
-<https://github.com/salesforce/WikiSQL>`__ (by Salesforce). It achieves state-of-the-art on both SQA and WTQ, while
-having comparable performance to SOTA on WikiSQL, with a much simpler architecture.
+summing) among selected cells. TAPAS has been fine-tuned on several datasets: SQA (Sequential Question Answering by
+Microsoft), WTQ (Wiki Table Questions by Stanford University) and WikiSQL (by Salesforce). It achieves state-of-the-art
+on both SQA and WTQ, while having comparable performance to SOTA on WikiSQL, with a much simpler architecture.

 The abstract from the paper is the following:

@@ -39,17 +30,17 @@ improving state-of-the-art accuracy on SQA from 55.1 to 67.2 and performing on p
 and WIKITQ, but with a simpler model architecture. We additionally find that transfer learning, which is trivial in our
 setting, from WIKISQL to WIKITQ, yields 48.7 accuracy, 4.2 points above the state-of-the-art.*

-In addition, the authors have further pre-trained TAPAS to recognize **table entailment**, by creating a balanced
-dataset of millions of automatically created training examples which are learned in an intermediate step prior to
-fine-tuning. The authors of TAPAS call this further pre-training intermediate pre-training (since TAPAS is first
-pre-trained on MLM, and then on another dataset). They found that intermediate pre-training further improves
-performance on SQA, achieving a new state-of-the-art as well as state-of-the-art on `TabFact
-<https://github.com/wenhuchen/Table-Fact-Checking>`__, a large-scale dataset with 16k Wikipedia tables for table
-entailment (a binary classification task). For more details, see their follow-up paper: `Understanding tables with
-intermediate pre-training <https://www.aclweb.org/anthology/2020.findings-emnlp.27/>`__ by Julian Martin Eisenschlos,
-Syrine Krichene and Thomas Müller.
+In addition, the authors have further pre-trained TAPAS to recognize table entailment, by creating a balanced dataset
+of millions of automatically created training examples which are learned in an intermediate step prior to fine-tuning.
+The authors of TAPAS call this further pre-training intermediate pre-training (since TAPAS is first pre-trained on MLM,
+and then on another dataset). They found that intermediate pre-training further improves performance on SQA, achieving
+a new state-of-the-art as well as state-of-the-art on TabFact, a large-scale dataset with 16k Wikipedia tables for
+table entailment (a binary classification task). For more details, see their follow-up paper: `Understanding tables
+with intermediate pre-training <https://arxiv.org/abs/2010.00571>`__ by Julian Martin Eisenschlos, Syrine Krichene and
+Thomas Müller.

-The original code can be found `here <https://github.com/google-research/tapas>`__.
+The original code can be found `here <https://github.com/google-research/tapas>`__. This page also includes links to
+the datasets (SQA, WTQ, WikiSQL and TabFact).

 Tips:

@@ -57,20 +48,19 @@ Tips:
  of the table). Note that this is something that was added after the publication of the original TAPAS paper.
  According to the authors, this usually results in a slightly better performance, and allows you to encode longer
  sequences without running out of embeddings. This is reflected in the ``reset_position_index_per_cell`` parameter of
-  :class:`~transformers.TapasConfig`, which is set to ``True`` by default. The default versions of the models available
+  :class:`~transformers.TapasConfig`, which is set to ``True`` by default. The latest versions of the models available
  in the `model hub <https://huggingface.co/models?search=tapas>`_ all use relative position embeddings. You can still
-  use the ones with absolute position embeddings by passing in an additional argument ``revision="no_reset"`` when
-  calling the ``.from_pretrained()`` method. Note that it's usually advised to pad the inputs on the right rather than
-  the left.
+  use the ones with absolute position embeddings by passing in a certain version when calling the `.from_pretrained`
+  method as explained in the model cards. Note that it's usually advised to pad the inputs on the right rather than the
+  left.
 - TAPAS is based on BERT, so ``TAPAS-base`` for example corresponds to a ``BERT-base`` architecture. Of course,
-  TAPAS-large will result in the best performance (the results reported in the paper are from TAPAS-large). Results of
+  TAPAS-large will result in the best performance (the results reported in the paper are from TAPAS-large). Metrics of
  the various sized models are shown on the `original Github repository <https://github.com/google-research/tapas>`_.
 - TAPAS has checkpoints fine-tuned on SQA, which are capable of answering questions related to a table in a
  conversational set-up. This means that you can ask follow-up questions such as "what is his age?" related to the
  previous question. Note that the forward pass of TAPAS is a bit different in case of a conversational set-up: in that
-  case, you have to feed every table-question pair one by one to the model, such that the `prev_labels` token type ids
-  can be overwritten by the predicted `labels` of the model to the previous question. See "Usage" section for more
-  info.
+  case, you have to feed every training example one by one to the model, such that the `prev_labels` token type ids can
+  be overwritten by the predicted `label_ids` of the model to the previous question. See "Usage" section for more info.
 - TAPAS is similar to BERT and therefore relies on the masked language modeling (MLM) objective. It is therefore
  efficient at predicting masked tokens and at NLU in general, but is not optimal for text generation. Models trained
  with a causal language modeling (CLM) objective are better in that regard.
@@ -81,7 +71,9 @@ Usage: fine-tuning

 Here we explain how you can fine-tune :class:`~transformers.TapasForQuestionAnswering` on your own dataset.

-**STEP 1: Choose one of the 3 ways in which you can use TAPAS - or experiment**
+=========================================================================== STEP 1: Choose one of the 3 ways in which
+you can use TAPAS - or experiment
+=======================================================================================================================

 Basically, there are 3 different ways in which one can fine-tune :class:`~transformers.TapasForQuestionAnswering`,
 corresponding to the different datasets on which Tapas was fine-tuned:
@@ -101,7 +93,7 @@ corresponding to the different datasets on which Tapas was fine-tuned:
 To summarize:

 +------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+
-| **Task**                           | **Example dataset**  | **Description**                                                                                                   |
+| **Task**                           | **Example datasets** | **Description**                                                                                                   |
 +------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+
 | Conversational                     | SQA                  | Conversational, only cell selection questions                                                                     |
 +------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+
@@ -110,24 +102,15 @@ To summarize:
 | Strong supervision for aggregation | WikiSQL-supervised   | Questions might involve aggregation, and the model must learn this given the gold aggregation operator            |
 +------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+

-Initializing a model with a pre-trained base and randomly initialized classification heads from the model hub can be
-done as follows (be sure to have installed the `torch-scatter dependency <https://github.com/rusty1s/pytorch_scatter>`_
-for your environment):
+Initializing a model with a pre-trained base and randomly initialized classification heads from the model hub is as
+easy as:

 .. code-block::

-        >>> from transformers import TapasConfig, TapasForQuestionAnswering
+        >>> from transformers import TapasForQuestionAnswering

-        >>> # for example, the base sized model with default SQA configuration
-        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base')
-
-        >>> # or, the base sized model with WTQ configuration
-        >>> config = TapasConfig.from_pretrained('google/tapas-base-finetuned-wtq')
-        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base', config=config)
-
-        >>> # or, the base sized model with WikiSQL configuration
-        >>> config = TapasConfig('google-base-finetuned-wikisql-supervised')
-        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base', config=config)
+        >>> # for example, the base sized model 
+        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base-uncased')


 Of course, you don't necessarily have to follow one of these three ways in which TAPAS was fine-tuned. You can also
@@ -143,7 +126,7 @@ way. Here's an example:
        >>> # you can initialize the classification heads any way you want (see docs of TapasConfig)
        >>> config = TapasConfig(num_aggregation_labels=3, average_logits_per_cell=True, select_one_column=False)
        >>> # initializing the pre-trained base sized model with our custom classification heads
-        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base', config=config)
+        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base-uncased', config=config)

 What you can also do is start from an already fine-tuned checkpoint. A note here is that the already fine-tuned
 checkpoint on WTQ has some issues due to the L2-loss which is somewhat brittle. See `here
@@ -152,7 +135,8 @@ checkpoint on WTQ has some issues due to the L2-loss which is somewhat brittle.
 For a list of all pre-trained and fine-tuned TAPAS checkpoints available in the HuggingFace model hub, see `here
 <https://huggingface.co/models?search=tapas>`__.

-**STEP 2: Prepare your data in the SQA format**
+=========================================== STEP 2: Prepare your data in the SQA format
+=======================================================================================================================

 Second, no matter what you picked above, you should prepare your dataset in the `SQA format
 <https://www.microsoft.com/en-us/download/details.aspx?id=54253>`__. This format is a TSV/CSV file with the following
@@ -179,7 +163,10 @@ SQA format. The author explains this `here
 are not perfect (the ``answer_coordinates`` and ``float_answer`` fields are populated based on the ``answer_text``),
 meaning that WTQ and WikiSQL results could actually be improved.

-**STEP 3: Convert your data into PyTorch tensors using TapasTokenizer**
+
+========================================================================================== STEP 3: Convert your data
+into PyTorch tensors using :class:`~transformers.TapasTokenizer`
+=======================================================================================================================

 Third, given that you've prepared your data in this TSV/CSV format (and corresponding CSV files containing the tabular
 data), you can then use :class:`~transformers.TapasTokenizer` to convert table-question pairs into :obj:`input_ids`,
@@ -189,37 +176,37 @@ data), you can then use :class:`~transformers.TapasTokenizer` to convert table-q
 +------------------------------------+----------------------------------------------------------------------------------------------+
 | **Task**                           | **Required inputs**                                                                          |
 +------------------------------------+----------------------------------------------------------------------------------------------+
-| Conversational                     | ``input_ids``, ``attention_mask``, ``token_type_ids``, ``labels``                            |
+| Conversational                     | ``input_ids``, ``attention_mask``, ``token_type_ids``, ``label_ids``                         |
 +------------------------------------+----------------------------------------------------------------------------------------------+
-| Weak supervision for aggregation   | ``input_ids``, ``attention_mask``, ``token_type_ids``, ``labels``, ``numeric_values``,       |
+| Weak supervision for aggregation   | ``input_ids``, ``attention_mask``, ``token_type_ids``, ``label_ids``, ``numeric_values``,    |
 |                                    | ``numeric_values_scale``, ``float_answer``                                                   |
 +------------------------------------+----------------------------------------------------------------------------------------------+
-| Strong supervision for aggregation | ``input ids``, ``attention mask``, ``token type ids``, ``labels``, ``aggregation_labels``    |
+| Strong supervision for aggregation | ``input ids``, ``attention mask``, ``token type ids``, ``label ids``, ``aggregation_labels`` |
 +------------------------------------+----------------------------------------------------------------------------------------------+

-:class:`~transformers.TapasTokenizer` creates the ``labels``, ``numeric_values`` and ``numeric_values_scale`` based on
-the ``answer_coordinates`` and ``answer_text`` columns of the TSV file. The ``float_answer`` and ``aggregation_labels``
-are already in the TSV file of step 2. Here's an example:
+:class:`~transformers.TapasTokenizer` creates the ``label_ids``, ``numeric_values`` and ``numeric_values_scale`` based
+on the ``answer_coordinates`` and ``answer_text`` columns of the TSV file. The ``float_answer`` and
+``aggregation_labels`` are already in the TSV file of step 2. Here's an example:

 .. code-block::

        >>> from transformers import TapasTokenizer
        >>> import pandas as pd

-        >>> model_name = 'google/tapas-base'
+        >>> model_name = 'google/tapas-base-uncased'
        >>> tokenizer = TapasTokenizer.from_pretrained(model_name)

        >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], 'Number of movies': ["87", "53", "69"]}
        >>> queries = ["What is the name of the first actor?", "How many movies has George Clooney played in?", "What is the total number of movies?"]
-        >>> answer_coordinates = [[(0, 0)], [(2, 1)], [(0, 1), (1, 1), (2, 1)]]
+        >>> answer_coordinates = [[(0, 0)], [(1, 0)], [(0, 2), (1, 2), (2, 2)]]
        >>> answer_text = [["Brad Pitt"], ["69"], ["209"]]
-        >>> table = pd.DataFrame.from_dict(data)
+        >>> table = pd.Dataframe(data)
        >>> inputs = tokenizer(table=table, queries=queries, answer_coordinates=answer_coordinates, answer_text=answer_text, padding='max_length', return_tensors='pt')
        >>> inputs
        {'input_ids': tensor([[ ... ]]), 'attention_mask': tensor([[...]]), 'token_type_ids': tensor([[[...]]]),
-        'numeric_values': tensor([[ ... ]]), 'numeric_values_scale: tensor([[ ... ]]), labels: tensor([[ ... ]])}
+        'numeric_values': tensor([[ ... ]]), 'numeric_values_scale: tensor([[ ... ]]), label_ids: tensor([[ ... ]])}

-Note that :class:`~transformers.TapasTokenizer` expects the data of the table to be **text-only**. You can use
+Note that :class:`~transformers.TapasTokenizer` expects the data of the table to be text-only. You can use
 ``.astype(str)`` on a dataframe to turn it into text-only data. Of course, this only shows how to encode a single
 training example. It is advised to create a PyTorch dataset and a corresponding dataloader:

@@ -238,18 +225,15 @@ training example. It is advised to create a PyTorch dataset and a corresponding
        ...
        ...     def __getitem__(self, idx):
        ...         item = data.iloc[idx]
-        ...         table = pd.read_csv(table_csv_path + item.table_file).astype(str) # be sure to make your table data text only
+        ...         table = pd.read_csv(table_csv_path + item.table_file).astype(str)
        ...         encoding = self.tokenizer(table=table, 
-        ...                                   queries=item.question, 
-        ...                                   answer_coordinates=item.answer_coordinates, 
-        ...                                   answer_text=item.answer_text,
-        ...                                   truncation=True,
-        ...                                   padding="max_length",
-        ...                                   return_tensors="pt"
+        ...                                 queries=item.question, 
+        ...                                 answer_coordinates=item.answer_coordinates, 
+        ...                                 answer_text=item.answer_text,
+        ...                                 padding="max_length",
+        ...                                 return_tensors="pt"
        ...         )
-        ...         # remove the batch dimension which the tokenizer adds by default
-        ...         encoding = {key: val.squeeze(0) for key, val in encoding.items()}
-        ...         # add the float_answer which is also required (weak supervision for aggregation case)
+        ...         # we add the float_answer which is also required (weak supervision for aggregation)
        ...         encoding["float_answer"] = torch.tensor(item.float_answer) 
        ...         return encoding
        ...
@@ -264,52 +248,30 @@ Note that here, we encode each table-question pair independently. This is fine a
 conversational**. In case your dataset involves conversational questions (such as in SQA), then you should first group
 together the ``queries``, ``answer_coordinates`` and ``answer_text`` per table (in the order of their ``position``
 index) and batch encode each table with its questions. This will make sure that the ``prev_labels`` token types (see
-docs of :class:`~transformers.TapasTokenizer`) are set correctly. See `this notebook
-<https://github.com/NielsRogge/Transformers-Tutorials/blob/master/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb>`__
-for more info.
+docs of :class:`~transformers.TapasTokenizer`) are set correctly.

-**STEP 4: Train (fine-tune) TapasForQuestionAnswering**
+=================================================== STEP 4: Train (fine-tune) TapasForQuestionAnswering
+=======================================================================================================================

-You can then fine-tune :class:`~transformers.TapasForQuestionAnswering` using native PyTorch as follows (shown here for
-the weak supervision for aggregation case):
+You can then fine-tune :class:`~transformers.TapasForQuestionAnswering` using native PyTorch as follows:

 .. code-block::

-        >>> from transformers import TapasConfig, TapasForQuestionAnswering, AdamW
+        >>> from transformers import TapasForQuestionAnswering

-        >>> # this is the default WTQ configuration
-        >>> config = TapasConfig(
-        ...            num_aggregation_labels = 4,
-        ...            use_answer_as_supervision = True,
-        ...            answer_loss_cutoff = 0.664694,
-        ...            cell_selection_preference = 0.207951,
-        ...            huber_loss_delta = 0.121194,
-        ...            init_cell_selection_weights_to_zero = True,
-        ...            select_one_column = True,
-        ...            allow_empty_column_selection = False,
-        ...            temperature = 0.0352513,
-        ... )
-        >>> model = TapasForQuestionAnswering.from_pretrained("google/tapas-base", config=config)
-
-        >>> optimizer = AdamW(model.parameters(), lr=5e-5)
+        >>> model = TapasForQuestionAnswering.from_pretrained("google/tapas-base-uncased")

        >>> for epoch in range(2):  # loop over the dataset multiple times
        ...    for idx, batch in enumerate(train_dataloader):
        ...         # get the inputs; 
-        ...         input_ids = batch["input_ids"]
-        ...         attention_mask = batch["attention_mask"]
-        ...         token_type_ids = batch["token_type_ids"]
-        ...         labels = batch["labels"]
-        ...         numeric_values = batch["numeric_values"]
-        ...         numeric_values_scale = batch["numeric_values_scale"]
-        ...         float_answer = batch["float_answer"]
+        ...         input_ids, attention_mask, token_type_ids, label_ids, numeric_values, numeric_values_scale, float_answer = batch

        ...         # zero the parameter gradients
        ...         optimizer.zero_grad()

        ...         # forward + backward + optimize
        ...         outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, 
-        ...                        labels=labels, numeric_values=numeric_values, numeric_values_scale=numeric_values_scale, 
+        ...                        label_ids=label_ids, numeric_values=numeric_values, numeric_values_scale=numeric_values_scale, 
        ...                        float_answer=float_answer)
        ...         loss = outputs.loss
        ...         loss.backward()
@@ -333,20 +295,20 @@ of that:
        >>> from transformers import TapasTokenizer, TapasForQuestionAnswering
        >>> import pandas as pd 

-        >>> model_name = 'google/tapas-base-finetuned-wtq'
+        >>> model_name = 'google/tapas-base-uncased-finetuned-wtq'
        >>> model = TapasForQuestionAnswering.from_pretrained(model_name)
        >>> tokenizer = TapasTokenizer.from_pretrained(model_name)

        >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], 'Number of movies': ["87", "53", "69"]}
        >>> queries = ["What is the name of the first actor?", "How many movies has George Clooney played in?", "What is the total number of movies?"]
-        >>> table = pd.DataFrame.from_dict(data)
+        >>> table = pd.Dataframe(data)
        >>> inputs = tokenizer(table=table, queries=queries, padding='max_length', return_tensors="pt") 
        >>> outputs = model(**inputs)
        >>> predicted_answer_coordinates, predicted_aggregation_indices = tokenizer.convert_logits_to_predictions(
        ...         inputs, 
-        ...         outputs.logits.detach(), 
-        ...         outputs.logits_aggregation.detach()
-        ... )
+        ...         output.logits, 
+        ...         outputs.logits_aggregation
+        ...)

        >>> # let's print out the results:
        >>> id2aggregation = {0: "NONE", 1: "SUM", 2: "AVERAGE", 3:"COUNT"}
@@ -356,15 +318,15 @@ of that:
        >>> for coordinates in predicted_answer_coordinates:
        ...   if len(coordinates) == 1:
        ...     # only a single cell:
-        ...     answers.append(table.iat[coordinates[0]])
+        ...     answers.append(df.iat[coordinates[0]])
        ...   else:
        ...     # multiple cells
        ...     cell_values = []
        ...     for coordinate in coordinates:
-        ...        cell_values.append(table.iat[coordinate])
+        ...        cell_values.append(df.iat[coordinate])
        ...     answers.append(", ".join(cell_values))

-        >>> display(table)
+        >>> display(df)
        >>> print("")
        >>> for query, answer, predicted_agg in zip(queries, answers, aggregation_predictions_string):
        ...   print(query)
@@ -372,17 +334,16 @@ of that:
        ...     print("Predicted answer: " + answer)
        ...   else:
        ...     print("Predicted answer: " + predicted_agg + " > " + answer)    
-        What is the name of the first actor?
-        Predicted answer: Brad Pitt
-        How many movies has George Clooney played in?
-        Predicted answer: COUNT > 69
-        What is the total number of movies?
-        Predicted answer: SUM > 87, 53, 69
+        When was Brad Pitt born?
+        Predicted answer: 18 december 1963
+        Which actor appeared in the least number of movies?
+        Predicted answer: Leonardo Di Caprio
+        What is the average number of movies?
+        Predicted answer: AVERAGE > 87, 53, 69

 In case of a conversational set-up, then each table-question pair must be provided **sequentially** to the model, such
-that the ``prev_labels`` token types can be overwritten by the predicted ``labels`` of the previous table-question
-pair. Again, more info can be found in `this notebook
-<https://github.com/NielsRogge/Transformers-Tutorials/blob/master/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb>`__.
+that the ``prev_labels`` token types can be overwritten by the predicted ``label_ids`` of the previous table-question
+pair.


 Tapas specific outputs
@@ -410,14 +371,14 @@ TapasModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.TapasModel
-    :members: forward
+    :members:


 TapasForMaskedLM
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.TapasForMaskedLM
-    :members: forward
+    :members:


 TapasForSequenceClassification
@@ -431,4 +392,4 @@ TapasForQuestionAnswering
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.TapasForQuestionAnswering
-    :members: forward
+    :members:
@@ -87,14 +87,12 @@ TransfoXLLMHeadModel
 .. autoclass:: transformers.TransfoXLLMHeadModel
    :members: forward

-
 TransfoXLForSequenceClassification
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.TransfoXLForSequenceClassification
    :members: forward

-
 TFTransfoXLModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -107,18 +105,3 @@ TFTransfoXLLMHeadModel

 .. autoclass:: transformers.TFTransfoXLLMHeadModel
    :members: call
-
-
-TFTransfoXLForSequenceClassification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFTransfoXLForSequenceClassification
-    :members: call
-
-
-Internal Layers
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.AdaptiveEmbedding
-
-.. autoclass:: transformers.TFAdaptiveEmbedding
@@ -62,13 +62,6 @@ XLMRobertaTokenizer
        create_token_type_ids_from_sequences, save_vocabulary


-XLMRobertaTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLMRobertaTokenizerFast
-    :members:
-
-
 XLMRobertaModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -62,13 +62,6 @@ XLNetTokenizer
        create_token_type_ids_from_sequences, save_vocabulary


-XLNetTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLNetTokenizerFast
-    :members:
-
-
 XLNet specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -60,7 +60,7 @@ Basic steps
 In order to upload a model, you'll need to first create a git repo. This repo will live on the model hub, allowing
 users to clone it and you (and your organization members) to push to it.

-You can create a model repo **directly from `the /new page on the website <https://huggingface.co/new>`__.**
+You can create a model repo directly from the website, `here <https://huggingface.co/new>`.

 Alternatively, you can use the ``transformers-cli``. The next steps describe that process:

@@ -82,12 +82,12 @@ This creates a repo on the model hub, which can be cloned.

 .. code-block:: bash

+    git clone https://huggingface.co/username/your-model-name
+
    # Make sure you have git-lfs installed
    # (https://git-lfs.github.com/)
    git lfs install

-    git clone https://huggingface.co/username/your-model-name
-
 When you have your local clone of your repo and lfs installed, you can then add/remove from that clone as you would
 with any other git repo.

@@ -98,12 +98,8 @@ with any other git repo.
    echo "hello" >> README.md
    git add . && git commit -m "Update from $USER"

-We are intentionally not wrapping git too much, so that you can go on with the workflow you're used to and the tools
-you already know.
+We are intentionally not wrapping git too much, so as to stay intuitive and easy-to-use.

-The only learning curve you might have compared to regular git is the one for git-lfs. The documentation at
-`git-lfs.github.com <https://git-lfs.github.com/>`__ is decent, but we'll work on a tutorial with some tips and tricks
-in the coming weeks!

 Make your model work on all frameworks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -114,7 +110,7 @@ Make your model work on all frameworks
 You probably have your favorite framework, but so will other users! That's why it's best to upload your model with both
 PyTorch `and` TensorFlow checkpoints to make it easier to use (if you skip this step, users will still be able to load
 your model in another framework, but it will be slower, as it will have to be converted on the fly). Don't worry, it's
-super easy to do (and in a future version, it might all be automatic). You will need to install both PyTorch and
+super easy to do (and in a future version, it will all be automatic). You will need to install both PyTorch and
 TensorFlow for this step, but you don't need to worry about the GPU, so it should be very easy. Check the `TensorFlow
 installation page <https://www.tensorflow.org/install/pip#tensorflow-2.0-rc-is-available>`__ and/or the `PyTorch
 installation page <https://pytorch.org/get-started/locally/#start-locally>`__ to see how.
@@ -196,7 +192,7 @@ status`` command:
    git add --all
    git status

-Finally, the files should be committed:
+Finally, the files should be comitted:

 .. code-block:: bash

@@ -214,20 +210,23 @@ This will upload the folder containing the weights, tokenizer and configuration
 Add a model card
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-To make sure everyone knows what your model can do, what its limitations, potential bias or ethical considerations are,
-please add a README.md model card to your model repo. You can just create it, or there's also a convenient button
-titled "Add a README.md" on your model page. A model card template can be found `here
-<https://github.com/huggingface/model_card>`__ (meta-suggestions are welcome). model card template (meta-suggestions
-are welcome).
-
-.. note::
-
-    Model cards used to live in the 🤗 Transformers repo under `model_cards/`, but for consistency and scalability we
-    migrated every model card from the repo to its corresponding huggingface.co model repo.
+To make sure everyone knows what your model can do, what its limitations and potential bias or ethetical
+considerations, please add a README.md model card to the 🤗 Transformers repo under `model_cards/`. It should then be
+placed in a subfolder with your username or organization, then another subfolder named like your model
+(`awesome-name-you-picked`). Or just click on the "Create a model card on GitHub" button on the model page, it will get
+you directly to the right location. If you need one, `here <https://github.com/huggingface/model_card>`__ is a model
+card template (meta-suggestions are welcome).

 If your model is fine-tuned from another model coming from the model hub (all 🤗 Transformers pretrained models do),
 don't forget to link to its model card so that people can fully trace how your model was built.

+If you have never made a pull request to the 🤗 Transformers repo, look at the :doc:`contributing guide <contributing>`
+to see the steps to follow.
+
+.. note::
+
+    You can also send your model card in the folder you uploaded with the CLI by placing it in a `README.md` file
+    inside `path/to/awesome-name-you-picked/`.

 Using your model
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -263,8 +262,7 @@ First you need to install `git-lfs` in the environment used by the notebook:

    sudo apt-get install git-lfs

-Then you can use either create a repo directly from `huggingface.co <https://huggingface.co/>`__ , or use the
-:obj:`transformers-cli` to create it:
+Then you can use the :obj:`transformers-cli` to create your new repo:


 .. code-block:: bash
@@ -276,14 +274,13 @@ Once it's created, you can clone it and configure it (replace username by your u

 .. code-block:: bash

-    git lfs install
-
    git clone https://username:password@huggingface.co/username/your-model-name
    # Alternatively if you have a token,
    # you can use it instead of your password
    git clone https://username:token@huggingface.co/username/your-model-name

    cd your-model-name
+    git lfs install
    git config --global user.email "email@example.com"
    # Tip: using the same email than for your huggingface.co account will link your commits to your profile
    git config --global user.name "Your name"
@@ -10,7 +10,7 @@
    an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
    specific language governing permissions and limitations under the License.

-Preprocessing data
+reprocessing data
 =======================================================================================================================

 In this tutorial, we'll explore how to preprocess your data using 🤗 Transformers. The main tool for this is what we
@@ -1142,66 +1142,3 @@ To start a debugger at the point of the warning, do this:
 .. code-block:: bash

    pytest tests/test_logging.py -W error::UserWarning --pdb
-
-
-
-Testing Experimental CI Features
-----------------------------------------------------------------------------------------------------------------------
-
-Testing CI features can be potentially problematic as it can interfere with the normal CI functioning. Therefore if a
-new CI feature is to be added, it should be done as following.
-
-1. Create a new dedicated job that tests what needs to be tested
-2. The new job must always succeed so that it gives us a green ✓ (details below).
-3. Let it run for some days to see that a variety of different PR types get to run on it (user fork branches,
-   non-forked branches, branches originating from github.com UI direct file edit, various forced pushes, etc. - there
-   are so many) while monitoring the experimental job's logs (not the overall job green as it's purposefully always
-   green)
-4. When it's clear that everything is solid, then merge the new changes into existing jobs.
-
-That way experiments on CI functionality itself won't interfere with the normal workflow.
-
-Now how can we make the job always succeed while the new CI feature is being developed?
-
-Some CIs, like TravisCI support ignore-step-failure and will report the overall job as successful, but CircleCI and
-Github Actions as of this writing don't support that.
-
-So the following workaround can be used:
-
-1. ``set +euo pipefail`` at the beginning of the run command to suppress most potential failures in the bash script.
-2. the last command must be a success: ``echo "done"`` or just ``true`` will do
-
-Here is an example:
-
-.. code-block:: yaml
-
-    - run:
-        name: run CI experiment
-        command: |
-            set +euo pipefail
-            echo "setting run-all-despite-any-errors-mode"
-            this_command_will_fail
-            echo "but bash continues to run"
-            # emulate another failure
-            false
-            # but the last command must be a success
-            echo "during experiment do not remove: reporting success to CI, even if there were failures"
-
-For simple commands you could also do:
-
-.. code-block:: bash
-
-    cmd_that_may_fail || true
-
-Of course, once satisfied with the results, integrate the experimental step or job with the rest of the normal jobs,
-while removing ``set +euo pipefail`` or any other things you may have added to ensure that the experimental job doesn't
-interfere with the normal CI functioning.
-
-This whole process would have been much easier if we only could set something like ``allow-failure`` for the
-experimental step, and let it fail without impacting the overall status of PRs. But as mentioned earlier CircleCI and
-Github Actions don't support it at the moment.
-
-You can vote for this feature and see where it is at at these CI-specific threads:
-
-* `Github Actions: <https://github.com/actions/toolkit/issues/399>`__
-* `CircleCI: <https://ideas.circleci.com/ideas/CCI-I-344>`__
@@ -16,95 +16,59 @@ limitations under the License.

 # Examples

-This folder contains actively maintained examples of use of 🤗 Transformers organized along NLP tasks. If you are looking for an example that used to
-be in this folder, it may have moved to our [research projects](https://github.com/huggingface/transformers/tree/master/examples/research_projects) subfolder (which contains frozen snapshots of research projects).
+Version 2.9 of 🤗 Transformers introduced a new [`Trainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py) class for PyTorch, and its equivalent [`TFTrainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer_tf.py) for TF 2.
+Running the examples requires PyTorch 1.3.1+ or TensorFlow 2.2+.
+
+Here is the list of all our examples:
+- **grouped by task** (all official examples work for multiple models)
+- with information on whether they are **built on top of `Trainer`/`TFTrainer`** (if not, they still work, they might
+  just lack some features),
+- whether or not they leverage the [🤗 Datasets](https://github.com/huggingface/datasets) library.
+- links to **Colab notebooks** to walk through the scripts and run them easily,
+- links to **Cloud deployments** to be able to deploy large-scale trainings in the Cloud with little to no setup.
+

 ## Important note

 **Important**

-To make sure you can successfully run the latest versions of the example scripts, you have to **install the library from source** and install some example-specific requirements. To do this, execute the following steps in a new virtual environment:
+To make sure you can successfully run the latest versions of the example scripts, you have to **install the library from source** and install some example-specific requirements.
+Execute the following steps in a new virtual environment:
+
 ```bash
 git clone https://github.com/huggingface/transformers
 cd transformers
 pip install .
-```
-Then cd in the example folder of your choice and run
-```bash
-pip install -r requirements.txt
+pip install -r ./examples/requirements.txt
 ```

-Alternatively, you can run the version of the examples as they were for your current version of Transformers via (for instance with v3.5.1):
+Alternatively, you can run the version of the examples as they were for your current version of Transformers via (for instance with v3.4.0):
 ```bash
-git checkout tags/v3.5.1
+git checkout tags/v3.4.0
 ```

 ## The Big Table of Tasks

-Here is the list of all our examples:
- with information on whether they are **built on top of `Trainer`/`TFTrainer`** (if not, they still work, they might
-  just lack some features),
- whether or not they leverage the [🤗 Datasets](https://github.com/huggingface/datasets) library.
- links to **Colab notebooks** to walk through the scripts and run them easily,
-<!--
-Coming soon!
- links to **Cloud deployments** to be able to deploy large-scale trainings in the Cloud with little to no setup.
-->
-
 | Task | Example datasets | Trainer support | TFTrainer support | 🤗 Datasets | Colab
 |---|---|:---:|:---:|:---:|:---:|
 | [**`language-modeling`**](https://github.com/huggingface/transformers/tree/master/examples/language-modeling)       | Raw text        | ✅ | -  | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb)
-| [**`multiple-choice`**](https://github.com/huggingface/transformers/tree/master/examples/multiple-choice)           | SWAG, RACE, ARC | ✅ | ✅ | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ViktorAlm/notebooks/blob/master/MPC_GPU_Demo_for_TF_and_PT.ipynb)
-| [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering)     | SQuAD           | ✅ | ✅ | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://github.com/huggingface/notebooks/blob/master/examples/question_answering.ipynb)
-| [**`summarization`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq)                     | CNN/Daily Mail  | ✅  | - | - | -
 | [**`text-classification`**](https://github.com/huggingface/transformers/tree/master/examples/text-classification)   | GLUE, XNLI      | ✅ | ✅ | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://github.com/huggingface/notebooks/blob/master/examples/text_classification.ipynb)
+| [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER       | ✅ | ✅ | ✅ | -
+| [**`multiple-choice`**](https://github.com/huggingface/transformers/tree/master/examples/multiple-choice)           | SWAG, RACE, ARC | ✅ | ✅ | - | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ViktorAlm/notebooks/blob/master/MPC_GPU_Demo_for_TF_and_PT.ipynb)
+| [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering)     | SQuAD           | ✅ | ✅ | ✅ | -
 | [**`text-generation`**](https://github.com/huggingface/transformers/tree/master/examples/text-generation)           | -               | n/a | n/a | - | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)
-| [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER       | ✅ | ✅ | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://github.com/huggingface/notebooks/blob/master/examples/token_classification.ipynb)
+| [**`distillation`**](https://github.com/huggingface/transformers/tree/master/examples/distillation)                 | All             | - | -  | - | -
+| [**`summarization`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq)                     | CNN/Daily Mail  | ✅  | - | - | -
 | [**`translation`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq)                       | WMT             | ✅  | - | - | -
+| [**`bertology`**](https://github.com/huggingface/transformers/tree/master/examples/bertology)                       | -               | - | - | - | -
+| [**`adversarial`**](https://github.com/huggingface/transformers/tree/master/examples/adversarial)                   | HANS            | ✅ | - | - | -


-<!--
+<br>
+
 ## One-click Deploy to Cloud (wip)

 **Coming soon!**
-->
-
-## Distributed training and mixed precision
-
-All the PyTorch scripts mentioned above work out of the box with distributed training and mixed precision, thanks to
-the [Trainer API](https://huggingface.co/transformers/main_classes/trainer.html). To launch one of them on _n_ GPUS,
-use the following command:
-
-```bash
-python -m torch.distributed.launch \
-    --nproc_per_node number_of_gpu_you_have path_to_script.py \
-	--all_arguments_of_the_script 
-```
-
-As an example, here is how you would fine-tune the BERT large model (with whole word masking) on the text
-classification MNLI task using the `run_glue` script, with 8 GPUs:
-
-```bash
-python -m torch.distributed.launch \
-    --nproc_per_node 8 text-classification/run_glue.py \
-    --model_name_or_path bert-large-uncased-whole-word-masking \
-    --task_name mnli \
-    --do_train \
-    --do_eval \
-    --max_seq_length 128 \
-    --per_device_train_batch_size 8 \
-    --learning_rate 2e-5 \
-    --num_train_epochs 3.0 \
-    --output_dir /tmp/mnli_output/
-```
-
-If you have a GPU with mixed precision capabilities (architecture Pascal or more recent), you can use mixed precision
-training with PyTorch 1.6.0 or latest, or by installing the [Apex](https://github.com/NVIDIA/apex) library for previous
-versions. Just add the flag `--fp16` to your command launching one of the scripts mentioned above!
-
-Using mixed precision training usually results in 2x-speedup for training with the same final results (as shown in
-[this table](https://github.com/huggingface/transformers/tree/master/examples/text-classification#mixed-precision-training)
-for text classification).

 ## Running on TPUs

@@ -113,34 +77,27 @@ When using Tensorflow, TPUs are supported out of the box as a `tf.distribute.Str
 When using PyTorch, we support TPUs thanks to `pytorch/xla`. For more context and information on how to setup your TPU environment refer to Google's documentation and to the
 very detailed [pytorch/xla README](https://github.com/pytorch/xla/blob/master/README.md).

-In this repo, we provide a very simple launcher script named
-[xla_spawn.py](https://github.com/huggingface/transformers/tree/master/examples/xla_spawn.py) that lets you run our
-example scripts on multiple TPU cores without any boilerplate. Just pass a `--num_cores` flag to this script, then your
-regular training script with its arguments (this is similar to the `torch.distributed.launch` helper for
-`torch.distributed`):
+In this repo, we provide a very simple launcher script named [xla_spawn.py](https://github.com/huggingface/transformers/tree/master/examples/xla_spawn.py) that lets you run our example scripts on multiple TPU cores without any boilerplate.
+Just pass a `--num_cores` flag to this script, then your regular training script with its arguments (this is similar to the `torch.distributed.launch` helper for torch.distributed). 
+Note that this approach does not work for examples that use `pytorch-lightning`.
+
+For example for `run_glue`:

 ```bash
-python xla_spawn.py --num_cores num_tpu_you_have \
-    path_to_script.py \
-	--all_arguments_of_the_script 
+python examples/xla_spawn.py --num_cores 8 \
+	examples/text-classification/run_glue.py \
+	--model_name_or_path bert-base-cased \
+	--task_name mnli \
+	--data_dir ./data/glue_data/MNLI \
+	--output_dir ./models/tpu \
+	--overwrite_output_dir \
+	--do_train \
+	--do_eval \
+	--num_train_epochs 1 \
+	--save_steps 20000
 ```

-As an example, here is how you would fine-tune the BERT large model (with whole word masking) on the text
-classification MNLI task using the `run_glue` script, with 8 TPUs:
-
-```bash
-python xla_spawn.py --num_cores 8 \
-    text-classification/run_glue.py \
-    --model_name_or_path bert-large-uncased-whole-word-masking \
-    --task_name mnli \
-    --do_train \
-    --do_eval \
-    --max_seq_length 128 \
-    --per_device_train_batch_size 8 \
-    --learning_rate 2e-5 \
-    --num_train_epochs 3.0 \
-    --output_dir /tmp/mnli_output/
-```
+Feedback and more use cases and benchmarks involving TPUs are welcome, please share with the community.

 ## Logging & Experiment tracking

@@ -1,20 +0,0 @@
-tensorboard
-scikit-learn
-seqeval
-psutil
-sacrebleu
-rouge-score
-tensorflow_datasets
-matplotlib
-git-python==1.0.3
-faiss-cpu
-streamlit
-elasticsearch
-nltk
-pandas
-datasets >= 1.1.3
-fire
-pytest
-conllu
-sentencepiece != 0.1.92
-protobuf
@@ -1,19 +1,3 @@
-<!---
-Copyright 2020 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-->
-
 # 🤗 Benchmark results

 Here, you can find a list of the different benchmark results created by the community.
@@ -1,17 +1,3 @@
-# Copyright 2020 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import csv
 from collections import defaultdict
 from dataclasses import dataclass, field
@@ -1,17 +1,3 @@
-# Copyright 2020 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 # tests directory-specific settings - this file is run automatically
 # by pytest before any tests are run

@@ -0,0 +1,5 @@
+# Community contributed examples
+
+This folder contains examples which are not actively maintained (mostly contributed by the community).
+
+Using these examples together with a recent version of the library usually requires to make small (sometimes big) adaptations to get the scripts working.
@@ -1,7 +1,5 @@
 # Distil*

-Author: @VictorSanh
-
 This folder contains the original code used to train Distil* as well as examples showcasing how to use DistilBERT, DistilRoBERTa and DistilGPT2.

 **January 20, 2020 - Bug fixing** We have recently discovered and fixed [a bug](https://github.com/huggingface/transformers/commit/48cbf267c988b56c71a2380f748a3e6092ccaed3) in the evaluation of our `run_*.py` scripts that caused the reported metrics to be over-estimated on average. We have updated all the metrics with the latest runs.
@@ -1,19 +1,3 @@
-<!---
-Copyright 2020 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-->
-
 ## Language model training

 Fine-tuning (or training from scratch) the library models for language modeling on a text dataset for GPT, GPT-2,
@@ -25,7 +9,8 @@ objectives in our [model summary](https://huggingface.co/transformers/model_summ
 These scripts leverage the 🤗 Datasets library and the Trainer API. You can easily customize them to your needs if you
 need extra processing on your datasets.

-**Note:** The old script `run_language_modeling.py` is still available [here](https://github.com/huggingface/transformers/blob/master/examples/legacy/run_language_modeling.py).
+**Note:** The old script `run_language_modeling.py` is still available
+[here](https://github.com/huggingface/transformers/blob/master/examples/contrib/legacy/run_language_modeling.py).

 The following examples, will run on a datasets hosted on our [hub](https://huggingface.co/datasets) or with your own
 text files for training and validation. We give examples of both below.
@@ -1,3 +0,0 @@
-datasets >= 1.1.3
-sentencepiece != 0.1.92
-protobuf
@@ -113,12 +113,6 @@ class DataTrainingArguments:
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )
-    validation_split_percentage: Optional[int] = field(
-        default=5,
-        metadata={
-            "help": "The percentage of the train set used as validation set in case there's no validation split"
-        },
-    )
    preprocessing_num_workers: Optional[int] = field(
        default=None,
        metadata={"help": "The number of processes to use for the preprocessing."},
@@ -194,17 +188,6 @@ def main():
    if data_args.dataset_name is not None:
        # Downloading and loading a dataset from the hub.
        datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
-        if "validation" not in datasets.keys():
-            datasets["validation"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[:{data_args.validation_split_percentage}%]",
-            )
-            datasets["train"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[{data_args.validation_split_percentage}%:]",
-            )
    else:
        data_files = {}
        if data_args.train_file is not None:
@@ -341,20 +324,9 @@ def main():
            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
            else None
        )
-        train_result = trainer.train(model_path=model_path)
+        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

-        output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
-        if trainer.is_world_process_zero():
-            with open(output_train_file, "w") as writer:
-                logger.info("***** Train results *****")
-                for key, value in sorted(train_result.metrics.items()):
-                    logger.info(f"  {key} = {value}")
-                    writer.write(f"{key} = {value}\n")
-
-            # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
-            trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
-
    # Evaluation
    results = {}
    if training_args.do_eval:
@@ -369,7 +341,7 @@ def main():
        if trainer.is_world_process_zero():
            with open(output_eval_file, "w") as writer:
                logger.info("***** Eval results *****")
-                for key, value in sorted(results.items()):
+                for key, value in results.items():
                    logger.info(f"  {key} = {value}")
                    writer.write(f"{key} = {value}\n")

@@ -103,12 +103,6 @@ class DataTrainingArguments:
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )
-    validation_split_percentage: Optional[int] = field(
-        default=5,
-        metadata={
-            "help": "The percentage of the train set used as validation set in case there's no validation split"
-        },
-    )
    max_seq_length: Optional[int] = field(
        default=None,
        metadata={
@@ -205,17 +199,6 @@ def main():
    if data_args.dataset_name is not None:
        # Downloading and loading a dataset from the hub.
        datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
-        if "validation" not in datasets.keys():
-            datasets["validation"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[:{data_args.validation_split_percentage}%]",
-            )
-            datasets["train"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[{data_args.validation_split_percentage}%:]",
-            )
    else:
        data_files = {}
        if data_args.train_file is not None:
@@ -376,20 +359,9 @@ def main():
            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
            else None
        )
-        train_result = trainer.train(model_path=model_path)
+        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

-        output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
-        if trainer.is_world_process_zero():
-            with open(output_train_file, "w") as writer:
-                logger.info("***** Train results *****")
-                for key, value in sorted(train_result.metrics.items()):
-                    logger.info(f"  {key} = {value}")
-                    writer.write(f"{key} = {value}\n")
-
-            # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
-            trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
-
    # Evaluation
    results = {}
    if training_args.do_eval:
@@ -404,7 +376,7 @@ def main():
        if trainer.is_world_process_zero():
            with open(output_eval_file, "w") as writer:
                logger.info("***** Eval results *****")
-                for key, value in sorted(results.items()):
+                for key, value in results.items():
                    logger.info(f"  {key} = {value}")
                    writer.write(f"{key} = {value}\n")

@@ -134,12 +134,6 @@ class DataTrainingArguments:
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )
-    validation_split_percentage: Optional[int] = field(
-        default=5,
-        metadata={
-            "help": "The percentage of the train set used as validation set in case there's no validation split"
-        },
-    )
    max_seq_length: Optional[int] = field(
        default=None,
        metadata={
@@ -385,7 +379,7 @@ def training_step(optimizer, batch, dropout_rng):
        # Hide away tokens which doesn't participate in the optimization
        token_mask = jnp.where(targets > 0, 1.0, 0.0)

-        logits = model(**batch, params=params, dropout_rng=dropout_rng, train=True)[0]
+        pooled, logits = model(**batch, params=params, dropout_rng=dropout_rng, train=True)
        loss, weight_sum = cross_entropy(logits, targets, token_mask)
        return loss / weight_sum

@@ -407,7 +401,7 @@ def eval_step(params, batch):

    # Hide away tokens which doesn't participate in the optimization
    token_mask = jnp.where(targets > 0, 1.0, 0.0)
-    logits = model(**batch, params=params, train=False)[0]
+    _, logits = model(**batch, params=params, train=False)

    return compute_metrics(logits, targets, token_mask)

@@ -419,7 +413,7 @@ def generate_batch_splits(samples_idx: jnp.ndarray, batch_size: int) -> jnp.ndar
    if samples_to_remove != 0:
        samples_idx = samples_idx[:-samples_to_remove]
    sections_split = nb_samples // batch_size
-    batch_idx = np.split(samples_idx, sections_split)
+    batch_idx = jnp.split(samples_idx, sections_split)
    return batch_idx


@@ -479,17 +473,6 @@ if __name__ == "__main__":
    if data_args.dataset_name is not None:
        # Downloading and loading a dataset from the hub.
        datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
-        if "validation" not in datasets.keys():
-            datasets["validation"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[:{data_args.validation_split_percentage}%]",
-            )
-            datasets["train"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[{data_args.validation_split_percentage}%:]",
-            )
    else:
        data_files = {}
        if data_args.train_file is not None:
@@ -542,9 +525,9 @@ if __name__ == "__main__":

    def tokenize_function(examples):
        # Remove empty lines
-        examples = [line for line in examples if len(line) > 0 and not line.isspace()]
+        examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
        return tokenizer(
-            examples,
+            examples["text"],
            return_special_tokens_mask=True,
            padding=padding,
            truncation=True,
@@ -553,10 +536,9 @@ if __name__ == "__main__":

    tokenized_datasets = datasets.map(
        tokenize_function,
-        input_columns=[text_column_name],
        batched=True,
        num_proc=data_args.preprocessing_num_workers,
-        remove_columns=column_names,
+        remove_columns=[text_column_name],
        load_from_cache_file=not data_args.overwrite_cache,
    )

@@ -572,13 +554,8 @@ if __name__ == "__main__":
    rng = jax.random.PRNGKey(training_args.seed)
    dropout_rngs = jax.random.split(rng, jax.local_device_count())

-    model = FlaxBertForMaskedLM.from_pretrained(
-        "bert-base-cased",
-        dtype=jnp.float32,
-        input_shape=(training_args.train_batch_size, config.max_position_embeddings),
-        seed=training_args.seed,
-        dropout_rate=0.1,
-    )
+    model = FlaxBertForMaskedLM.from_pretrained("bert-base-cased", dtype=jnp.float32, dropout_rate=0.1)
+    model.init(jax.random.PRNGKey(training_args.seed), (training_args.train_batch_size, model.config.max_length))

    # Setup optimizer
    optimizer = Adam(
@@ -589,9 +566,8 @@ if __name__ == "__main__":
    ).create(model.params)

    # Create learning rate scheduler
-    # warmup_steps = 0 causes the Flax optimizer to return NaNs; warmup_steps = 1 is functionally equivalent.
    lr_scheduler_fn = create_learning_rate_scheduler(
-        base_learning_rate=training_args.learning_rate, warmup_steps=min(training_args.warmup_steps, 1)
+        base_learning_rate=training_args.learning_rate, warmup_steps=training_args.warmup_steps
    )

    # Create parallel version of the training and evaluation steps
@@ -630,13 +606,13 @@ if __name__ == "__main__":
        epochs.write(f"Loss: {loss}")

        # ======================== Evaluating ==============================
-        nb_eval_samples = len(tokenized_datasets["validation"])
+        nb_eval_samples = len(tokenized_datasets["test"])
        eval_samples_idx = jnp.arange(nb_eval_samples)
        eval_batch_idx = generate_batch_splits(eval_samples_idx, eval_batch_size)

        eval_metrics = []
        for i, batch_idx in enumerate(tqdm(eval_batch_idx, desc="Evaluating ...", position=2)):
-            samples = [tokenized_datasets["validation"][int(idx)] for idx in batch_idx]
+            samples = [tokenized_datasets["test"][int(idx)] for idx in batch_idx]
            model_inputs = data_collator(samples, pad_to_multiple_of=16)

            # Model forward
@@ -91,12 +91,6 @@ class DataTrainingArguments:
    Arguments pertaining to what data we are going to input our model for training and eval.
    """

-    dataset_name: Optional[str] = field(
-        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
-    )
-    dataset_config_name: Optional[str] = field(
-        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
-    )
    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
    validation_file: Optional[str] = field(
        default=None,
@@ -113,12 +107,6 @@ class DataTrainingArguments:
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )
-    validation_split_percentage: Optional[int] = field(
-        default=5,
-        metadata={
-            "help": "The percentage of the train set used as validation set in case there's no validation split"
-        },
-    )
    max_seq_length: Optional[int] = field(
        default=None,
        metadata={
@@ -215,30 +203,15 @@ def main():
    #
    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
    # download the dataset.
-    if data_args.dataset_name is not None:
-        # Downloading and loading a dataset from the hub.
-        datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
-        if "validation" not in datasets.keys():
-            datasets["validation"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[:{data_args.validation_split_percentage}%]",
-            )
-            datasets["train"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[{data_args.validation_split_percentage}%:]",
-            )
-    else:
-        data_files = {}
-        if data_args.train_file is not None:
-            data_files["train"] = data_args.train_file
-        if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
-        extension = data_args.train_file.split(".")[-1]
-        if extension == "txt":
-            extension = "text"
-        datasets = load_dataset(extension, data_files=data_files)
+    data_files = {}
+    if data_args.train_file is not None:
+        data_files["train"] = data_args.train_file
+    if data_args.validation_file is not None:
+        data_files["validation"] = data_args.validation_file
+    extension = data_args.train_file.split(".")[-1]
+    if extension == "txt":
+        extension = "text"
+    datasets = load_dataset(extension, data_files=data_files)
    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
    # https://huggingface.co/docs/datasets/loading_datasets.html.

@@ -308,7 +281,7 @@ def main():
    # Add the chinese references if provided
    if data_args.train_ref_file is not None:
        tokenized_datasets["train"] = add_chinese_references(tokenized_datasets["train"], data_args.train_ref_file)
-    if data_args.validation_ref_file is not None:
+    if data_args.valid_ref_file is not None:
        tokenized_datasets["validation"] = add_chinese_references(
            tokenized_datasets["validation"], data_args.validation_ref_file
        )
@@ -334,20 +307,9 @@ def main():
            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
            else None
        )
-        train_result = trainer.train(model_path=model_path)
+        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

-        output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
-        if trainer.is_world_process_zero():
-            with open(output_train_file, "w") as writer:
-                logger.info("***** Train results *****")
-                for key, value in sorted(train_result.metrics.items()):
-                    logger.info(f"  {key} = {value}")
-                    writer.write(f"{key} = {value}\n")
-
-            # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
-            trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
-
    # Evaluation
    results = {}
    if training_args.do_eval:
@@ -362,7 +324,7 @@ def main():
        if trainer.is_world_process_zero():
            with open(output_eval_file, "w") as writer:
                logger.info("***** Eval results *****")
-                for key, value in sorted(results.items()):
+                for key, value in results.items():
                    logger.info(f"  {key} = {value}")
                    writer.write(f"{key} = {value}\n")

@@ -93,12 +93,6 @@ class DataTrainingArguments:
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )
-    validation_split_percentage: Optional[int] = field(
-        default=5,
-        metadata={
-            "help": "The percentage of the train set used as validation set in case there's no validation split"
-        },
-    )
    max_seq_length: int = field(
        default=512,
        metadata={
@@ -202,17 +196,6 @@ def main():
    if data_args.dataset_name is not None:
        # Downloading and loading a dataset from the hub.
        datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
-        if "validation" not in datasets.keys():
-            datasets["validation"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[:{data_args.validation_split_percentage}%]",
-            )
-            datasets["train"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[{data_args.validation_split_percentage}%:]",
-            )
    else:
        data_files = {}
        if data_args.train_file is not None:
@@ -363,20 +346,9 @@ def main():
            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
            else None
        )
-        train_result = trainer.train(model_path=model_path)
+        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

-        output_train_file = os.path.join(training_args.output_dir, "train_results.txt")
-        if trainer.is_world_process_zero():
-            with open(output_train_file, "w") as writer:
-                logger.info("***** Train results *****")
-                for key, value in sorted(train_result.metrics.items()):
-                    logger.info(f"  {key} = {value}")
-                    writer.write(f"{key} = {value}\n")
-
-            # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
-            trainer.state.save_to_json(os.path.join(training_args.output_dir, "trainer_state.json"))
-
    # Evaluation
    results = {}
    if training_args.do_eval:
@@ -391,7 +363,7 @@ def main():
        if trainer.is_world_process_zero():
            with open(output_eval_file, "w") as writer:
                logger.info("***** Eval results *****")
-                for key, value in sorted(results.items()):
+                for key, value in results.items():
                    logger.info(f"  {key} = {value}")
                    writer.write(f"{key} = {value}\n")

@@ -1,21 +0,0 @@
-<!---
-Copyright 2020 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-->
-
-# Legacy examples
-
-This folder contains examples which are not actively maintained (mostly contributed by the community).
-
-Using these examples together with a recent version of the library usually requires to make small (sometimes big) adaptations to get the scripts working.
Author	SHA1	Message	Date
LysandreJik	93843345dd	Truncate when overflowing	2020-12-10 17:13:02 -05:00
LysandreJik	f872d4c5ed	Fix the masked LM model	2020-12-10 15:25:33 -05:00
LysandreJik	a1ec0a6a07	Test the Masked LM model	2020-12-10 15:18:02 -05:00
LysandreJik	29d7e4a1ae	Address CI needs	2020-12-10 14:55:17 -05:00
LysandreJik	d240f471a0	Quality	2020-12-09 16:31:08 -05:00
Rogge Niels	4ceab1bc88	Make style & make quality	2020-12-09 21:18:21 +01:00
Lysandre	1f20a93df2	Disable torchscript test	2020-12-09 21:18:21 +01:00
Lysandre	212f4b0d59	Fix truncation when no max length is specified	2020-12-09 21:18:21 +01:00
Rogge Niels	71890e5082	Added more model names	2020-12-09 21:18:21 +01:00
Rogge Niels	abb1e4cb3b	Update PRETRAINED_VOCAB_FILES_MAP and PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES	2020-12-09 21:18:21 +01:00
Rogge Niels	0b8a882226	Comment out positional embedding sizes	2020-12-09 21:18:21 +01:00
Rogge Niels	9e7f856ad8	Change position embeddings sizes to 512 instead of 1024	2020-12-09 21:18:21 +01:00
LysandreJik	de8308d41f	Fix remaining tests	2020-12-09 21:18:21 +01:00
Rogge Niels	9fa0881bc2	Incorporate suggestions by Google authors	2020-12-09 21:18:21 +01:00
Rogge Niels	4cb0339945	Add test for convert_logits_to_predictions method of TapasTokenizer	2020-12-09 21:18:21 +01:00
Rogge Niels	0a40f4300e	Remove print statement	2020-12-09 21:18:21 +01:00
Rogge Niels	0072132a11	All model integration tests pass	2020-12-09 21:18:21 +01:00
Rogge Niels	15788934bc	Fix multiple bugs with integration tests	2020-12-09 21:18:21 +01:00
Rogge Niels	40afab66e6	Remove model cards as they are already on the hub	2020-12-09 21:18:21 +01:00
Rogge Niels	3da59dd1a6	First draft model cards of base sized models	2020-12-09 21:18:21 +01:00
Rogge Niels	ebf850205b	Use Python 3 inheritance rather than Python 2	2020-12-09 21:18:21 +01:00
Rogge Niels	53eade1f89	Improve testing by adding torch_device to inputs and adding dependency on scatter	2020-12-09 21:18:21 +01:00
Rogge Niels	e1c74ea4ac	Fix multiple bugs including soft dependency and new structure of the library	2020-12-09 21:18:21 +01:00
Rogge Niels	a149897eef	First commit: adding all files from tapas_v3	2020-12-09 21:18:15 +01:00