fix positions ids bug

fix inputs handling in tf
Update src/transformers/models/mpnet/modeling_tf_mpnet.py
2020-12-09 11:26:10 +01:00 · 2020-12-09 11:26:10 +01:00 · 2020-12-09 11:26:10 +01:00 · 2020-12-09 11:26:10 +01:00 · 2020-12-09 11:26:10 +01:00 · 2020-12-09 11:26:10 +01:00
306 changed files with 2439 additions and 17602 deletions
@@ -3,6 +3,28 @@ orbs:
    gcp-gke: circleci/gcp-gke@1.0.4
    go: circleci/go@1.3.0

+commands:
+  skip-job-on-doc-only-changes:
+    description: "Do not continue this job and exit with success for PRs with only doc changes"
+    steps:
+
+      - run:
+          name: docs-only changes skip check
+          command: |
+            # pipeline.git.base_revision is not always defined, so only proceed if all external vars are defined
+            if test -n "<< pipeline.git.base_revision >>" && test -n "<< pipeline.git.revision >>" && test -n "$(git diff --name-only << pipeline.git.base_revision >>...<< pipeline.git.revision >>)"
+            then
+                if git diff --name-only << pipeline.git.base_revision >>...<< pipeline.git.revision >> | egrep -qv '\.(md|rst)$'
+                then
+                    echo "Non-docs were modified in this PR, proceeding normally"
+                else
+                    echo "Only docs were modified in this PR, quitting this job"
+                    # disable skipping for now, as circleCI's base_revision is inconsistent leading to invalid ranges
+                    # circleci step halt
+                fi
+            else
+                echo "Can't perform skipping check w/o base_revision defined, continuing the job"
+            fi

 # TPU REFERENCES
 references:
@@ -73,13 +95,13 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch_and_tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece]
-            - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html
            - save_cache:
                key: v0.4-{{ checksum "setup.py" }}
                paths:
@@ -100,13 +122,13 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,torch,testing,sentencepiece]
-            - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html
            - save_cache:
                  key: v0.4-torch-{{ checksum "setup.py" }}
                  paths:
@@ -127,6 +149,7 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-tf-{{ checksum "setup.py" }}
@@ -153,6 +176,7 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                keys:
                    - v0.4-flax-{{ checksum "setup.py" }}
@@ -179,13 +203,13 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,torch,testing,sentencepiece]
-            - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html
            - save_cache:
                  key: v0.4-torch-{{ checksum "setup.py" }}
                  paths:
@@ -206,6 +230,7 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-tf-{{ checksum "setup.py" }}
@@ -230,6 +255,7 @@ jobs:
            RUN_CUSTOM_TOKENIZERS: yes
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
@@ -257,13 +283,14 @@ jobs:
        parallelism: 1
        steps:
            - checkout
+            - skip-job-on-doc-only-changes
            - restore_cache:
                  keys:
                      - v0.4-torch_examples-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,torch,sentencepiece,testing]
-            - run: pip install -r examples/_tests_requirements.txt
+            - run: pip install -r examples/requirements.txt
            - save_cache:
                  key: v0.4-torch_examples-{{ checksum "setup.py" }}
                  paths:
@@ -50,7 +50,6 @@ jobs:
          pip install --upgrade pip
          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
          pip install git+https://github.com/huggingface/datasets
-          pip install pandas torch-scatter -f https://pytorch-geometric.com/whl/torch-$(python -c "import torch; print(''.join(torch.__version__)")+$(python -c "import torch; print(''.join(torch.version.cuda.split('.')))").html

      - name: Are GPUs recognized by our DL frameworks
        run: |
@@ -188,7 +187,6 @@ jobs:
          pip install --upgrade pip
          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
          pip install git+https://github.com/huggingface/datasets
-          pip install pandas torch-scatter -f https://pytorch-geometric.com/whl/torch-$(python -c "import torch; print(''.join(torch.__version__)")+$(python -c "import torch; print(''.join(torch.version.cuda.split('.')))").html

      - name: Are GPUs recognized by our DL frameworks
        run: |
@@ -222,7 +222,6 @@ Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
 ultilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT.
 1. **[SqueezeBert](https://huggingface.co/transformers/model_doc/squeezebert.html)** released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://arxiv.org/abs/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
 1. **[T5](https://huggingface.co/transformers/model_doc/t5.html)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
-1. **[TAPAS](https://huggingface.co/transformers/master/model_doc/tapas.html)** released with the paper [TAPAS: Weakly Supervised Table Parsing via Pre-training](https://arxiv.org/abs/2004.02349) by Jonathan Herzig, Paweł Krzysztof Nowak, Thomas Müller, Francesco Piccinno and Julian Martin Eisenschlos.
 1. **[Transformer-XL](https://huggingface.co/transformers/model_doc/transformerxl.html)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
 1. **[XLM](https://huggingface.co/transformers/model_doc/xlm.html)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau.
 1. **[XLM-ProphetNet](https://huggingface.co/transformers/model_doc/xlmprophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
@@ -1,10 +1,10 @@
 // These two things need to be updated at each release for the version selector.
 // Last stable version
 const stableVersion = "v4.0.0"
-// Dictionary doc folder to label. The last stable version should have an empty key.
+// Dictionary doc folder to label
 const versionMapping = {
    "master": "master",
-    "": "v4.0.0 (stable)",
+    "v4.0.0": "v4.0.0",
    "v3.5.1": "v3.5.0/v3.5.1",
    "v3.4.0": "v3.4.0",
    "v3.3.1": "v3.3.0/v3.3.1",
@@ -176,22 +176,19 @@ and conversion utilities for the following models:
 30. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
    Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by Colin Raffel and Noam Shazeer and Adam
    Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
-31. `TAPAS <https://huggingface.co/transformers/master/model_doc/tapas.html>`__ released with the paper `TAPAS: Weakly
-    Supervised Table Parsing via Pre-training <https://arxiv.org/abs/2004.02349>`__ by Jonathan Herzig, Paweł Krzysztof
-    Nowak, Thomas Müller, Francesco Piccinno and Julian Martin Eisenschlos.
-32. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
+31. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
    Attentive Language Models Beyond a Fixed-Length Context <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*,
    Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
-33. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
+32. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
    Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis Conneau.
-34. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
+33. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
    Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan,
    Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-35. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
+34. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
    Cross-lingual Representation Learning at Scale <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay
    Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke
    Zettlemoyer and Veselin Stoyanov.
-36. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `XLNet: Generalized Autoregressive
+35. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `XLNet: Generalized Autoregressive
    Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by Zhilin Yang*, Zihang Dai*, Yiming
    Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.

@@ -272,8 +269,6 @@ TensorFlow and/or Flax.
 +-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
 |             T5              |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
 +-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
-|            TAPAS            |       ✅       |       ❌       |       ✅        |         ❌         |      ❌      |
-+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
 |       Transformer-XL        |       ✅       |       ❌       |       ✅        |         ✅         |      ❌      |
 +-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
 |             XLM             |       ✅       |       ❌       |       ✅        |         ✅         |      ❌      |
@@ -387,7 +382,6 @@ TensorFlow and/or Flax.
    model_doc/roberta
    model_doc/squeezebert
    model_doc/t5
-    model_doc/tapas
    model_doc/transformerxl
    model_doc/xlm
    model_doc/xlmprophetnet
@@ -15,8 +15,7 @@ Utilities for Generation

 This page lists all the utility functions used by :meth:`~transformers.PretrainedModel.generate`,
 :meth:`~transformers.PretrainedModel.greedy_search`, :meth:`~transformers.PretrainedModel.sample`,
-:meth:`~transformers.PretrainedModel.beam_search`, :meth:`~transformers.PretrainedModel.beam_sample`, and
-:meth:`~transformers.PretrainedModel.group_beam_search`.
+:meth:`~transformers.PretrainedModel.beam_search`, and :meth:`~transformers.PretrainedModel.beam_sample`.

 Most of those are only useful if you are studying the code of the generate methods in the library.

@@ -32,9 +31,6 @@ generation.
 .. autoclass:: transformers.LogitsProcessorList
    :members: __call__

-.. autoclass:: transformers.LogitsWarper
-    :members: __call__
-
 .. autoclass:: transformers.MinLengthLogitsProcessor
    :members: __call__

@@ -56,12 +52,6 @@ generation.
 .. autoclass:: transformers.NoBadWordsLogitsProcessor
    :members: __call__

-.. autoclass:: transformers.PrefixConstrainedLogitsProcessor
-    :members: __call__
-
-.. autoclass:: transformers.HammingDiversityLogitsProcessor
-    :members: __call__
-
 BeamSearch
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -70,10 +60,3 @@ BeamSearch

 .. autoclass:: transformers.BeamSearchScorer
    :members: process, finalize
-
-Utilities
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autofunction:: transformers.top_k_top_p_filtering
-
-.. autofunction:: transformers.tf_top_k_top_p_filtering
@@ -22,8 +22,6 @@ Utilities

 .. autoclass:: transformers.EvalPrediction

-.. autoclass:: transformers.EvaluationStrategy
-
 .. autofunction:: transformers.set_seed

 .. autofunction:: transformers.torch_distributed_zero_first
@@ -34,15 +32,8 @@ Callbacks internals

 .. autoclass:: transformers.trainer_callback.CallbackHandler

-
 Distributed Evaluation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.trainer_pt_utils.DistributedTensorGatherer
    :members:
-
-
-Distributed Evaluation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.HfArgumentParser
@@ -74,10 +74,6 @@ Learning Rate Schedules (Pytorch)
    :target: /imgs/warmup_linear_schedule.png
    :alt:

-
-.. autofunction:: transformers.get_polynomial_decay_schedule_with_warmup
-
-
 Warmup (TensorFlow)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

@@ -73,9 +73,8 @@ FillMaskPipeline
 NerPipeline
 =======================================================================================================================

-.. autoclass:: transformers.NerPipeline
-
-See :class:`~transformers.TokenClassificationPipeline` for all details.
+This class is an alias of the :class:`~transformers.TokenClassificationPipeline` defined below. Please refer to that
+pipeline for documentation and usage examples.

 QuestionAnsweringPipeline
 =======================================================================================================================
@@ -119,13 +118,6 @@ TokenClassificationPipeline
    :special-members: __call__
    :members:

-TranslationPipeline
-=======================================================================================================================
-
-.. autoclass:: transformers.TranslationPipeline
-    :special-members: __call__
-    :members:
-
 ZeroShotClassificationPipeline
 =======================================================================================================================

@@ -60,13 +60,6 @@ AlbertTokenizer
        create_token_type_ids_from_sequences, save_vocabulary


-AlbertTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.AlbertTokenizerFast
-    :members:
-
-
 Albert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -175,10 +175,3 @@ TFAutoModelForQuestionAnswering

 .. autoclass:: transformers.TFAutoModelForQuestionAnswering
    :members:
-
-
-FlaxAutoModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxAutoModel
-    :members:
@@ -98,12 +98,6 @@ BartTokenizer
    :members:


-BartTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.BartTokenizerFast
-    :members:
-

 BartModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -111,6 +105,8 @@ BartModel
 .. autoclass:: transformers.BartModel
    :members: forward

+.. autofunction:: transformers.models.bart.modeling_bart._prepare_bart_decoder_inputs
+

 BartForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -51,9 +51,3 @@ BarthezTokenizer
 .. autoclass:: transformers.BarthezTokenizer
    :members:

-
-BarthezTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.BarthezTokenizerFast
-    :members:
@@ -207,10 +207,3 @@ FlaxBertModel

 .. autoclass:: transformers.FlaxBertModel
    :members: __call__
-
-
-FlaxBertForMaskedLM
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxBertForMaskedLM
-    :members: __call__
@@ -54,13 +54,6 @@ CamembertTokenizer
        create_token_type_ids_from_sequences, save_vocabulary


-CamembertTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.CamembertTokenizerFast
-    :members:
-
-
 CamembertModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -57,13 +57,6 @@ LayoutLMTokenizer
    :members:


-LayoutLMTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.LayoutLMTokenizerFast
-    :members:
-
-
 LayoutLMModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -90,13 +90,6 @@ MBartTokenizer
    :members: build_inputs_with_special_tokens, prepare_seq2seq_batch


-MBartTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MBartTokenizerFast
-    :members:
-
-
 MBartForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -1,15 +1,3 @@
-.. 
-    Copyright 2020 The HuggingFace Team. All rights reserved.
-
-    Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-    the License. You may obtain a copy of the License at
-
-        http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
-    an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-    specific language governing permissions and limitations under the License.
-
 MPNet
 -----------------------------------------------------------------------------------------------------------------------

@@ -37,22 +37,6 @@ MT5Config
    :members:


-MT5Tokenizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5Tokenizer
-
-See :class:`~transformers.T5Tokenizer` for all details.
-
-
-MT5TokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5TokenizerFast
-
-See :class:`~transformers.T5TokenizerFast` for all details.
-
-
 MT5Model
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -112,13 +112,6 @@ warning: ``add_tokens`` does not work at the moment.
    :members: __call__, prepare_seq2seq_batch


-PegasusTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.PegasusTokenizerFast
-    :members:
-
-
 PegasusForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -163,13 +163,6 @@ ReformerTokenizer
    :members: save_vocabulary


-ReformerTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ReformerTokenizerFast
-    :members:
-
-
 ReformerModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -107,13 +107,6 @@ T5Tokenizer
        create_token_type_ids_from_sequences, prepare_seq2seq_batch, save_vocabulary


-T5TokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.T5TokenizerFast
-    :members:
-
-
 T5Model
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -1,444 +0,0 @@
-TAPAS
-----------------------------------------------------------------------------------------------------------------------
-
-Overview
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The TAPAS model was proposed in `TAPAS: Weakly Supervised Table Parsing via Pre-training
-<https://www.aclweb.org/anthology/2020.acl-main.398>`__ by Jonathan Herzig, Paweł Krzysztof Nowak, Thomas Müller,
-Francesco Piccinno and Julian Martin Eisenschlos. It's a BERT-based model specifically designed (and pre-trained) for
-answering questions about tabular data. Compared to BERT, TAPAS uses relative position embeddings and has 7 token types
-that encode tabular structure. TAPAS is pre-trained on the masked language modeling (MLM) objective on a large dataset
-comprising millions of tables from English Wikipedia and corresponding texts. For question answering, TAPAS has 2 heads
-on top: a cell selection head and an aggregation head, for (optionally) performing aggregations (such as counting or
-summing) among selected cells. TAPAS has been fine-tuned on several datasets: `SQA
-<https://www.microsoft.com/en-us/download/details.aspx?id=54253>`_ (Sequential Question Answering by Microsoft), `WTQ
-<https://github.com/ppasupat/WikiTableQuestions>`__ (Wiki Table Questions by Stanford University) and `WikiSQL
-<https://github.com/salesforce/WikiSQL>`__ (by Salesforce). It achieves state-of-the-art on both SQA and WTQ, while
-having comparable performance to SOTA on WikiSQL, with a much simpler architecture.
-
-The abstract from the paper is the following:
-
-*Answering natural language questions over tables is usually seen as a semantic parsing task. To alleviate the
-collection cost of full logical forms, one popular approach focuses on weak supervision consisting of denotations
-instead of logical forms. However, training semantic parsers from weak supervision poses difficulties, and in addition,
-the generated logical forms are only used as an intermediate step prior to retrieving the denotation. In this paper, we
-present TAPAS, an approach to question answering over tables without generating logical forms. TAPAS trains from weak
-supervision, and predicts the denotation by selecting table cells and optionally applying a corresponding aggregation
-operator to such selection. TAPAS extends BERT's architecture to encode tables as input, initializes from an effective
-joint pre-training of text segments and tables crawled from Wikipedia, and is trained end-to-end. We experiment with
-three different semantic parsing datasets, and find that TAPAS outperforms or rivals semantic parsing models by
-improving state-of-the-art accuracy on SQA from 55.1 to 67.2 and performing on par with the state-of-the-art on WIKISQL
-and WIKITQ, but with a simpler model architecture. We additionally find that transfer learning, which is trivial in our
-setting, from WIKISQL to WIKITQ, yields 48.7 accuracy, 4.2 points above the state-of-the-art.*
-
-In addition, the authors have further pre-trained TAPAS to recognize **table entailment**, by creating a balanced
-dataset of millions of automatically created training examples which are learned in an intermediate step prior to
-fine-tuning. The authors of TAPAS call this further pre-training intermediate pre-training (since TAPAS is first
-pre-trained on MLM, and then on another dataset). They found that intermediate pre-training further improves
-performance on SQA, achieving a new state-of-the-art as well as state-of-the-art on `TabFact
-<https://github.com/wenhuchen/Table-Fact-Checking>`__, a large-scale dataset with 16k Wikipedia tables for table
-entailment (a binary classification task). For more details, see their follow-up paper: `Understanding tables with
-intermediate pre-training <https://www.aclweb.org/anthology/2020.findings-emnlp.27/>`__ by Julian Martin Eisenschlos,
-Syrine Krichene and Thomas Müller.
-
-The original code can be found `here <https://github.com/google-research/tapas>`__.
-
-Tips:
-
- TAPAS is a model that uses relative position embeddings by default (restarting the position embeddings at every cell
-  of the table). Note that this is something that was added after the publication of the original TAPAS paper.
-  According to the authors, this usually results in a slightly better performance, and allows you to encode longer
-  sequences without running out of embeddings. This is reflected in the ``reset_position_index_per_cell`` parameter of
-  :class:`~transformers.TapasConfig`, which is set to ``True`` by default. The latest versions of the models available
-  in the `model hub <https://huggingface.co/models?search=tapas>`_ all use relative position embeddings. You can still
-  use the ones with absolute position embeddings by passing in a certain version when calling the
-  ``.from_pretrained()`` method as explained in the model cards. Note that it's usually advised to pad the inputs on
-  the right rather than the left.
- TAPAS is based on BERT, so ``TAPAS-base`` for example corresponds to a ``BERT-base`` architecture. Of course,
-  TAPAS-large will result in the best performance (the results reported in the paper are from TAPAS-large). Results of
-  the various sized models are shown on the `original Github repository <https://github.com/google-research/tapas>`_.
- TAPAS has checkpoints fine-tuned on SQA, which are capable of answering questions related to a table in a
-  conversational set-up. This means that you can ask follow-up questions such as "what is his age?" related to the
-  previous question. Note that the forward pass of TAPAS is a bit different in case of a conversational set-up: in that
-  case, you have to feed every table-question pair one by one to the model, such that the `prev_labels` token type ids
-  can be overwritten by the predicted `labels` of the model to the previous question. See "Usage" section for more
-  info.
- TAPAS is similar to BERT and therefore relies on the masked language modeling (MLM) objective. It is therefore
-  efficient at predicting masked tokens and at NLU in general, but is not optimal for text generation. Models trained
-  with a causal language modeling (CLM) objective are better in that regard.
-
-
-Usage: fine-tuning
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Here we explain how you can fine-tune :class:`~transformers.TapasForQuestionAnswering` on your own dataset.
-
-**STEP 1: Choose one of the 3 ways in which you can use TAPAS - or experiment**
-
-Basically, there are 3 different ways in which one can fine-tune :class:`~transformers.TapasForQuestionAnswering`,
-corresponding to the different datasets on which Tapas was fine-tuned:
-
-1. SQA: if you're interested in asking follow-up questions related to a table, in a conversational set-up. For example
-   if you first ask "what's the name of the first actor?" then you can ask a follow-up question such as "how old is
-   he?". Here, questions do not involve any aggregation (all questions are cell selection questions).
-2. WTQ: if you're not interested in asking questions in a conversational set-up, but rather just asking questions
-   related to a table, which might involve aggregation, such as counting a number of rows, summing up cell values or
-   averaging cell values. You can then for example ask "what's the total number of goals Cristiano Ronaldo made in his
-   career?". This case is also called **weak supervision**, since the model itself must learn the appropriate
-   aggregation operator (SUM/COUNT/AVERAGE/NONE) given only the answer to the question as supervision.
-3. WikiSQL-supervised: this dataset is based on WikiSQL with the model being given the ground truth aggregation
-   operator during training. This is also called **strong supervision**. Here, learning the appropriate aggregation
-   operator is much easier.
-
-To summarize:
-
-+------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+
-| **Task**                           | **Example dataset**  | **Description**                                                                                                   |
-+------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+
-| Conversational                     | SQA                  | Conversational, only cell selection questions                                                                     |
-+------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+
-| Weak supervision for aggregation   | WTQ                  | Questions might involve aggregation, and the model must learn this given only the answer as supervision           |
-+------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+
-| Strong supervision for aggregation | WikiSQL-supervised   | Questions might involve aggregation, and the model must learn this given the gold aggregation operator            |
-+------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+
-
-Initializing a model with a pre-trained base and randomly initialized classification heads from the model hub can be
-done as follows (be sure to have installed the `torch-scatter dependency <https://github.com/rusty1s/pytorch_scatter>`_
-for your environment):
-
-.. code-block::
-
-        >>> from transformers import TapasConfig, TapasForQuestionAnswering
-
-        >>> # for example, the base sized model with default SQA configuration
-        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base')
-
-        >>> # or, the base sized model with WTQ configuration
-        >>> config = TapasConfig(
-        ...            num_aggregation_labels = 4,
-        ...            use_answer_as_supervision = True,
-        ...            answer_loss_cutoff = 0.664694,
-        ...            cell_selection_preference = 0.207951,
-        ...            huber_loss_delta = 0.121194,
-        ...            init_cell_selection_weights_to_zero = True,
-        ...            select_one_column = True,
-        ...            allow_empty_column_selection = False,
-        ...            temperature = 0.0352513,
-        ... )
-        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base', config=config)
-
-        >>> # or, the base sized model with WikiSQL configuration
-        >>> config = TapasConfig(
-        ...            num_aggregation_labels = 4
-        ...            use_answer_as_supervision = False
-        ...            answer_loss_cutoff = 36.4519
-        ...            cell_selection_preference = 0.903421
-        ...            huber_loss_delta = 222.088
-        ...            init_cell_selection_weights_to_zero = True
-        ...            select_one_column = True
-        ...            allow_empty_column_selection = True
-        ...            temperature = 0.763141
-        ... )
-        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base', config=config)
-
-
-Of course, you don't necessarily have to follow one of these three ways in which TAPAS was fine-tuned. You can also
-experiment by defining any hyperparameters you want when initializing :class:`~transformers.TapasConfig`, and then
-create a :class:`~transformers.TapasForQuestionAnswering` based on that configuration. For example, if you have a
-dataset that has both conversational questions and questions that might involve aggregation, then you can do it this
-way. Here's an example:
-
-.. code-block::
-
-        >>> from transformers import TapasConfig, TapasForQuestionAnswering
-
-        >>> # you can initialize the classification heads any way you want (see docs of TapasConfig)
-        >>> config = TapasConfig(num_aggregation_labels=3, average_logits_per_cell=True, select_one_column=False)
-        >>> # initializing the pre-trained base sized model with our custom classification heads
-        >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base', config=config)
-
-What you can also do is start from an already fine-tuned checkpoint. A note here is that the already fine-tuned
-checkpoint on WTQ has some issues due to the L2-loss which is somewhat brittle. See `here
-<https://github.com/google-research/tapas/issues/91#issuecomment-735719340>`__ for more info.
-
-For a list of all pre-trained and fine-tuned TAPAS checkpoints available in the HuggingFace model hub, see `here
-<https://huggingface.co/models?search=tapas>`__.
-
-**STEP 2: Prepare your data in the SQA format**
-
-Second, no matter what you picked above, you should prepare your dataset in the `SQA format
-<https://www.microsoft.com/en-us/download/details.aspx?id=54253>`__. This format is a TSV/CSV file with the following
-columns:
-
- ``id``: optional, id of the table-question pair, for bookkeeping purposes.
- ``annotator``: optional, id of the person who annotated the table-question pair, for bookkeeping purposes.
- ``position``: integer indicating if the question is the first, second, third,... related to the table. Only required
-  in case of conversational setup (SQA). You don't need this column in case you're going for WTQ/WikiSQL-supervised.
- ``question``: string
- ``table_file``: string, name of a csv file containing the tabular data
- ``answer_coordinates``: list of one or more tuples (each tuple being a cell coordinate, i.e. row, column pair that is
-  part of the answer)
- ``answer_text``: list of one or more strings (each string being a cell value that is part of the answer)
- ``aggregation_label``: index of the aggregation operator. Only required in case of strong supervision for aggregation
-  (the WikiSQL-supervised case)
- ``float_answer``: the float answer to the question, if there is one (np.nan if there isn't). Only required in case of
-  weak supervision for aggregation (such as WTQ and WikiSQL)
-
-The tables themselves should be present in a folder, each table being a separate csv file. Note that the authors of the
-TAPAS algorithm used conversion scripts with some automated logic to convert the other datasets (WTQ, WikiSQL) into the
-SQA format. The author explains this `here
-<https://github.com/google-research/tapas/issues/50#issuecomment-705465960>`__. Interestingly, these conversion scripts
-are not perfect (the ``answer_coordinates`` and ``float_answer`` fields are populated based on the ``answer_text``),
-meaning that WTQ and WikiSQL results could actually be improved.
-
-**STEP 3: Convert your data into PyTorch tensors using TapasTokenizer**
-
-Third, given that you've prepared your data in this TSV/CSV format (and corresponding CSV files containing the tabular
-data), you can then use :class:`~transformers.TapasTokenizer` to convert table-question pairs into :obj:`input_ids`,
-:obj:`attention_mask`, :obj:`token_type_ids` and so on. Again, based on which of the three cases you picked above,
-:class:`~transformers.TapasForQuestionAnswering` requires different inputs to be fine-tuned:
-
-+------------------------------------+----------------------------------------------------------------------------------------------+
-| **Task**                           | **Required inputs**                                                                          |
-+------------------------------------+----------------------------------------------------------------------------------------------+
-| Conversational                     | ``input_ids``, ``attention_mask``, ``token_type_ids``, ``labels``                            |
-+------------------------------------+----------------------------------------------------------------------------------------------+
-| Weak supervision for aggregation   | ``input_ids``, ``attention_mask``, ``token_type_ids``, ``labels``, ``numeric_values``,       |
-|                                    | ``numeric_values_scale``, ``float_answer``                                                   |
-+------------------------------------+----------------------------------------------------------------------------------------------+
-| Strong supervision for aggregation | ``input ids``, ``attention mask``, ``token type ids``, ``labels``, ``aggregation_labels``    |
-+------------------------------------+----------------------------------------------------------------------------------------------+
-
-:class:`~transformers.TapasTokenizer` creates the ``labels``, ``numeric_values`` and ``numeric_values_scale`` based on
-the ``answer_coordinates`` and ``answer_text`` columns of the TSV file. The ``float_answer`` and ``aggregation_labels``
-are already in the TSV file of step 2. Here's an example:
-
-.. code-block::
-
-        >>> from transformers import TapasTokenizer
-        >>> import pandas as pd
-
-        >>> model_name = 'google/tapas-base'
-        >>> tokenizer = TapasTokenizer.from_pretrained(model_name)
-
-        >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], 'Number of movies': ["87", "53", "69"]}
-        >>> queries = ["What is the name of the first actor?", "How many movies has George Clooney played in?", "What is the total number of movies?"]
-        >>> answer_coordinates = [[(0, 0)], [(2, 1)], [(0, 1), (1, 1), (2, 1)]]
-        >>> answer_text = [["Brad Pitt"], ["69"], ["209"]]
-        >>> table = pd.DataFrame.from_dict(data)
-        >>> inputs = tokenizer(table=table, queries=queries, answer_coordinates=answer_coordinates, answer_text=answer_text, padding='max_length', return_tensors='pt')
-        >>> inputs
-        {'input_ids': tensor([[ ... ]]), 'attention_mask': tensor([[...]]), 'token_type_ids': tensor([[[...]]]),
-        'numeric_values': tensor([[ ... ]]), 'numeric_values_scale: tensor([[ ... ]]), labels: tensor([[ ... ]])}
-
-Note that :class:`~transformers.TapasTokenizer` expects the data of the table to be **text-only**. You can use
-``.astype(str)`` on a dataframe to turn it into text-only data. Of course, this only shows how to encode a single
-training example. It is advised to create a PyTorch dataset and a corresponding dataloader:
-
-.. code-block::
-
-        >>> import torch
-        >>> import pandas as pd
-
-        >>> tsv_path = "your_path_to_the_tsv_file"
-        >>> table_csv_path = "your_path_to_a_directory_containing_all_csv_files"
-
-        >>> class TableDataset(torch.utils.data.Dataset):
-        ...     def __init__(self, data, tokenizer):
-        ...         self.data = data
-        ...         self.tokenizer = tokenizer
-        ...
-        ...     def __getitem__(self, idx):
-        ...         item = data.iloc[idx]
-        ...         table = pd.read_csv(table_csv_path + item.table_file).astype(str) # be sure to make your table data text only
-        ...         encoding = self.tokenizer(table=table, 
-        ...                                   queries=item.question, 
-        ...                                   answer_coordinates=item.answer_coordinates, 
-        ...                                   answer_text=item.answer_text,
-        ...                                   truncation=True,
-        ...                                   padding="max_length",
-        ...                                   return_tensors="pt"
-        ...         )
-        ...         # remove the batch dimension which the tokenizer adds by default
-        ...         encoding = {key: val.squeeze(0) for key, val in encoding.items()}
-        ...         # add the float_answer which is also required (weak supervision for aggregation case)
-        ...         encoding["float_answer"] = torch.tensor(item.float_answer) 
-        ...         return encoding
-        ...
-        ...     def __len__(self):
-        ...        return len(self.data)
-
-        >>> data = pd.read_csv(tsv_path, sep='\t')
-        >>> train_dataset = TableDataset(data, tokenizer)
-        >>> train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=32)
-
-Note that here, we encode each table-question pair independently. This is fine as long as your dataset is **not
-conversational**. In case your dataset involves conversational questions (such as in SQA), then you should first group
-together the ``queries``, ``answer_coordinates`` and ``answer_text`` per table (in the order of their ``position``
-index) and batch encode each table with its questions. This will make sure that the ``prev_labels`` token types (see
-docs of :class:`~transformers.TapasTokenizer`) are set correctly.
-
-**STEP 4: Train (fine-tune) TapasForQuestionAnswering**
-
-You can then fine-tune :class:`~transformers.TapasForQuestionAnswering` using native PyTorch as follows (shown here for
-the weak supervision for aggregation case):
-
-.. code-block::
-
-        >>> from transformers import TapasConfig, TapasForQuestionAnswering, AdamW
-
-        >>> # this is the default WTQ configuration
-        >>> config = TapasConfig(
-        ...            num_aggregation_labels = 4,
-        ...            use_answer_as_supervision = True,
-        ...            answer_loss_cutoff = 0.664694,
-        ...            cell_selection_preference = 0.207951,
-        ...            huber_loss_delta = 0.121194,
-        ...            init_cell_selection_weights_to_zero = True,
-        ...            select_one_column = True,
-        ...            allow_empty_column_selection = False,
-        ...            temperature = 0.0352513,
-        ... )
-        >>> model = TapasForQuestionAnswering.from_pretrained("google/tapas-base", config=config)
-
-        >>> optimizer = AdamW(model.parameters(), lr=5e-5)
-
-        >>> for epoch in range(2):  # loop over the dataset multiple times
-        ...    for idx, batch in enumerate(train_dataloader):
-        ...         # get the inputs; 
-        ...         input_ids = batch["input_ids"]
-        ...         attention_mask = batch["attention_mask"]
-        ...         token_type_ids = batch["token_type_ids"]
-        ...         labels = batch["labels"]
-        ...         numeric_values = batch["numeric_values"]
-        ...         numeric_values_scale = batch["numeric_values_scale"]
-        ...         float_answer = batch["float_answer"]
-
-        ...         # zero the parameter gradients
-        ...         optimizer.zero_grad()
-
-        ...         # forward + backward + optimize
-        ...         outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, 
-        ...                        labels=labels, numeric_values=numeric_values, numeric_values_scale=numeric_values_scale, 
-        ...                        float_answer=float_answer)
-        ...         loss = outputs.loss
-        ...         loss.backward()
-        ...         optimizer.step()
-
-Usage: inference
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Here we explain how you can use :class:`~transformers.TapasForQuestionAnswering` for inference (i.e. making predictions
-on new data). For inference, only ``input_ids``, ``attention_mask`` and ``token_type_ids`` (which you can obtain using
-:class:`~transformers.TapasTokenizer`) have to be provided to the model to obtain the logits. Next, you can use the
-handy ``convert_logits_to_predictions`` method of :class:`~transformers.TapasTokenizer` to convert these into predicted
-coordinates and optional aggregation indices.
-
-However, note that inference is **different** depending on whether or not the setup is conversational. In a
-non-conversational set-up, inference can be done in parallel on all table-question pairs of a batch. Here's an example
-of that:
-
-.. code-block::
-
-        >>> from transformers import TapasTokenizer, TapasForQuestionAnswering
-        >>> import pandas as pd 
-
-        >>> model_name = 'google/tapas-base-finetuned-wtq'
-        >>> model = TapasForQuestionAnswering.from_pretrained(model_name)
-        >>> tokenizer = TapasTokenizer.from_pretrained(model_name)
-
-        >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], 'Number of movies': ["87", "53", "69"]}
-        >>> queries = ["What is the name of the first actor?", "How many movies has George Clooney played in?", "What is the total number of movies?"]
-        >>> table = pd.DataFrame.from_dict(data)
-        >>> inputs = tokenizer(table=table, queries=queries, padding='max_length', return_tensors="pt") 
-        >>> outputs = model(**inputs)
-        >>> predicted_answer_coordinates, predicted_aggregation_indices = tokenizer.convert_logits_to_predictions(
-        ...         inputs, 
-        ...         outputs.logits.detach(), 
-        ...         outputs.logits_aggregation.detach()
-        ...)
-
-        >>> # let's print out the results:
-        >>> id2aggregation = {0: "NONE", 1: "SUM", 2: "AVERAGE", 3:"COUNT"}
-        >>> aggregation_predictions_string = [id2aggregation[x] for x in predicted_aggregation_indices]
-
-        >>> answers = []
-        >>> for coordinates in predicted_answer_coordinates:
-        ...   if len(coordinates) == 1:
-        ...     # only a single cell:
-        ...     answers.append(table.iat[coordinates[0]])
-        ...   else:
-        ...     # multiple cells
-        ...     cell_values = []
-        ...     for coordinate in coordinates:
-        ...        cell_values.append(table.iat[coordinate])
-        ...     answers.append(", ".join(cell_values))
-
-        >>> display(table)
-        >>> print("")
-        >>> for query, answer, predicted_agg in zip(queries, answers, aggregation_predictions_string):
-        ...   print(query)
-        ...   if predicted_agg == "NONE":
-        ...     print("Predicted answer: " + answer)
-        ...   else:
-        ...     print("Predicted answer: " + predicted_agg + " > " + answer)    
-        What is the name of the first actor?
-        Predicted answer: Brad Pitt
-        How many movies has George Clooney played in?
-        Predicted answer: COUNT > 69
-        What is the total number of movies?
-        Predicted answer: SUM > 87, 53, 69
-
-In case of a conversational set-up, then each table-question pair must be provided **sequentially** to the model, such
-that the ``prev_labels`` token types can be overwritten by the predicted ``labels`` of the previous table-question
-pair.
-
-
-Tapas specific outputs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.models.tapas.modeling_tapas.TableQuestionAnsweringOutput
-    :members:
-
-
-TapasConfig
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TapasConfig
-    :members:
-
-
-TapasTokenizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TapasTokenizer
-    :members: __call__, convert_logits_to_predictions, save_vocabulary
-
-
-TapasModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TapasModel
-    :members:
-
-
-TapasForMaskedLM
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TapasForMaskedLM
-    :members:
-
-
-TapasForSequenceClassification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TapasForSequenceClassification
-    :members: forward
-
-
-TapasForQuestionAnswering
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TapasForQuestionAnswering
-    :members:
@@ -105,11 +105,3 @@ TFTransfoXLLMHeadModel

 .. autoclass:: transformers.TFTransfoXLLMHeadModel
    :members: call
-
-
-Internal Layers
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.AdaptiveEmbedding
-
-.. autoclass:: transformers.TFAdaptiveEmbedding
@@ -62,13 +62,6 @@ XLMRobertaTokenizer
        create_token_type_ids_from_sequences, save_vocabulary


-XLMRobertaTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLMRobertaTokenizerFast
-    :members:
-
-
 XLMRobertaModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -62,13 +62,6 @@ XLNetTokenizer
        create_token_type_ids_from_sequences, save_vocabulary


-XLNetTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLNetTokenizerFast
-    :members:
-
-
 XLNet specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -16,58 +16,59 @@ limitations under the License.

 # Examples

-This folder contains actively maintained examples of use of 🤗 Transformers organized along NLP tasks. If you are looking for an example that used to
-be in this folder, it may have moved to our [research projects](https://github.com/huggingface/transformers/tree/master/examples/research_projects) subfolder (which contains frozen snapshots of research projects).
+Version 2.9 of 🤗 Transformers introduced a new [`Trainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py) class for PyTorch, and its equivalent [`TFTrainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer_tf.py) for TF 2.
+Running the examples requires PyTorch 1.3.1+ or TensorFlow 2.2+.
+
+Here is the list of all our examples:
+- **grouped by task** (all official examples work for multiple models)
+- with information on whether they are **built on top of `Trainer`/`TFTrainer`** (if not, they still work, they might
+  just lack some features),
+- whether or not they leverage the [🤗 Datasets](https://github.com/huggingface/datasets) library.
+- links to **Colab notebooks** to walk through the scripts and run them easily,
+- links to **Cloud deployments** to be able to deploy large-scale trainings in the Cloud with little to no setup.
+

 ## Important note

 **Important**

-To make sure you can successfully run the latest versions of the example scripts, you have to **install the library from source** and install some example-specific requirements. To do this, execute the following steps in a new virtual environment:
+To make sure you can successfully run the latest versions of the example scripts, you have to **install the library from source** and install some example-specific requirements.
+Execute the following steps in a new virtual environment:
+
 ```bash
 git clone https://github.com/huggingface/transformers
 cd transformers
 pip install .
-```
-Then cd in the example folder of your choice and run
-```bash
-pip install -r requirements.txt
+pip install -r ./examples/requirements.txt
 ```

-Alternatively, you can run the version of the examples as they were for your current version of Transformers via (for instance with v3.5.1):
+Alternatively, you can run the version of the examples as they were for your current version of Transformers via (for instance with v3.4.0):
 ```bash
-git checkout tags/v3.5.1
+git checkout tags/v3.4.0
 ```

 ## The Big Table of Tasks

-Here is the list of all our examples:
- with information on whether they are **built on top of `Trainer`/`TFTrainer`** (if not, they still work, they might
-  just lack some features),
- whether or not they leverage the [🤗 Datasets](https://github.com/huggingface/datasets) library.
- links to **Colab notebooks** to walk through the scripts and run them easily,
-<!--
-Coming soon!
- links to **Cloud deployments** to be able to deploy large-scale trainings in the Cloud with little to no setup.
-->
-
 | Task | Example datasets | Trainer support | TFTrainer support | 🤗 Datasets | Colab
 |---|---|:---:|:---:|:---:|:---:|
 | [**`language-modeling`**](https://github.com/huggingface/transformers/tree/master/examples/language-modeling)       | Raw text        | ✅ | -  | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb)
+| [**`text-classification`**](https://github.com/huggingface/transformers/tree/master/examples/text-classification)   | GLUE, XNLI      | ✅ | ✅ | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://github.com/huggingface/notebooks/blob/master/examples/text_classification.ipynb)
+| [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER       | ✅ | ✅ | ✅ | -
 | [**`multiple-choice`**](https://github.com/huggingface/transformers/tree/master/examples/multiple-choice)           | SWAG, RACE, ARC | ✅ | ✅ | - | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ViktorAlm/notebooks/blob/master/MPC_GPU_Demo_for_TF_and_PT.ipynb)
 | [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering)     | SQuAD           | ✅ | ✅ | ✅ | -
-| [**`summarization`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq)                     | CNN/Daily Mail  | ✅  | - | - | -
-| [**`text-classification`**](https://github.com/huggingface/transformers/tree/master/examples/text-classification)   | GLUE, XNLI      | ✅ | ✅ | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://github.com/huggingface/notebooks/blob/master/examples/text_classification.ipynb)
 | [**`text-generation`**](https://github.com/huggingface/transformers/tree/master/examples/text-generation)           | -               | n/a | n/a | - | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)
-| [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER       | ✅ | ✅ | ✅ | -
+| [**`distillation`**](https://github.com/huggingface/transformers/tree/master/examples/distillation)                 | All             | - | -  | - | -
+| [**`summarization`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq)                     | CNN/Daily Mail  | ✅  | - | - | -
 | [**`translation`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq)                       | WMT             | ✅  | - | - | -
+| [**`bertology`**](https://github.com/huggingface/transformers/tree/master/examples/bertology)                       | -               | - | - | - | -
+| [**`adversarial`**](https://github.com/huggingface/transformers/tree/master/examples/adversarial)                   | HANS            | ✅ | - | - | -


-<!--
+<br>
+
 ## One-click Deploy to Cloud (wip)

 **Coming soon!**
-->

 ## Running on TPUs

@@ -1,20 +0,0 @@
-tensorboard
-scikit-learn
-seqeval
-psutil
-sacrebleu
-rouge-score
-tensorflow_datasets
-matplotlib
-git-python==1.0.3
-faiss-cpu
-streamlit
-elasticsearch
-nltk
-pandas
-datasets >= 1.1.3
-fire
-pytest
-conllu
-sentencepiece != 0.1.92
-protobuf
@@ -1,19 +1,3 @@
-<!---
-Copyright 2020 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-->
-
 # 🤗 Benchmark results

 Here, you can find a list of the different benchmark results created by the community.
@@ -1,17 +1,3 @@
-# Copyright 2020 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 import csv
 from collections import defaultdict
 from dataclasses import dataclass, field
@@ -1,17 +1,3 @@
-# Copyright 2020 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 # tests directory-specific settings - this file is run automatically
 # by pytest before any tests are run

@@ -0,0 +1,5 @@
+# Community contributed examples
+
+This folder contains examples which are not actively maintained (mostly contributed by the community).
+
+Using these examples together with a recent version of the library usually requires to make small (sometimes big) adaptations to get the scripts working.
@@ -1,7 +1,5 @@
 # Distil*

-Author: @VictorSanh
-
 This folder contains the original code used to train Distil* as well as examples showcasing how to use DistilBERT, DistilRoBERTa and DistilGPT2.

 **January 20, 2020 - Bug fixing** We have recently discovered and fixed [a bug](https://github.com/huggingface/transformers/commit/48cbf267c988b56c71a2380f748a3e6092ccaed3) in the evaluation of our `run_*.py` scripts that caused the reported metrics to be over-estimated on average. We have updated all the metrics with the latest runs.
@@ -1,19 +1,3 @@
-<!---
-Copyright 2020 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-->
-
 ## Language model training

 Fine-tuning (or training from scratch) the library models for language modeling on a text dataset for GPT, GPT-2,
@@ -1,3 +0,0 @@
-datasets >= 1.1.3
-sentencepiece != 0.1.92
-protobuf
@@ -1,636 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The HuggingFace Team All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Fine-tuning the library models for masked language modeling (BERT, ALBERT, RoBERTa...) with whole word masking on a
-text file or a dataset.
-
-Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
-https://huggingface.co/models?filter=masked-lm
-"""
-import logging
-import os
-import sys
-from dataclasses import dataclass, field
-
-# You can also adapt this script on your own masked language modeling task. Pointers for this are left as comments.
-from pathlib import Path
-from typing import Dict, List, Optional, Tuple
-
-import numpy as np
-from datasets import load_dataset
-from tqdm import tqdm
-
-import jax
-import jax.numpy as jnp
-from flax import jax_utils
-from flax.optim import Adam
-from flax.training import common_utils
-from flax.training.common_utils import get_metrics
-from jax.nn import log_softmax
-from transformers import (
-    CONFIG_MAPPING,
-    MODEL_FOR_MASKED_LM_MAPPING,
-    AutoConfig,
-    AutoTokenizer,
-    FlaxBertForMaskedLM,
-    HfArgumentParser,
-    PreTrainedTokenizerBase,
-    TensorType,
-    TrainingArguments,
-    is_tensorboard_available,
-    set_seed,
-)
-
-
-# Cache the result
-has_tensorboard = is_tensorboard_available()
-if has_tensorboard:
-    try:
-        from flax.metrics.tensorboard import SummaryWriter
-    except ImportError as ie:
-        has_tensorboard = False
-        print(f"Unable to display metrics through TensorBoard because some package are not installed: {ie}")
-
-else:
-    print(
-        "Unable to display metrics through TensorBoard because the package is not installed: "
-        "Please run pip install tensorboard to enable."
-    )
-
-
-MODEL_CONFIG_CLASSES = list(MODEL_FOR_MASKED_LM_MAPPING.keys())
-MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
-
-
-@dataclass
-class ModelArguments:
-    """
-    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
-    """
-
-    model_name_or_path: Optional[str] = field(
-        default=None,
-        metadata={
-            "help": "The model checkpoint for weights initialization."
-            "Don't set if you want to train a model from scratch."
-        },
-    )
-    model_type: Optional[str] = field(
-        default=None,
-        metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
-    )
-    config_name: Optional[str] = field(
-        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
-    )
-    tokenizer_name: Optional[str] = field(
-        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
-    )
-    cache_dir: Optional[str] = field(
-        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
-    )
-    use_fast_tokenizer: bool = field(
-        default=True,
-        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
-    )
-
-
-@dataclass
-class DataTrainingArguments:
-    """
-    Arguments pertaining to what data we are going to input our model for training and eval.
-    """
-
-    dataset_name: Optional[str] = field(
-        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
-    )
-    dataset_config_name: Optional[str] = field(
-        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
-    )
-    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
-    validation_file: Optional[str] = field(
-        default=None,
-        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
-    )
-    train_ref_file: Optional[str] = field(
-        default=None,
-        metadata={"help": "An optional input train ref data file for whole word masking in Chinese."},
-    )
-    validation_ref_file: Optional[str] = field(
-        default=None,
-        metadata={"help": "An optional input validation ref data file for whole word masking in Chinese."},
-    )
-    overwrite_cache: bool = field(
-        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
-    )
-    max_seq_length: Optional[int] = field(
-        default=None,
-        metadata={
-            "help": "The maximum total input sequence length after tokenization. Sequences longer "
-            "than this will be truncated. Default to the max input length of the model."
-        },
-    )
-    preprocessing_num_workers: Optional[int] = field(
-        default=None,
-        metadata={"help": "The number of processes to use for the preprocessing."},
-    )
-    mlm_probability: float = field(
-        default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
-    )
-    pad_to_max_length: bool = field(
-        default=False,
-        metadata={
-            "help": "Whether to pad all samples to `max_seq_length`. "
-            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
-        },
-    )
-
-    def __post_init__(self):
-        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
-            raise ValueError("Need either a dataset name or a training/validation file.")
-        else:
-            if self.train_file is not None:
-                extension = self.train_file.split(".")[-1]
-                assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
-            if self.validation_file is not None:
-                extension = self.validation_file.split(".")[-1]
-                assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."
-
-
-# Adapted from transformers/data/data_collator.py
-# Letting here for now, let's discuss where it should live
-@dataclass
-class FlaxDataCollatorForLanguageModeling:
-    """
-    Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
-    are not all of the same length.
-
-    Args:
-        tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
-            The tokenizer used for encoding the data.
-        mlm (:obj:`bool`, `optional`, defaults to :obj:`True`):
-            Whether or not to use masked language modeling. If set to :obj:`False`, the labels are the same as the
-            inputs with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for
-            non-masked tokens and the value to predict for the masked token.
-        mlm_probability (:obj:`float`, `optional`, defaults to 0.15):
-            The probability with which to (randomly) mask tokens in the input, when :obj:`mlm` is set to :obj:`True`.
-
-    .. note::
-
-        For best performance, this data collator should be used with a dataset having items that are dictionaries or
-        BatchEncoding, with the :obj:`"special_tokens_mask"` key, as returned by a
-        :class:`~transformers.PreTrainedTokenizer` or a :class:`~transformers.PreTrainedTokenizerFast` with the
-        argument :obj:`return_special_tokens_mask=True`.
-    """
-
-    tokenizer: PreTrainedTokenizerBase
-    mlm: bool = True
-    mlm_probability: float = 0.15
-
-    def __post_init__(self):
-        if self.mlm and self.tokenizer.mask_token is None:
-            raise ValueError(
-                "This tokenizer does not have a mask token which is necessary for masked language modeling. "
-                "You should pass `mlm=False` to train on causal language modeling instead."
-            )
-
-    def __call__(self, examples: List[Dict[str, np.ndarray]], pad_to_multiple_of: int) -> Dict[str, np.ndarray]:
-        # Handle dict or lists with proper padding and conversion to tensor.
-        batch = self.tokenizer.pad(examples, pad_to_multiple_of=pad_to_multiple_of, return_tensors=TensorType.NUMPY)
-
-        # If special token mask has been preprocessed, pop it from the dict.
-        special_tokens_mask = batch.pop("special_tokens_mask", None)
-        if self.mlm:
-            batch["input_ids"], batch["labels"] = self.mask_tokens(
-                batch["input_ids"], special_tokens_mask=special_tokens_mask
-            )
-        else:
-            labels = batch["input_ids"].copy()
-            if self.tokenizer.pad_token_id is not None:
-                labels[labels == self.tokenizer.pad_token_id] = -100
-            batch["labels"] = labels
-        return batch
-
-    def mask_tokens(
-        self, inputs: np.ndarray, special_tokens_mask: Optional[np.ndarray]
-    ) -> Tuple[jnp.ndarray, jnp.ndarray]:
-        """
-        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
-        """
-        labels = inputs.copy()
-        # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`)
-        probability_matrix = np.full(labels.shape, self.mlm_probability)
-        special_tokens_mask = special_tokens_mask.astype("bool")
-
-        probability_matrix[special_tokens_mask] = 0.0
-        masked_indices = np.random.binomial(1, probability_matrix).astype("bool")
-        labels[~masked_indices] = -100  # We only compute loss on masked tokens
-
-        # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
-        indices_replaced = np.random.binomial(1, np.full(labels.shape, 0.8)).astype("bool") & masked_indices
-        inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)
-
-        # 10% of the time, we replace masked input tokens with random word
-        indices_random = np.random.binomial(1, np.full(labels.shape, 0.5)).astype("bool")
-        indices_random &= masked_indices & ~indices_replaced
-
-        random_words = np.random.randint(self.tokenizer.vocab_size, size=labels.shape, dtype="i4")
-        inputs[indices_random] = random_words[indices_random]
-
-        # The rest of the time (10% of the time) we keep the masked input tokens unchanged
-        return inputs, labels
-
-
-def create_learning_rate_scheduler(
-    factors="constant * linear_warmup * rsqrt_decay",
-    base_learning_rate=0.5,
-    warmup_steps=1000,
-    decay_factor=0.5,
-    steps_per_decay=20000,
-    steps_per_cycle=100000,
-):
-    """Creates learning rate schedule.
-    Interprets factors in the factors string which can consist of:
-    * constant: interpreted as the constant value,
-    * linear_warmup: interpreted as linear warmup until warmup_steps,
-    * rsqrt_decay: divide by square root of max(step, warmup_steps)
-    * rsqrt_normalized_decay: divide by square root of max(step/warmup_steps, 1)
-    * decay_every: Every k steps decay the learning rate by decay_factor.
-    * cosine_decay: Cyclic cosine decay, uses steps_per_cycle parameter.
-    Args:
-      factors: string, factors separated by "*" that defines the schedule.
-      base_learning_rate: float, the starting constant for the lr schedule.
-      warmup_steps: int, how many steps to warm up for in the warmup schedule.
-      decay_factor: float, the amount to decay the learning rate by.
-      steps_per_decay: int, how often to decay the learning rate.
-      steps_per_cycle: int, steps per cycle when using cosine decay.
-    Returns:
-      a function learning_rate(step): float -> {"learning_rate": float}, the
-      step-dependent lr.
-    """
-    factors = [n.strip() for n in factors.split("*")]
-
-    def step_fn(step):
-        """Step to learning rate function."""
-        ret = 1.0
-        for name in factors:
-            if name == "constant":
-                ret *= base_learning_rate
-            elif name == "linear_warmup":
-                ret *= jnp.minimum(1.0, step / warmup_steps)
-            elif name == "rsqrt_decay":
-                ret /= jnp.sqrt(jnp.maximum(step, warmup_steps))
-            elif name == "rsqrt_normalized_decay":
-                ret *= jnp.sqrt(warmup_steps)
-                ret /= jnp.sqrt(jnp.maximum(step, warmup_steps))
-            elif name == "decay_every":
-                ret *= decay_factor ** (step // steps_per_decay)
-            elif name == "cosine_decay":
-                progress = jnp.maximum(0.0, (step - warmup_steps) / float(steps_per_cycle))
-                ret *= jnp.maximum(0.0, 0.5 * (1.0 + jnp.cos(jnp.pi * (progress % 1.0))))
-            else:
-                raise ValueError("Unknown factor %s." % name)
-        return jnp.asarray(ret, dtype=jnp.float32)
-
-    return step_fn
-
-
-def compute_metrics(logits, labels, weights, label_smoothing=0.0):
-    """Compute summary metrics."""
-    loss, normalizer = cross_entropy(logits, labels, weights, label_smoothing)
-    acc, _ = accuracy(logits, labels, weights)
-    metrics = {"loss": loss, "accuracy": acc, "normalizer": normalizer}
-    metrics = jax.lax.psum(metrics, axis_name="batch")
-    return metrics
-
-
-def accuracy(logits, targets, weights=None):
-    """Compute weighted accuracy for log probs and targets.
-    Args:
-     logits: [batch, length, num_classes] float array.
-     targets: categorical targets [batch, length] int array.
-     weights: None or array of shape [batch, length]
-    Returns:
-      Tuple of scalar loss and batch normalizing factor.
-    """
-    if logits.ndim != targets.ndim + 1:
-        raise ValueError(
-            "Incorrect shapes. Got shape %s logits and %s targets" % (str(logits.shape), str(targets.shape))
-        )
-
-    loss = jnp.equal(jnp.argmax(logits, axis=-1), targets)
-    loss *= weights
-
-    return loss.sum(), weights.sum()
-
-
-def cross_entropy(logits, targets, weights=None, label_smoothing=0.0):
-    """Compute cross entropy and entropy for log probs and targets.
-    Args:
-     logits: [batch, length, num_classes] float array.
-     targets: categorical targets [batch, length] int array.
-     weights: None or array of shape [batch, length]
-     label_smoothing: label smoothing constant, used to determine the on and off values.
-    Returns:
-      Tuple of scalar loss and batch normalizing factor.
-    """
-    if logits.ndim != targets.ndim + 1:
-        raise ValueError(
-            "Incorrect shapes. Got shape %s logits and %s targets" % (str(logits.shape), str(targets.shape))
-        )
-
-    vocab_size = logits.shape[-1]
-    confidence = 1.0 - label_smoothing
-    low_confidence = (1.0 - confidence) / (vocab_size - 1)
-    normalizing_constant = -(
-        confidence * jnp.log(confidence) + (vocab_size - 1) * low_confidence * jnp.log(low_confidence + 1e-20)
-    )
-    soft_targets = common_utils.onehot(targets, vocab_size, on_value=confidence, off_value=low_confidence)
-
-    loss = -jnp.sum(soft_targets * log_softmax(logits), axis=-1)
-    loss = loss - normalizing_constant
-
-    if weights is not None:
-        loss = loss * weights
-        normalizing_factor = weights.sum()
-    else:
-        normalizing_factor = np.prod(targets.shape)
-
-    return loss.sum(), normalizing_factor
-
-
-def training_step(optimizer, batch, dropout_rng):
-    dropout_rng, new_dropout_rng = jax.random.split(dropout_rng)
-
-    def loss_fn(params):
-        targets = batch.pop("labels")
-
-        # Hide away tokens which doesn't participate in the optimization
-        token_mask = jnp.where(targets > 0, 1.0, 0.0)
-
-        pooled, logits = model(**batch, params=params, dropout_rng=dropout_rng, train=True)
-        loss, weight_sum = cross_entropy(logits, targets, token_mask)
-        return loss / weight_sum
-
-    step = optimizer.state.step
-    lr = lr_scheduler_fn(step)
-    grad_fn = jax.value_and_grad(loss_fn)
-    loss, grad = grad_fn(optimizer.target)
-    grad = jax.lax.pmean(grad, "batch")
-    optimizer = optimizer.apply_gradient(grad, learning_rate=lr)
-
-    return loss, optimizer, new_dropout_rng
-
-
-def eval_step(params, batch):
-    """
-    Calculate evaluation metrics on a batch.
-    """
-    targets = batch.pop("labels")
-
-    # Hide away tokens which doesn't participate in the optimization
-    token_mask = jnp.where(targets > 0, 1.0, 0.0)
-    _, logits = model(**batch, params=params, train=False)
-
-    return compute_metrics(logits, targets, token_mask)
-
-
-def generate_batch_splits(samples_idx: jnp.ndarray, batch_size: int) -> jnp.ndarray:
-    nb_samples = len(samples_idx)
-    samples_to_remove = nb_samples % batch_size
-
-    if samples_to_remove != 0:
-        samples_idx = samples_idx[:-samples_to_remove]
-    sections_split = nb_samples // batch_size
-    batch_idx = jnp.split(samples_idx, sections_split)
-    return batch_idx
-
-
-if __name__ == "__main__":
-    # See all possible arguments in src/transformers/training_args.py
-    # or by passing the --help flag to this script.
-    # We now keep distinct sets of args, for a cleaner separation of concerns.
-
-    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
-    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
-        # If we pass only one argument to the script and it's the path to a json file,
-        # let's parse it to get our arguments.
-        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
-    else:
-        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
-
-    if (
-        os.path.exists(training_args.output_dir)
-        and os.listdir(training_args.output_dir)
-        and training_args.do_train
-        and not training_args.overwrite_output_dir
-    ):
-        raise ValueError(
-            f"Output directory ({training_args.output_dir}) already exists and is not empty."
-            "Use --overwrite_output_dir to overcome."
-        )
-
-    # Setup logging
-    logging.basicConfig(
-        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
-        level="NOTSET",
-        datefmt="[%X]",
-    )
-
-    # Log on each process the small summary:
-    logger = logging.getLogger(__name__)
-    logger.warning(
-        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
-        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
-    )
-
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    logger.info("Training/evaluation parameters %s", training_args)
-
-    # Set seed before initializing model.
-    set_seed(training_args.seed)
-
-    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
-    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
-    # (the dataset will be downloaded automatically from the datasets Hub).
-    #
-    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
-    # 'text' is found. You can easily tweak this behavior (see below).
-    #
-    # In distributed training, the load_dataset function guarantees that only one local process can concurrently
-    # download the dataset.
-    if data_args.dataset_name is not None:
-        # Downloading and loading a dataset from the hub.
-        datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
-    else:
-        data_files = {}
-        if data_args.train_file is not None:
-            data_files["train"] = data_args.train_file
-        if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
-        extension = data_args.train_file.split(".")[-1]
-        if extension == "txt":
-            extension = "text"
-        datasets = load_dataset(extension, data_files=data_files)
-    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
-    # https://huggingface.co/docs/datasets/loading_datasets.html.
-
-    # Load pretrained model and tokenizer
-
-    # Distributed training:
-    # The .from_pretrained methods guarantee that only one local process can concurrently
-    # download model & vocab.
-    if model_args.config_name:
-        config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir)
-    elif model_args.model_name_or_path:
-        config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
-    else:
-        config = CONFIG_MAPPING[model_args.model_type]()
-        logger.warning("You are instantiating a new config instance from scratch.")
-
-    if model_args.tokenizer_name:
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_args.tokenizer_name, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer
-        )
-    elif model_args.model_name_or_path:
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer
-        )
-    else:
-        raise ValueError(
-            "You are instantiating a new tokenizer from scratch. This is not supported by this script."
-            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
-        )
-
-    # Preprocessing the datasets.
-    # First we tokenize all the texts.
-    if training_args.do_train:
-        column_names = datasets["train"].column_names
-    else:
-        column_names = datasets["validation"].column_names
-    text_column_name = "text" if "text" in column_names else column_names[0]
-
-    padding = "max_length" if data_args.pad_to_max_length else False
-
-    def tokenize_function(examples):
-        # Remove empty lines
-        examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
-        return tokenizer(
-            examples["text"],
-            return_special_tokens_mask=True,
-            padding=padding,
-            truncation=True,
-            max_length=data_args.max_seq_length,
-        )
-
-    tokenized_datasets = datasets.map(
-        tokenize_function,
-        batched=True,
-        num_proc=data_args.preprocessing_num_workers,
-        remove_columns=[text_column_name],
-        load_from_cache_file=not data_args.overwrite_cache,
-    )
-
-    # Enable tensorboard only on the master node
-    if has_tensorboard and jax.host_id() == 0:
-        summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir).joinpath("logs").as_posix())
-
-    # Data collator
-    # This one will take care of randomly masking the tokens.
-    data_collator = FlaxDataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=data_args.mlm_probability)
-
-    # Initialize our training
-    rng = jax.random.PRNGKey(training_args.seed)
-    dropout_rngs = jax.random.split(rng, jax.local_device_count())
-
-    model = FlaxBertForMaskedLM.from_pretrained("bert-base-cased", dtype=jnp.float32, dropout_rate=0.1)
-    model.init(jax.random.PRNGKey(training_args.seed), (training_args.train_batch_size, model.config.max_length))
-
-    # Setup optimizer
-    optimizer = Adam(
-        learning_rate=training_args.learning_rate,
-        weight_decay=training_args.weight_decay,
-        beta1=training_args.adam_beta1,
-        beta2=training_args.adam_beta2,
-    ).create(model.params)
-
-    # Create learning rate scheduler
-    lr_scheduler_fn = create_learning_rate_scheduler(
-        base_learning_rate=training_args.learning_rate, warmup_steps=training_args.warmup_steps
-    )
-
-    # Create parallel version of the training and evaluation steps
-    p_training_step = jax.pmap(training_step, "batch", donate_argnums=(0,))
-    p_eval_step = jax.pmap(eval_step, "batch", donate_argnums=(0,))
-
-    # Replicate the optimizer on each device
-    optimizer = jax_utils.replicate(optimizer)
-
-    # Store some constant
-    nb_epochs = int(training_args.num_train_epochs)
-    batch_size = int(training_args.train_batch_size)
-    eval_batch_size = int(training_args.eval_batch_size)
-
-    epochs = tqdm(range(nb_epochs), desc=f"Epoch ... (1/{nb_epochs})", position=0)
-    for epoch in epochs:
-
-        # ======================== Training ================================
-        # Create sampling rng
-        rng, training_rng, eval_rng = jax.random.split(rng, 3)
-
-        # Generate an epoch by shuffling sampling indices from the train dataset
-        nb_training_samples = len(tokenized_datasets["train"])
-        training_samples_idx = jax.random.permutation(training_rng, jnp.arange(nb_training_samples))
-        training_batch_idx = generate_batch_splits(training_samples_idx, batch_size)
-
-        # Gather the indexes for creating the batch and do a training step
-        for batch_idx in tqdm(training_batch_idx, desc="Training...", position=1):
-            samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx]
-            model_inputs = data_collator(samples, pad_to_multiple_of=16)
-
-            # Model forward
-            model_inputs = common_utils.shard(model_inputs.data)
-            loss, optimizer, dropout_rngs = p_training_step(optimizer, model_inputs, dropout_rngs)
-
-        epochs.write(f"Loss: {loss}")
-
-        # ======================== Evaluating ==============================
-        nb_eval_samples = len(tokenized_datasets["test"])
-        eval_samples_idx = jnp.arange(nb_eval_samples)
-        eval_batch_idx = generate_batch_splits(eval_samples_idx, eval_batch_size)
-
-        eval_metrics = []
-        for i, batch_idx in enumerate(tqdm(eval_batch_idx, desc="Evaluating ...", position=2)):
-            samples = [tokenized_datasets["test"][int(idx)] for idx in batch_idx]
-            model_inputs = data_collator(samples, pad_to_multiple_of=16)
-
-            # Model forward
-            model_inputs = common_utils.shard(model_inputs.data)
-            metrics = p_eval_step(optimizer.target, model_inputs)
-            eval_metrics.append(metrics)
-
-        eval_metrics_np = get_metrics(eval_metrics)
-        eval_metrics_np = jax.tree_map(jnp.sum, eval_metrics_np)
-        eval_normalizer = eval_metrics_np.pop("normalizer")
-        eval_summary = jax.tree_map(lambda x: x / eval_normalizer, eval_metrics_np)
-
-        # Update progress bar
-        epochs.desc = (
-            f"Epoch... ({epoch + 1}/{nb_epochs} | Loss: {eval_summary['loss']}, Acc: {eval_summary['accuracy']})"
-        )
-
-        # Save metrics
-        if has_tensorboard and jax.host_id() == 0:
-            for name, value in eval_summary.items():
-                summary_writer.scalar(name, value, epoch)
@@ -281,7 +281,7 @@ def main():
    # Add the chinese references if provided
    if data_args.train_ref_file is not None:
        tokenized_datasets["train"] = add_chinese_references(tokenized_datasets["train"], data_args.train_ref_file)
-    if data_args.validation_ref_file is not None:
+    if data_args.valid_ref_file is not None:
        tokenized_datasets["validation"] = add_chinese_references(
            tokenized_datasets["validation"], data_args.validation_ref_file
        )
@@ -1,21 +0,0 @@
-<!---
-Copyright 2020 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-->
-
-# Legacy examples
-
-This folder contains examples which are not actively maintained (mostly contributed by the community).
-
-Using these examples together with a recent version of the library usually requires to make small (sometimes big) adaptations to get the scripts working.
@@ -1,229 +0,0 @@
-## Token classification
-
-Based on the scripts [`run_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/contrib/legacy/token-classification/run_ner.py).
-
-The following examples are covered in this section:
-
-* NER on the GermEval 2014 (German NER) dataset
-* Emerging and Rare Entities task: WNUT’17 (English NER) dataset
-
-Details and results for the fine-tuning provided by @stefan-it.
-
-### GermEval 2014 (German NER) dataset
-
-#### Data (Download and pre-processing steps)
-
-Data can be obtained from the [GermEval 2014](https://sites.google.com/site/germeval2014ner/data) shared task page.
-
-Here are the commands for downloading and pre-processing train, dev and test datasets. The original data format has four (tab-separated) columns, in a pre-processing step only the two relevant columns (token and outer span NER annotation) are extracted:
-
-```bash
-curl -L 'https://drive.google.com/uc?export=download&id=1Jjhbal535VVz2ap4v4r_rN1UEHTdLK5P' \
-| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > train.txt.tmp
-curl -L 'https://drive.google.com/uc?export=download&id=1ZfRcQThdtAR5PPRjIDtrVP7BtXSCUBbm' \
-| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > dev.txt.tmp
-curl -L 'https://drive.google.com/uc?export=download&id=1u9mb7kNJHWQCWyweMDRMuTFoOHOfeBTH' \
-| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > test.txt.tmp
-```
-
-The GermEval 2014 dataset contains some strange "control character" tokens like `'\x96', '\u200e', '\x95', '\xad' or '\x80'`.
-One problem with these tokens is, that `BertTokenizer` returns an empty token for them, resulting in misaligned `InputExample`s.
-The `preprocess.py` script located in the `scripts` folder a) filters these tokens and b) splits longer sentences into smaller ones (once the max. subtoken length is reached).
-
-Let's define some variables that we need for further pre-processing steps and training the model:
-
-```bash
-export MAX_LENGTH=128
-export BERT_MODEL=bert-base-multilingual-cased
-```
-
-Run the pre-processing script on training, dev and test datasets:
-
-```bash
-python3 scripts/preprocess.py train.txt.tmp $BERT_MODEL $MAX_LENGTH > train.txt
-python3 scripts/preprocess.py dev.txt.tmp $BERT_MODEL $MAX_LENGTH > dev.txt
-python3 scripts/preprocess.py test.txt.tmp $BERT_MODEL $MAX_LENGTH > test.txt
-```
-
-The GermEval 2014 dataset has much more labels than CoNLL-2002/2003 datasets, so an own set of labels must be used:
-
-```bash
-cat train.txt dev.txt test.txt | cut -d " " -f 2 | grep -v "^$"| sort | uniq > labels.txt
-```
-
-#### Prepare the run
-
-Additional environment variables must be set:
-
-```bash
-export OUTPUT_DIR=germeval-model
-export BATCH_SIZE=32
-export NUM_EPOCHS=3
-export SAVE_STEPS=750
-export SEED=1
-```
-
-#### Run the Pytorch version
-
-To start training, just run:
-
-```bash
-python3 run_ner.py --data_dir ./ \
--labels ./labels.txt \
--model_name_or_path $BERT_MODEL \
--output_dir $OUTPUT_DIR \
--max_seq_length  $MAX_LENGTH \
--num_train_epochs $NUM_EPOCHS \
--per_device_train_batch_size $BATCH_SIZE \
--save_steps $SAVE_STEPS \
--seed $SEED \
--do_train \
--do_eval \
--do_predict
-```
-
-If your GPU supports half-precision training, just add the `--fp16` flag. After training, the model will be both evaluated on development and test datasets.
-
-#### JSON-based configuration file
-
-Instead of passing all parameters via commandline arguments, the `run_ner.py` script also supports reading parameters from a json-based configuration file:
-
-```json
-{
-    "data_dir": ".",
-    "labels": "./labels.txt",
-    "model_name_or_path": "bert-base-multilingual-cased",
-    "output_dir": "germeval-model",
-    "max_seq_length": 128,
-    "num_train_epochs": 3,
-    "per_device_train_batch_size": 32,
-    "save_steps": 750,
-    "seed": 1,
-    "do_train": true,
-    "do_eval": true,
-    "do_predict": true
-}
-```
-
-It must be saved with a `.json` extension and can be used by running `python3 run_ner.py config.json`.
-
-#### Evaluation
-
-Evaluation on development dataset outputs the following for our example:
-
-```bash
-10/04/2019 00:42:06 - INFO - __main__ -   ***** Eval results  *****
-10/04/2019 00:42:06 - INFO - __main__ -     f1 = 0.8623348017621146
-10/04/2019 00:42:06 - INFO - __main__ -     loss = 0.07183869666975543
-10/04/2019 00:42:06 - INFO - __main__ -     precision = 0.8467916366258111
-10/04/2019 00:42:06 - INFO - __main__ -     recall = 0.8784592370979806
-```
-
-On the test dataset the following results could be achieved:
-
-```bash
-10/04/2019 00:42:42 - INFO - __main__ -   ***** Eval results  *****
-10/04/2019 00:42:42 - INFO - __main__ -     f1 = 0.8614389652384803
-10/04/2019 00:42:42 - INFO - __main__ -     loss = 0.07064602487454782
-10/04/2019 00:42:42 - INFO - __main__ -     precision = 0.8604651162790697
-10/04/2019 00:42:42 - INFO - __main__ -     recall = 0.8624150210424085
-```
-
-### Emerging and Rare Entities task: WNUT’17 (English NER) dataset
-
-Description of the WNUT’17 task from the [shared task website](http://noisy-text.github.io/2017/index.html):
-
-> The WNUT’17 shared task focuses on identifying unusual, previously-unseen entities in the context of emerging discussions.
-> Named entities form the basis of many modern approaches to other tasks (like event clustering and summarization), but recall on
-> them is a real problem in noisy text - even among annotators. This drop tends to be due to novel entities and surface forms.
-
-Six labels are available in the dataset. An overview can be found on this [page](http://noisy-text.github.io/2017/files/).
-
-#### Data (Download and pre-processing steps)
-
-The dataset can be downloaded from the [official GitHub](https://github.com/leondz/emerging_entities_17) repository.
-
-The following commands show how to prepare the dataset for fine-tuning:
-
-```bash
-mkdir -p data_wnut_17
-
-curl -L 'https://github.com/leondz/emerging_entities_17/raw/master/wnut17train.conll'  | tr '\t' ' ' > data_wnut_17/train.txt.tmp
-curl -L 'https://github.com/leondz/emerging_entities_17/raw/master/emerging.dev.conll' | tr '\t' ' ' > data_wnut_17/dev.txt.tmp
-curl -L 'https://raw.githubusercontent.com/leondz/emerging_entities_17/master/emerging.test.annotated' | tr '\t' ' ' > data_wnut_17/test.txt.tmp
-```
-
-Let's define some variables that we need for further pre-processing steps:
-
-```bash
-export MAX_LENGTH=128
-export BERT_MODEL=bert-large-cased
-```
-
-Here we use the English BERT large model for fine-tuning.
-The `preprocess.py` scripts splits longer sentences into smaller ones (once the max. subtoken length is reached):
-
-```bash
-python3 scripts/preprocess.py data_wnut_17/train.txt.tmp $BERT_MODEL $MAX_LENGTH > data_wnut_17/train.txt
-python3 scripts/preprocess.py data_wnut_17/dev.txt.tmp $BERT_MODEL $MAX_LENGTH > data_wnut_17/dev.txt
-python3 scripts/preprocess.py data_wnut_17/test.txt.tmp $BERT_MODEL $MAX_LENGTH > data_wnut_17/test.txt
-```
-
-In the last pre-processing step, the `labels.txt` file needs to be generated. This file contains all available labels:
-
-```bash
-cat data_wnut_17/train.txt data_wnut_17/dev.txt data_wnut_17/test.txt | cut -d " " -f 2 | grep -v "^$"| sort | uniq > data_wnut_17/labels.txt
-```
-
-#### Run the Pytorch version
-
-Fine-tuning with the PyTorch version can be started using the `run_ner.py` script. In this example we use a JSON-based configuration file.
-
-This configuration file looks like:
-
-```json
-{
-    "data_dir": "./data_wnut_17",
-    "labels": "./data_wnut_17/labels.txt",
-    "model_name_or_path": "bert-large-cased",
-    "output_dir": "wnut-17-model-1",
-    "max_seq_length": 128,
-    "num_train_epochs": 3,
-    "per_device_train_batch_size": 32,
-    "save_steps": 425,
-    "seed": 1,
-    "do_train": true,
-    "do_eval": true,
-    "do_predict": true,
-    "fp16": false
-}
-```
-
-If your GPU supports half-precision training, please set `fp16` to `true`.
-
-Save this JSON-based configuration under `wnut_17.json`. The fine-tuning can be started with `python3 run_ner_old.py wnut_17.json`.
-
-#### Evaluation
-
-Evaluation on development dataset outputs the following:
-
-```bash
-05/29/2020 23:33:44 - INFO - __main__ -   ***** Eval results *****
-05/29/2020 23:33:44 - INFO - __main__ -     eval_loss = 0.26505235286212275
-05/29/2020 23:33:44 - INFO - __main__ -     eval_precision = 0.7008264462809918
-05/29/2020 23:33:44 - INFO - __main__ -     eval_recall = 0.507177033492823
-05/29/2020 23:33:44 - INFO - __main__ -     eval_f1 = 0.5884802220680084
-05/29/2020 23:33:44 - INFO - __main__ -     epoch = 3.0
-```
-
-On the test dataset the following results could be achieved:
-
-```bash
-05/29/2020 23:33:44 - INFO - transformers.trainer -   ***** Running Prediction *****
-05/29/2020 23:34:02 - INFO - __main__ -     eval_loss = 0.30948806500973547
-05/29/2020 23:34:02 - INFO - __main__ -     eval_precision = 0.5840108401084011
-05/29/2020 23:34:02 - INFO - __main__ -     eval_recall = 0.3994439295644115
-05/29/2020 23:34:02 - INFO - __main__ -     eval_f1 = 0.47440836543753434
-```
-
-WNUT’17 is a very difficult task. Current state-of-the-art results on this dataset can be found [here](http://nlpprogress.com/english/named_entity_recognition.html).
@@ -1,7 +1,5 @@
 # Long Form Question Answering

-Author: @yjernite
-
 This folder contains the code for the Long Form Question answering [demo](http://35.226.96.115:8080/) as well as methods to train and use a fully end-to-end Long Form Question Answering system using the [🤗transformers](https://github.com/huggingface/transformers) and [🤗datasets](https://github.com/huggingface/datasets) libraries.

 You can use these methods to train your own system by following along the associate [notebook](https://github.com/huggingface/notebooks/blob/master/longform-qa/Long_Form_Question_Answering_with_ELI5_and_Wikipedia.ipynb) or [blog post](https://yjernite.github.io/lfqa.html).
@@ -47,7 +47,7 @@ nbclient==0.5.0
 nbconvert==6.0.1
 nbformat==5.0.7
 nest-asyncio==1.4.0
-notebook==6.1.5
+notebook==6.1.4
 numpy==1.19.2
 opencv-python==4.4.0.42
 packaging==20.3
@@ -90,7 +90,7 @@ torchvision==0.7.0
 tornado==6.0.4
 tqdm==4.48.2
 traitlets
-transformers==3.5.1
+git+https://github.com/huggingface/transformers.git
 urllib3==1.25.8
 wcwidth==0.2.5
 webencodings==0.5.1
@@ -1,7 +1,5 @@
 # Movement Pruning: Adaptive Sparsity by Fine-Tuning

-Author: @VictorSanh
-
 *Magnitude pruning is a widely used strategy for reducing model size in pure supervised learning; however, it is less effective in the transfer learning regime that has become standard for state-of-the-art natural language processing applications. We propose the use of *movement pruning*, a simple, deterministic first-order weight pruning method that is more adaptive to pretrained model fine-tuning. Experiments show that when pruning large pretrained language models, movement pruning shows significant improvements in high-sparsity regimes. When combined with distillation, the approach achieves minimal accuracy loss with down to only 3% of the model parameters:*

 | Fine-pruning+Distillation<br>(Teacher=BERT-base fine-tuned) | BERT base<br>fine-tuned | Remaining<br>Weights (%) | Magnitude Pruning      | L0 Regularization      | Movement Pruning       | Soft Movement Pruning          |
Author	SHA1	Message	Date
StillKeepTry	067756e3eb	fix positions ids bug	2020-12-09 11:26:10 +01:00
StillKeepTry	c5151bebfc	fix inputs handling in tf	2020-12-09 11:26:10 +01:00
KaiTao SongandJulien Plu	3c17ed80b1	Update src/transformers/models/mpnet/modeling_tf_mpnet.py Co-authored-by: Julien Plu <plu.julien@gmail.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandJulien Plu	f98dd9eb74	Update src/transformers/models/mpnet/modeling_tf_mpnet.py Co-authored-by: Julien Plu <plu.julien@gmail.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandJulien Plu	3b3722f941	Update src/transformers/models/mpnet/modeling_tf_mpnet.py Co-authored-by: Julien Plu <plu.julien@gmail.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandJulien Plu	6d6fced543	Update src/transformers/models/mpnet/modeling_tf_mpnet.py Co-authored-by: Julien Plu <plu.julien@gmail.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandJulien Plu	eb0131e846	Update src/transformers/models/mpnet/modeling_tf_mpnet.py Co-authored-by: Julien Plu <plu.julien@gmail.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandJulien Plu	6f30941d12	Update src/transformers/models/mpnet/modeling_tf_mpnet.py Co-authored-by: Julien Plu <plu.julien@gmail.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandSylvain Gugger	bbbaa6640b	Update src/transformers/models/mpnet/tokenization_mpnet.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandSylvain Gugger	b2c548410c	Update src/transformers/models/mpnet/tokenization_mpnet.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandSylvain Gugger	ae9f31ee32	Update src/transformers/models/mpnet/tokenization_mpnet.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandSylvain Gugger	47819d7adf	Update src/transformers/models/mpnet/tokenization_mpnet.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandSylvain Gugger	efd0f2d858	Update src/transformers/models/mpnet/tokenization_mpnet.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandSylvain Gugger	755bfe794b	Update src/transformers/models/mpnet/tokenization_mpnet.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandSylvain Gugger	26bcbb5818	Update src/transformers/models/mpnet/tokenization_mpnet.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>	2020-12-09 11:26:10 +01:00
KaiTao SongandSylvain Gugger	a61c215689	Update src/transformers/models/mpnet/tokenization_mpnet.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>	2020-12-09 11:26:10 +01:00
StillKeepTry	65ad40b13e	fix blank	2020-12-09 11:26:10 +01:00
StillKeepTry	c18f349089	fix tests	2020-12-09 11:26:10 +01:00
StillKeepTry	90affec3d4	fix copies	2020-12-09 11:26:10 +01:00
StillKeepTry	d5018f869b	update copied	2020-12-09 11:26:10 +01:00
StillKeepTry	3193ad63c9	fixed copied	2020-12-09 11:26:10 +01:00
StillKeepTry	0f491ba307	fix style	2020-12-09 11:26:10 +01:00
StillKeepTry	55f4a22aba	fix problems	2020-12-09 11:26:10 +01:00
StillKeepTry	4cf6551899	fix problems	2020-12-09 11:26:10 +01:00
StillKeepTry	b5605e0933	update tests	2020-12-09 11:26:10 +01:00
StillKeepTry	27fc6ecec7	fix problems	2020-12-09 11:26:10 +01:00
StillKeepTry	553989be95	fix format	2020-12-09 11:26:10 +01:00
StillKeepTry	bcd9c95faf	update mpnet	2020-12-09 11:26:10 +01:00