Release: v3.5.1

.circleci/config.yml

@@ -281,7 +281,7 @@ jobs:
                       - v0.4-build_doc-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install ."[all, docs]"
+            - run: pip install .[tf,torch,sentencepiece,docs]
             - save_cache:
                   key: v0.4-build_doc-{{ checksum "setup.py" }}
                   paths:
@@ -303,7 +303,7 @@ jobs:
                   keys:
                       - v0.4-deploy_doc-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
-            - run: pip install ."[all,docs]"
+            - run: pip install .[tf,torch,sentencepiece,docs]
             - save_cache:
                   key: v0.4-deploy_doc-{{ checksum "setup.py" }}
                   paths:

.circleci/deploy.sh

@@ -51,5 +51,4 @@ deploy_doc "7fb8bdf" v3.0.2
 deploy_doc "4b3ee9c" v3.1.0
 deploy_doc "3ebb1b3" v3.2.0
 deploy_doc "0613f05" v3.3.1
-deploy_doc "eb0e0ce" v3.4.0
-deploy_doc "818878d" # v3.5.1 Latest stable release
+deploy_doc "eb0e0ce" # v3.4.0 Latest stable release

.github/ISSUE_TEMPLATE/question-help.md

@@ -1,6 +1,6 @@
 ---
 name: "❓ Questions & Help"
-about: Post your general questions on the Hugging Face forum: https://discuss.huggingface.co/
+about: Post your general questions on the Hugging Face forum or Stack Overflow tagged huggingface-transformers
 title: ''
 labels: ''
 assignees: ''
@@ -10,17 +10,18 @@ assignees: ''
 # ❓ Questions & Help
 
 <!-- The GitHub issue tracker is primarly intended for bugs, feature requests,
-     new models, benchmarks, and migration questions. For all other questions,
+     new models and benchmarks, and migration questions. For all other questions,
      we direct you to the Hugging Face forum: https://discuss.huggingface.co/ .
+     You can also try Stack Overflow (SO) where a whole community of PyTorch and
+     Tensorflow enthusiast can help you out. In this case, make sure to tag your
+     question with the right deep learning framework as well as the
+     huggingface-transformers tag: 
+     https://stackoverflow.com/questions/tagged/huggingface-transformers 
      -->
 
 ## Details
-
 <!-- Description of your issue -->
 
-<!-- You should first ask your question on the forum, and only if
-     you didn't get an answer after a few days ask it here on GitHub. -->
-
-**A link to original question on the forum**:
-
-<!-- Your issue will be closed if you don't fill this part. -->
+<!-- You should first ask your question on the forum or SO, and only if
+     you didn't get an answer ask it here on GitHub. -->
+**A link to original question on the forum/Stack Overflow**:

.github/workflows/self-push.yml

@@ -4,12 +4,10 @@ on:
   push:
     branches:
       - master
-      - model-templates
     paths:
       - "src/**"
       - "tests/**"
       - ".github/**"
-      - "templates/**"
   # pull_request:
   repository_dispatch:
 
@@ -57,14 +55,6 @@ jobs:
           python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
           python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
 
-      - name: Create model files
-        run: |
-          source .env/bin/activate
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-
       - name: Run all non-slow tests on GPU
         env:
           OMP_NUM_THREADS: 1
@@ -126,14 +116,6 @@ jobs:
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
 
-      - name: Create model files
-        run: |
-          source .env/bin/activate
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-
       - name: Run all non-slow tests on GPU
         env:
           OMP_NUM_THREADS: 1
@@ -153,7 +135,7 @@ jobs:
           name: run_all_tests_tf_gpu_test_reports
           path: reports
 
-  run_tests_torch_multi_gpu:
+  run_tests_torch_multiple_gpu:
     runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
@@ -172,7 +154,7 @@ jobs:
         id: cache
         with:
           path: .env
-          key: v1.1-tests_torch_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v1.1-tests_torch_multiple_gpu-${{ hashFiles('setup.py') }}
 
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         run: |
@@ -199,11 +181,11 @@ jobs:
           OMP_NUM_THREADS: 1
         run: |
           source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_multi_gpu tests
+          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_multiple_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_multi_gpu_failures_short.txt          
+        run: cat reports/tests_torch_multiple_gpu_failures_short.txt          
 
       - name: Test suite reports artifacts
         if: ${{ always() }}
@@ -212,7 +194,7 @@ jobs:
           name: run_all_tests_torch_multi_gpu_test_reports
           path: reports
 
-  run_tests_tf_multi_gpu:
+  run_tests_tf_multiple_gpu:
     runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
@@ -231,7 +213,7 @@ jobs:
         id: cache
         with:
           path: .env
-          key: v1.1-tests_tf_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v1.1-tests_tf_multiple_gpu-${{ hashFiles('setup.py') }}
 
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         run: |
@@ -258,11 +240,11 @@ jobs:
           OMP_NUM_THREADS: 1
         run: |
           source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_multi_gpu tests
+          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_multiple_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_failures_short.txt
+        run: cat reports/tests_tf_multiple_gpu_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}

.github/workflows/self-scheduled.yml

@@ -187,7 +187,7 @@ jobs:
           name: run_all_tests_tf_gpu_test_reports
           path: reports
           
-  run_all_tests_torch_multi_gpu:
+  run_all_tests_torch_multiple_gpu:
     runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
@@ -238,11 +238,11 @@ jobs:
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_multiple_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_multi_gpu_failures_short.txt
+        run: cat reports/tests_torch_multiple_gpu_failures_short.txt
 
       - name: Run examples tests on multi-GPU
         env:
@@ -250,11 +250,11 @@ jobs:
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_multi_gpu examples
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_multiple_gpu examples
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/examples_torch_multi_gpu_failures_short.txt
+        run: cat reports/examples_torch_multiple_gpu_failures_short.txt
 
       - name: Run all pipeline tests on multi-GPU
         if: ${{ always() }}
@@ -265,11 +265,11 @@ jobs:
           RUN_PIPELINE_TESTS: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_multiple_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_pipeline_multi_gpu_failures_short.txt
+        run: cat reports/tests_torch_pipeline_multiple_gpu_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}
@@ -278,7 +278,7 @@ jobs:
           name: run_all_tests_torch_multi_gpu_test_reports
           path: reports
 
-  run_all_tests_tf_multi_gpu:
+  run_all_tests_tf_multiple_gpu:
     runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
@@ -329,11 +329,11 @@ jobs:
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_multiple_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_failures_short.txt
+        run: cat reports/tests_tf_multiple_gpu_failures_short.txt
 
       - name: Run all pipeline tests on multi-GPU
         if: ${{ always() }}
@@ -344,11 +344,11 @@ jobs:
           RUN_PIPELINE_TESTS: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multiple_gpu tests
           
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_pipelines_failures_short.txt
+        run: cat reports/tests_tf_multiple_gpu_pipelines_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}

CONTRIBUTING.md

@@ -308,12 +308,3 @@ Check our [documentation writing guide](https://github.com/huggingface/transform
 for more information.
 
 #### This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/master/CONTRIBUTING.md)
-
-
-### Develop on Windows
-
-One way one can run the make command on Window is to pass by MSYS2:
-
-1. [Download MSYS2](https://www.msys2.org/), we assume to have it installed in C:\msys64
-2. Open the command line C:\msys64\msys2.exe (it should be available from the start menu)
-3. Run in the shell: `pacman -Syu` and install make with `pacman -S make`

docs/source/_static/js/custom.js

@@ -1,11 +1,10 @@
 // These two things need to be updated at each release for the version selector.
 // Last stable version
-const stableVersion = "v3.5.0"
+const stableVersion = "v3.4.0"
 // Dictionary doc folder to label
 const versionMapping = {
     "master": "master",
-    "": "v3.5.0/v3.5.1",
-    "v3.4.0": "v3.4.0",
+    "": "v3.4.0",
     "v3.3.1": "v3.3.0/v3.3.1",
     "v3.2.0": "v3.2.0",
     "v3.1.0": "v3.1.0 (stable)",

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'3.5.0'
+release = u'3.5.1'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/main_classes/trainer.rst

@@ -35,7 +35,7 @@ Here is an example of how to customize :class:`~transformers.Trainer` using a cu
     class MyTrainer(Trainer):
         def compute_loss(self, model, inputs):
             labels = inputs.pop("labels")
-            outputs = model(**inputs)
+            outputs = models(**inputs)
             logits = outputs[0]
             return my_custom_loss(logits, labels)
 

docs/source/model_doc/bart.rst

@@ -34,8 +34,6 @@ ________________________________________________________________________________
 - An example of how to train :class:`~transformers.BartForConditionalGeneration` with a Hugging Face :obj:`datasets`
   object can be found in this `forum discussion
   <https://discuss.huggingface.co/t/train-bart-for-conditional-generation-e-g-summarization/1904>`__.
-- `Distilled checkpoints <https://huggingface.co/models?search=distilbart>`__ are described in this `paper
-  <https://arxiv.org/abs/2010.13002>`__.
 
 
 Implementation Notes
@@ -46,31 +44,14 @@ Implementation Notes
 - The forward pass of :class:`~transformers.BartModel` will create decoder inputs (using the helper function
   :func:`transformers.modeling_bart._prepare_bart_decoder_inputs`) if they are not passed. This is different than some
   other modeling APIs.
-- Model predictions are intended to be identical to the original implementation when
-  :obj:`force_bos_token_to_be_generated=True`. This only works, however, if the string you pass to
-  :func:`fairseq.encode` starts with a space.
+- Model predictions are intended to be identical to the original implementation. This only works, however, if the
+  string you pass to :func:`fairseq.encode` starts with a space.
 - :meth:`~transformers.BartForConditionalGeneration.generate` should be used for conditional generation tasks like
   summarization, see the example in that docstrings.
 - Models that load the `facebook/bart-large-cnn` weights will not have a :obj:`mask_token_id`, or be able to perform
   mask-filling tasks.
 - For training/forward passes that don't involve beam search, pass :obj:`use_cache=False`.
 
-Mask Filling
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The :obj:`facebook/bart-base` and :obj:`facebook/bart-large` checkpoints can be used to fill multi-token masks.
-
-.. code-block::
-
-    from transformers import BartForConditionalGeneration, BartTokenizer
-    model = BartForConditionalGeneration.from_pretrained("facebook/bart-large", force_bos_token_to_be_generated=True)
-    tok = BartTokenizer.from_pretrained("facebook/bart-large")
-    example_english_phrase = "UN Chief Says There Is No <mask> in Syria"
-    batch = tok(example_english_phrase, return_tensors='pt')
-    generated_ids = model.generate(batch['input_ids'])
-    assert tok.batch_decode(generated_ids, skip_special_tokens=True) == ['UN Chief Says There Is No Plan to Stop Chemical Weapons in Syria']
-
-
 
 BartConfig
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/source/model_doc/bert.rst

@@ -188,10 +188,3 @@ TFBertForQuestionAnswering
 
 .. autoclass:: transformers.TFBertForQuestionAnswering
     :members: call
-
-
-FlaxBertModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxBertModel
-    :members: __call__

docs/source/model_doc/dpr.rst

@@ -99,22 +99,3 @@ DPRReader
 
 .. autoclass:: transformers.DPRReader
     :members: forward
-
-TFDPRContextEncoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFDPRContextEncoder
-    :members: call
-
-TFDPRQuestionEncoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFDPRQuestionEncoder
-    :members: call
-
-
-TFDPRReader
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFDPRReader
-    :members: call

docs/source/model_doc/marian.rst

@@ -5,7 +5,7 @@ MarianMT
 <https://github.com/huggingface/transformers/issues/new?assignees=sshleifer&labels=&template=bug-report.md&title>`__
 and assign @patrickvonplaten.
 
-Translations should be similar, but not identical to output in the test set linked to in each model card.
+Translations should be similar, but not identical to, output in the test set linked to in each model card.
 
 Implementation Notes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -35,109 +35,24 @@ Naming
   <https://developers.google.com/admin-sdk/directory/v1/languages>`__, three digit codes require googling "language
   code {code}".
 - Codes formatted like :obj:`es_AR` are usually :obj:`code_{region}`. That one is Spanish from Argentina.
-- The models were converted in two stages. The first 1000 models use ISO-639-2 codes to identify languages, the second
-  group use a combination of ISO-639-5 codes and ISO-639-2 codes.
 
 
-Examples
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- Since Marian models are smaller than many other translation models available in the library, they can be useful for
-  fine-tuning experiments and integration tests.
-- `Fine-tune on TPU
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/builtin_trainer/train_distil_marian_enro_tpu.sh>`__
-- `Fine-tune on GPU
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/builtin_trainer/train_distil_marian_enro.sh>`__
-- `Fine-tune on GPU with pytorch-lightning
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/distil_marian_no_teacher.sh>`__
-
 Multilingual Models
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- All model names use the following format: :obj:`Helsinki-NLP/opus-mt-{src}-{tgt}`:
-- If a model can output multiple languages, and you should specify a language code by prepending the desired output
-  language to the :obj:`src_text`.
-- You can see a models's supported language codes in its model card, under target constituents, like in `opus-mt-en-roa
-  <https://huggingface.co/Helsinki-NLP/opus-mt-en-roa>`__.
-- Note that if a model is only multilingual on the source side, like :obj:`Helsinki-NLP/opus-mt-roa-en`, no language
-  codes are required.
+All model names use the following format: :obj:`Helsinki-NLP/opus-mt-{src}-{tgt}`:
 
-New multi-lingual models from the `Tatoeba-Challenge repo <https://github.com/Helsinki-NLP/Tatoeba-Challenge>`__
-require 3 character language codes:
+    - If :obj:`src` is in all caps, the model supports multiple input languages, you can figure out which ones by
+      looking at the model card, or the Group Members `mapping
+      <https://gist.github.com/sshleifer/6d20e7761931b08e73c3219027b97b8a>`_ .
+    - If :obj:`tgt` is in all caps, the model can output multiple languages, and you should specify a language code by
+      prepending the desired output language to the :obj:`src_text`.
+    - You can see a tokenizer's supported language codes in ``tokenizer.supported_language_codes``
+
+Example of translating english to many romance languages, using language codes:
 
 .. code-block:: python
 
-    from transformers import MarianMTModel, MarianTokenizer
-    src_text = [
-        '>>fra<< this is a sentence in english that we want to translate to french',
-        '>>por<< This should go to portuguese',
-        '>>esp<< And this to Spanish'
-    ]
-
-    model_name = 'Helsinki-NLP/opus-mt-en-roa'
-    tokenizer = MarianTokenizer.from_pretrained(model_name)
-    print(tokenizer.supported_language_codes)
-    model = MarianMTModel.from_pretrained(model_name)
-    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text))
-    tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
-    # ["c'est une phrase en anglais que nous voulons traduire en français",
-    # 'Isto deve ir para o português.',
-    # 'Y esto al español']
-
-
-
-
-Code to see available pretrained models:
-
-.. code-block:: python
-
-    from transformers.hf_api import HfApi
-    model_list = HfApi().model_list()
-    org = "Helsinki-NLP"
-    model_ids = [x.modelId for x in model_list if x.modelId.startswith(org)]
-    suffix = [x.split('/')[1] for x in model_ids]
-    old_style_multi_models = [f'{org}/{s}' for s in suffix if s != s.lower()]
-
-
-
-Old Style Multi-Lingual Models
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-These are the old style multi-lingual models ported from the OPUS-MT-Train repo: and the members of each language
-group:
-
-.. code-block:: python
-
-    ['Helsinki-NLP/opus-mt-NORTH_EU-NORTH_EU',
-     'Helsinki-NLP/opus-mt-ROMANCE-en',
-     'Helsinki-NLP/opus-mt-SCANDINAVIA-SCANDINAVIA',
-     'Helsinki-NLP/opus-mt-de-ZH',
-     'Helsinki-NLP/opus-mt-en-CELTIC',
-     'Helsinki-NLP/opus-mt-en-ROMANCE',
-     'Helsinki-NLP/opus-mt-es-NORWAY',
-     'Helsinki-NLP/opus-mt-fi-NORWAY',
-     'Helsinki-NLP/opus-mt-fi-ZH',
-     'Helsinki-NLP/opus-mt-fi_nb_no_nn_ru_sv_en-SAMI',
-     'Helsinki-NLP/opus-mt-sv-NORWAY',
-     'Helsinki-NLP/opus-mt-sv-ZH']
-    GROUP_MEMBERS = {
-     'ZH': ['cmn', 'cn', 'yue', 'ze_zh', 'zh_cn', 'zh_CN', 'zh_HK', 'zh_tw', 'zh_TW', 'zh_yue', 'zhs', 'zht', 'zh'],
-     'ROMANCE': ['fr', 'fr_BE', 'fr_CA', 'fr_FR', 'wa', 'frp', 'oc', 'ca', 'rm', 'lld', 'fur', 'lij', 'lmo', 'es', 'es_AR', 'es_CL', 'es_CO', 'es_CR', 'es_DO', 'es_EC', 'es_ES', 'es_GT', 'es_HN', 'es_MX', 'es_NI', 'es_PA', 'es_PE', 'es_PR', 'es_SV', 'es_UY', 'es_VE', 'pt', 'pt_br', 'pt_BR', 'pt_PT', 'gl', 'lad', 'an', 'mwl', 'it', 'it_IT', 'co', 'nap', 'scn', 'vec', 'sc', 'ro', 'la'],
-     'NORTH_EU': ['de', 'nl', 'fy', 'af', 'da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
-     'SCANDINAVIA': ['da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
-     'SAMI': ['se', 'sma', 'smj', 'smn', 'sms'],
-     'NORWAY': ['nb_NO', 'nb', 'nn_NO', 'nn', 'nog', 'no_nb', 'no'],
-     'CELTIC': ['ga', 'cy', 'br', 'gd', 'kw', 'gv']
-    }
-
-
-
-
-Example of translating english to many romance languages, using old-style 2 character language codes
-
-
-.. code-block::python
-
     from transformers import MarianMTModel, MarianTokenizer
     src_text = [
         '>>fr<< this is a sentence in english that we want to translate to french',
@@ -148,12 +63,52 @@ Example of translating english to many romance languages, using old-style 2 char
     model_name = 'Helsinki-NLP/opus-mt-en-ROMANCE'
     tokenizer = MarianTokenizer.from_pretrained(model_name)
     print(tokenizer.supported_language_codes)
-
     model = MarianMTModel.from_pretrained(model_name)
     translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text))
     tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
-    # ["c'est une phrase en anglais que nous voulons traduire en français", 'Isto deve ir para o português.',  'Y esto al español']
+    # ["c'est une phrase en anglais que nous voulons traduire en français",
+    # 'Isto deve ir para o português.',
+    # 'Y esto al español']
 
+Sometimes, models were trained on collections of languages that do not resolve to a group. In this case, _ is used as a
+separator for src or tgt, as in :obj:`Helsinki-NLP/opus-mt-en_el_es_fi-en_el_es_fi`. These still require language
+codes.
+
+There are many supported regional language codes, like :obj:`>>es_ES<<` (Spain) and :obj:`>>es_AR<<` (Argentina), that
+do not seem to change translations. I have not found these to provide different results than just using :obj:`>>es<<`.
+
+For example:
+
+    - `Helsinki-NLP/opus-mt-NORTH_EU-NORTH_EU`: translates from all NORTH_EU languages (see `mapping
+      <https://gist.github.com/sshleifer/6d20e7761931b08e73c3219027b97b8a>`_) to all NORTH_EU languages. Use a special
+      language code like :obj:`>>de<<` to specify output language.
+    - `Helsinki-NLP/opus-mt-ROMANCE-en`: translates from many romance languages to english, no codes needed since there
+      is only one target language.
+
+
+
+.. code-block:: python
+
+    GROUP_MEMBERS = {
+     'ZH': ['cmn', 'cn', 'yue', 'ze_zh', 'zh_cn', 'zh_CN', 'zh_HK', 'zh_tw', 'zh_TW', 'zh_yue', 'zhs', 'zht', 'zh'],
+     'ROMANCE': ['fr', 'fr_BE', 'fr_CA', 'fr_FR', 'wa', 'frp', 'oc', 'ca', 'rm', 'lld', 'fur', 'lij', 'lmo', 'es', 'es_AR', 'es_CL', 'es_CO', 'es_CR', 'es_DO', 'es_EC', 'es_ES', 'es_GT', 'es_HN', 'es_MX', 'es_NI', 'es_PA', 'es_PE', 'es_PR', 'es_SV', 'es_UY', 'es_VE', 'pt', 'pt_br', 'pt_BR', 'pt_PT', 'gl', 'lad', 'an', 'mwl', 'it', 'it_IT', 'co', 'nap', 'scn', 'vec', 'sc', 'ro', 'la'],
+     'NORTH_EU': ['de', 'nl', 'fy', 'af', 'da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
+     'SCANDINAVIA': ['da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
+     'SAMI': ['se', 'sma', 'smj', 'smn', 'sms'],
+     'NORWAY': ['nb_NO', 'nb', 'nn_NO', 'nn', 'nog', 'no_nb', 'no'],
+     'CELTIC': ['ga', 'cy', 'br', 'gd', 'kw', 'gv']
+    }
+
+Code to see available pretrained models:
+
+.. code-block:: python
+
+    from transformers.hf_api import HfApi
+    model_list = HfApi().model_list()
+    org = "Helsinki-NLP"
+    model_ids = [x.modelId for x in model_list if x.modelId.startswith(org)]
+    suffix = [x.split('/')[1] for x in model_ids]
+    multi_models = [f'{org}/{s}' for s in suffix if s != s.lower()]
 
 
 MarianConfig

docs/source/model_doc/mbart.rst

@@ -19,13 +19,6 @@ on the encoder, decoder, or reconstructing parts of the text.
 
 The Authors' code can be found `here <https://github.com/pytorch/fairseq/tree/master/examples/mbart>`__
 
-Examples
-_______________________________________________________________________________________________________________________
-
-- Examples and scripts for fine-tuning mBART and other models for sequence to sequence tasks can be found in
-  `examples/seq2seq/ <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
-- Given the large embeddings table, mBART consumes a large amount of GPU RAM, especially for fine-tuning.
-  :class:`MarianMTModel` is usually a better choice for bilingual machine translation.
 
 Training
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -45,7 +38,11 @@ the sequences for sequence-to-sequence fine-tuning.
     example_english_phrase = "UN Chief Says There Is No Military Solution in Syria"
     expected_translation_romanian = "Şeful ONU declară că nu există o soluţie militară în Siria"
     batch = tokenizer.prepare_seq2seq_batch(example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian)
-    model(input_ids=batch['input_ids'], labels=batch['labels']) # forward pass
+    input_ids = batch["input_ids"]
+    target_ids = batch["decoder_input_ids"]
+    decoder_input_ids = target_ids[:, :-1].contiguous()
+    labels = target_ids[:, 1:].clone()
+    model(input_ids=input_ids, decoder_input_ids=decoder_input_ids, labels=labels) #forward
 
 - Generation
 

docs/source/model_doc/pegasus.rst

@@ -31,19 +31,10 @@ All the `checkpoints <https://huggingface.co/models?search=pegasus>`__ are fine-
 - Each checkpoint is 2.2 GB on disk and 568M parameters.
 - FP16 is not supported (help/ideas on this appreciated!).
 - Summarizing xsum in fp32 takes about 400ms/sample, with default parameters on a v100 GPU.
-- Full replication results and correctly pre-processed data can be found in this `Issue
-  <https://github.com/huggingface/transformers/issues/6844#issue-689259666>`__.
-- `Distilled checkpoints <https://huggingface.co/models?search=distill-pegasus>`__ are described in this `paper
-  <https://arxiv.org/abs/2010.13002>`__.
+- For XSUM, The paper reports rouge1,rouge2, rougeL of paper: 47.21/24.56/39.25. As of Aug 9, this port scores
+  46.91/24.34/39.1.
 
-Examples
-_______________________________________________________________________________________________________________________
-
-- `Script <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/finetune_pegasus_xsum.sh>`__ to
-  fine-tune pegasus on the XSUM dataset. Data download instructions at `examples/seq2seq/
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
-- FP16 is not supported (help/ideas on this appreciated!).
-- The adafactor optimizer is recommended for pegasus fine-tuning.
+The gap is likely because of different alpha/length_penalty implementations in beam search.
 
 
 Implementation Notes
@@ -54,7 +45,7 @@ Implementation Notes
 - Some key configuration differences:
 
     - static, sinusoidal position embeddings
-    - no :obj:`layernorm_embedding` (:obj:`PegasusConfig.normalize_embedding=False`)
+    - no :obj:`layernorm_embedding` (:obj`PegasusConfig.normalize_embedding=False`)
     - the model starts generating with pad_token_id (which has 0 token_embedding) as the prefix.
     - more beams are used (:obj:`num_beams=8`)
 - All pretrained pegasus checkpoints are the same besides three attributes: :obj:`tokenizer.model_max_length` (maximum

docs/source/model_doc/roberta.rst

@@ -146,10 +146,3 @@ TFRobertaForQuestionAnswering
 
 .. autoclass:: transformers.TFRobertaForQuestionAnswering
     :members: call
-
-
-FlaxRobertaModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxRobertaModel
-    :members: __call__

docs/source/model_sharing.rst

@@ -37,7 +37,7 @@ For instance:
 
 .. code-block::
 
-    >>> model = AutoModel.from_pretrained(
+    >>> tokenizer = AutoTokenizer.from_pretrained(
     >>>   "julien-c/EsperBERTo-small",
     >>>   revision="v2.0.1" # tag name, or branch name, or commit hash
     >>> )
@@ -46,42 +46,31 @@ Basic steps
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 In order to upload a model, you'll need to first create a git repo. This repo will live on the model hub, allowing
-users to clone it and you (and your organization members) to push to it.
+users to clone it and you (and your organization members) to push to it. First, you should ensure you are logged in the
+``transformers-cli``:
 
-You can create a model repo directly from the website, `here <https://huggingface.co/new>`.
-
-Alternatively, you can use the ``transformers-cli``. The next steps describe that process:
-
-Go to a terminal and run the following command. It should be in the virtual environment where you installed 🤗
+Go in a terminal and run the following command. It should be in the virtual environment where you installed 🤗
 Transformers, since that command :obj:`transformers-cli` comes from the library.
 
-.. code-block:: bash
+.. code-block::
 
     transformers-cli login
 
 
 Once you are logged in with your model hub credentials, you can start building your repositories. To create a repo:
 
-.. code-block:: bash
+.. code-block::
 
     transformers-cli repo create your-model-name
 
-This creates a repo on the model hub, which can be cloned.
+This creates a repo on the model hub, which can be cloned. You can then add/remove from that repo as you would with any
+other git repo.
 
-.. code-block:: bash
+.. code-block::
 
     git clone https://huggingface.co/username/your-model-name
 
-    # Make sure you have git-lfs installed
-    # (https://git-lfs.github.com/)
-    git lfs install
-
-When you have your local clone of your repo and lfs installed, you can then add/remove from that clone as you would
-with any other git repo.
-
-.. code-block:: bash
-
-    # Commit as usual
+    # Then commit as usual
     cd your-model-name
     echo "hello" >> README.md
     git add . && git commit -m "Update from $USER"
@@ -170,25 +159,24 @@ Or, if you're using the Trainer API
 .. code-block::
 
     >>> trainer.save_model("path/to/awesome-name-you-picked")
-    >>> tokenizer.save_pretrained("path/to/repo/clone/your-model-name")
 
 You can then add these files to the staging environment and verify that they have been correctly staged with the ``git
 status`` command:
 
-.. code-block:: bash
+.. code-block::
 
     git add --all
     git status
 
 Finally, the files should be comitted:
 
-.. code-block:: bash
+.. code-block::
 
     git commit -m "First version of the your-model-name model and tokenizer."
 
 And pushed to the remote:
 
-.. code-block:: bash
+.. code-block::
 
     git push
 
@@ -211,7 +199,7 @@ don't forget to link to its model card so that people can fully trace how your m
 If you have never made a pull request to the 🤗 Transformers repo, look at the :doc:`contributing guide <contributing>`
 to see the steps to follow.
 
-.. note::
+.. Note::
 
     You can also send your model card in the folder you uploaded with the CLI by placing it in a `README.md` file
     inside `path/to/awesome-name-you-picked/`.
@@ -237,47 +225,3 @@ You may specify a revision by using the ``revision`` flag in the ``from_pretrain
     >>>   "julien-c/EsperBERTo-small",
     >>>   revision="v2.0.1" # tag name, or branch name, or commit hash
     >>> )
-
-Workflow in a Colab notebook
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you're in a Colab notebook (or similar) with no direct access to a terminal, here is the workflow you can use to
-upload your model. You can execute each one of them in a cell by adding a ! at the beginning.
-
-First you need to install `git-lfs` in the environment used by the notebook:
-
-.. code-block:: bash
-
-    sudo apt-get install git-lfs
-
-Then you can use the :obj:`transformers-cli` to create your new repo:
-
-
-.. code-block:: bash
-
-    transformers-cli login
-    transformers-cli repo create your-model-name
-
-Once it's created, you can clone it and configure it (replace username by your username on huggingface.co):
-
-.. code-block:: bash
-
-    git clone https://username:password@huggingface.co/username/your-model-name
-    # Alternatively if you have a token,
-    # you can use it instead of your password
-    git clone https://username:token@huggingface.co/username/your-model-name
-
-    cd your-model-name
-    git lfs install
-    git config --global user.email "email@example.com"
-    # Tip: using the same email than for your huggingface.co account will link your commits to your profile
-    git config --global user.name "Your name"
-
-Once you've saved your model inside, and your clone is setup with the right remote URL, you can add it and push it with
-usual git commands.
-
-.. code-block:: bash
-
-    git add .
-    git commit -m "Initial commit"
-    git push

docs/source/task_summary.rst

@@ -513,7 +513,7 @@ Here, the model generates a random text with a total maximal length of *50* toke
 concerned, I will"*. The default arguments of ``PreTrainedModel.generate()`` can be directly overridden in the
 pipeline, as is shown above for the argument ``max_length``.
 
-Here is an example of text generation using ``XLNet`` and its tokenizer.
+Here is an example of text generation using ``XLNet`` and its tokenzier.
 
 .. code-block::
 
@@ -834,7 +834,7 @@ Here is an example of doing translation using a model and a tokenizer. The proce
 
 1. Instantiate a tokenizer and a model from the checkpoint name. Summarization is usually done using an encoder-decoder
    model, such as ``Bart`` or ``T5``.
-2. Define the article that should be summarized.
+2. Define the article that should be summarizaed.
 3. Add the T5 specific prefix "translate English to German: "
 4. Use the ``PreTrainedModel.generate()`` method to perform the translation.
 

docs/source/testing.rst

@@ -405,32 +405,32 @@ decorators are used to set the requirements of tests CPU/GPU/TPU-wise:
 
 * ``require_torch`` - this test will run only under torch
 * ``require_torch_gpu`` - as ``require_torch`` plus requires at least 1 GPU
-* ``require_torch_multi_gpu`` - as ``require_torch`` plus requires at least 2 GPUs
-* ``require_torch_non_multi_gpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
+* ``require_torch_multigpu`` - as ``require_torch`` plus requires at least 2 GPUs
+* ``require_torch_non_multigpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
 * ``require_torch_tpu`` - as ``require_torch`` plus requires at least 1 TPU
 
 Let's depict the GPU requirements in the following table:
 
 
-+----------+----------------------------------+
-| n gpus   |  decorator                       |
-+==========+==================================+
-| ``>= 0`` | ``@require_torch``               |
-+----------+----------------------------------+
-| ``>= 1`` | ``@require_torch_gpu``           |
-+----------+----------------------------------+
-| ``>= 2`` | ``@require_torch_multi_gpu``     |
-+----------+----------------------------------+
-| ``< 2``  | ``@require_torch_non_multi_gpu`` |
-+----------+----------------------------------+
++----------+---------------------------------+
+| n gpus   |  decorator                      |
++==========+=================================+
+| ``>= 0`` | ``@require_torch``              |
++----------+---------------------------------+
+| ``>= 1`` | ``@require_torch_gpu``          |
++----------+---------------------------------+
+| ``>= 2`` | ``@require_torch_multigpu``     |
++----------+---------------------------------+
+| ``< 2``  | ``@require_torch_non_multigpu`` |
++----------+---------------------------------+
 
 
 For example, here is a test that must be run only when there are 2 or more GPUs available and pytorch is installed:
 
 .. code-block:: python
 
-    @require_torch_multi_gpu
-    def test_example_with_multi_gpu():
+    @require_torch_multigpu
+    def test_example_with_multigpu():
 
 If a test requires ``tensorflow`` use the ``require_tf`` decorator. For example:
 
@@ -454,7 +454,7 @@ last for them to work correctly. Here is an example of the correct usage:
 .. code-block:: python
 
     @parameterized.expand(...)
-    @require_torch_multi_gpu
+    @require_torch_multigpu
     def test_integration_foo():
 
 This order problem doesn't exist with ``@pytest.mark.parametrize``, you can put it first or last and it will still
@@ -716,11 +716,11 @@ Temporary files and directories
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Using unique temporary files and directories are essential for parallel test running, so that the tests won't overwrite
-each other's data. Also we want to get the temporary files and directories removed at the end of each test that created
+each other's data. Also we want to get the temp files and directories removed at the end of each test that created
 them. Therefore, using packages like ``tempfile``, which address these needs is essential.
 
-However, when debugging tests, you need to be able to see what goes into the temporary file or directory and you want
-to know it's exact path and not having it randomized on every test re-run.
+However, when debugging tests, you need to be able to see what goes into the temp file or directory and you want to
+know it's exact path and not having it randomized on every test re-run.
 
 A helper class :obj:`transformers.test_utils.TestCasePlus` is best used for such purposes. It's a sub-class of
 :obj:`unittest.TestCase`, so we can easily inherit from it in the test modules.
@@ -736,33 +736,32 @@ Here is an example of its usage:
 
 This code creates a unique temporary directory, and sets :obj:`tmp_dir` to its location.
 
-* Create a unique temporary dir:
+In this and all the following scenarios the temporary directory will be auto-removed at the end of test, unless
+``after=False`` is passed to the helper function.
+
+* Create a temporary directory of my choice and delete it at the end - useful for debugging when you want to monitor a
+  specific directory:
 
 .. code-block:: python
 
     def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir()
+        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test")
 
-``tmp_dir`` will contain the path to the created temporary dir. It will be automatically removed at the end of the
-test.
-
-* Create a temporary dir of my choice, ensure it's empty before the test starts and don't empty it after the test.
+* Create a temporary directory of my choice and do not delete it at the end---useful for when you want to look at the
+  temp results:
 
 .. code-block:: python
 
     def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir("./xxx")
+        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", after=False)
 
-This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn't
-leave any data in there.
+* Create a temporary directory of my choice and ensure to delete it right away---useful for when you disabled deletion
+  in the previous test run and want to make sure the that temporary directory is empty before the new test is run:
 
-* You can override the default behavior by directly overriding the ``before`` and ``after`` args, leading to one of the
-  following behaviors:
+.. code-block:: python
 
-    - ``before=True``: the temporary dir will always be cleared at the beginning of the test.
-    - ``before=False``: if the temporary dir already existed, any existing files will remain there.
-    - ``after=True``: the temporary dir will always be deleted at the end of the test.
-    - ``after=False``: the temporary dir will always be left intact at the end of the test.
+   def test_whatever(self):
+        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", before=True)
 
 .. note::
    In order to run the equivalent of ``rm -r`` safely, only subdirs of the project repository checkout are allowed if
@@ -816,7 +815,7 @@ or the ``xfail`` way:
     @pytest.mark.xfail
     def test_feature_x():
 
-- Here is how to skip a test based on some internal check inside the test:
+Here is how to skip a test based on some internal check inside the test:
 
 .. code-block:: python
 
@@ -839,7 +838,7 @@ or the ``xfail`` way:
     def test_feature_x():
         pytest.xfail("expected to fail until bug XYZ is fixed")
 
-- Here is how to skip all tests in a module if some import is missing:
+Here is how to skip all tests in a module if some import is missing:
 
 .. code-block:: python
 

examples/adversarial/run_hans.py

@@ -23,7 +23,6 @@ from typing import Dict, List, Optional
 import numpy as np
 import torch
 
-import transformers
 from transformers import (
     AutoConfig,
     AutoModelForSequenceClassification,
@@ -34,7 +33,6 @@ from transformers import (
     default_data_collator,
     set_seed,
 )
-from transformers.trainer_utils import is_main_process
 from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels
 
 
@@ -126,11 +124,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/bert-loses-patience/run_glue_with_pabee.py

@@ -29,7 +29,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from pabee.modeling_pabee_albert import AlbertForSequenceClassificationWithPabee
 from pabee.modeling_pabee_bert import BertForSequenceClassificationWithPabee
 from transformers import (
@@ -45,7 +44,6 @@ from transformers import glue_compute_metrics as compute_metrics
 from transformers import glue_convert_examples_to_features as convert_examples_to_features
 from transformers import glue_output_modes as output_modes
 from transformers import glue_processors as processors
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -632,11 +630,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 

examples/bert-loses-patience/test_run_glue_with_pabee.py

@@ -4,7 +4,7 @@ import sys
 from unittest.mock import patch
 
 import run_glue_with_pabee
-from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me
+from transformers.testing_utils import TestCasePlus, require_torch_non_multigpu_but_fix_me
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -20,7 +20,7 @@ def get_setup_file():
 
 
 class PabeeTests(TestCasePlus):
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)

examples/bertology/run_bertology.py

@@ -30,7 +30,6 @@ from torch.utils.data import DataLoader, SequentialSampler, Subset
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm
 
-import transformers
 from transformers import (
     AutoConfig,
     AutoModelForSequenceClassification,
@@ -42,7 +41,6 @@ from transformers import (
     glue_processors,
     set_seed,
 )
-from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -370,11 +368,6 @@ def main():
     # Setup logging
     logging.basicConfig(level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
     logger.info("device: {} n_gpu: {}, distributed: {}".format(args.device, args.n_gpu, bool(args.local_rank != -1)))
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
 
     # Set seeds
     set_seed(args.seed)

examples/contrib/legacy/run_language_modeling.py

@@ -29,7 +29,6 @@ from typing import Optional
 
 from torch.utils.data import ConcatDataset
 
-import transformers
 from transformers import (
     CONFIG_MAPPING,
     MODEL_WITH_LM_HEAD_MAPPING,
@@ -48,7 +47,6 @@ from transformers import (
     TrainingArguments,
     set_seed,
 )
-from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -221,11 +219,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/contrib/mm-imdb/run_mmimdb.py

@@ -31,7 +31,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from transformers import (
     WEIGHTS_NAME,
     AdamW,
@@ -42,7 +41,6 @@ from transformers import (
     MMBTForClassification,
     get_linear_schedule_with_warmup,
 )
-from transformers.trainer_utils import is_main_process
 from utils_mmimdb import ImageEncoder, JsonlDataset, collate_fn, get_image_transforms, get_mmimdb_labels
 
 
@@ -478,11 +476,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 

examples/contrib/run_swag.py

@@ -31,10 +31,8 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from transformers import WEIGHTS_NAME, AdamW, AutoConfig, AutoTokenizer, get_linear_schedule_with_warmup
 from transformers.modeling_auto import AutoModelForMultipleChoice
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -622,11 +620,6 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
 
     # Set seed
     set_seed(args)

examples/deebert/run_glue_deebert.py

@@ -13,7 +13,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from src.modeling_highway_bert import DeeBertForSequenceClassification
 from src.modeling_highway_roberta import DeeRobertaForSequenceClassification
 from transformers import (
@@ -29,7 +28,6 @@ from transformers import glue_compute_metrics as compute_metrics
 from transformers import glue_convert_examples_to_features as convert_examples_to_features
 from transformers import glue_output_modes as output_modes
 from transformers import glue_processors as processors
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -582,11 +580,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 

examples/deebert/test_glue_deebert.py

@@ -5,7 +5,7 @@ import unittest
 from unittest.mock import patch
 
 import run_glue_deebert
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow
+from transformers.testing_utils import require_torch_non_multigpu_but_fix_me, slow
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -26,7 +26,7 @@ class DeeBertTests(unittest.TestCase):
         logger.addHandler(stream_handler)
 
     @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_glue_deebert_train(self):
 
         train_args = """

examples/distillation/run_squad_w_distillation.py

@@ -30,7 +30,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from transformers import (
     WEIGHTS_NAME,
     AdamW,
@@ -58,7 +57,6 @@ from transformers.data.metrics.squad_metrics import (
     squad_evaluate,
 )
 from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -747,11 +745,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 

examples/language-modeling/run_clm.py

@@ -168,8 +168,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -256,7 +254,7 @@ def main():
         tokenize_function,
         batched=True,
         num_proc=data_args.preprocessing_num_workers,
-        remove_columns=column_names,
+        remove_columns=[text_column_name],
         load_from_cache_file=not data_args.overwrite_cache,
     )
 

examples/language-modeling/run_mlm.py

@@ -179,8 +179,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -294,7 +292,7 @@ def main():
             tokenize_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
-            remove_columns=column_names,
+            remove_columns=[text_column_name],
             load_from_cache_file=not data_args.overwrite_cache,
         )
 

examples/language-modeling/run_mlm_wwm.py

@@ -186,8 +186,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.

examples/language-modeling/run_plm.py

@@ -176,8 +176,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -281,7 +279,7 @@ def main():
             tokenize_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
-            remove_columns=column_names,
+            remove_columns=[text_column_name],
             load_from_cache_file=not data_args.overwrite_cache,
         )
 

examples/lightning_base.py

@@ -4,7 +4,6 @@ import os
 from pathlib import Path
 from typing import Any, Dict
 
-import packaging
 import pytorch_lightning as pl
 from pytorch_lightning.utilities import rank_zero_info
 
@@ -34,17 +33,15 @@ from transformers.optimization import (
 
 logger = logging.getLogger(__name__)
 
+try:
+    pkg = "pytorch_lightning"
+    min_ver = "1.0.4"
+    pkg_resources.require(f"{pkg}>={min_ver}")
+except pkg_resources.VersionConflict:
+    logger.warning(
+        f"{pkg}>={min_ver} is required for a normal functioning of this module, but found {pkg}=={pkg_resources.get_distribution(pkg).version}. Try pip install -r examples/requirements.txt"
+    )
 
-def require_min_ver(pkg, min_ver):
-    got_ver = pkg_resources.get_distribution(pkg).version
-    if packaging.version.parse(got_ver) < packaging.version.parse(min_ver):
-        logger.warning(
-            f"{pkg}>={min_ver} is required for a normal functioning of this module, but found {pkg}=={got_ver}. "
-            "Try: pip install -r examples/requirements.txt"
-        )
-
-
-require_min_ver("pytorch_lightning", "1.0.4")
 
 MODEL_MODES = {
     "base": AutoModel,

examples/multiple-choice/run_multiple_choice.py

@@ -23,7 +23,6 @@ from typing import Dict, Optional
 
 import numpy as np
 
-import transformers
 from transformers import (
     AutoConfig,
     AutoModelForMultipleChoice,
@@ -34,7 +33,6 @@ from transformers import (
     TrainingArguments,
     set_seed,
 )
-from transformers.trainer_utils import is_main_process
 from utils_multiple_choice import MultipleChoiceDataset, Split, processors
 
 
@@ -117,11 +115,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/multiple-choice/run_tf_multiple_choice.py

@@ -33,15 +33,9 @@ from transformers import (
     TFTrainingArguments,
     set_seed,
 )
-from transformers.utils import logging as hf_logging
 from utils_multiple_choice import Split, TFMultipleChoiceDataset, processors
 
 
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()
-
-
 logger = logging.getLogger(__name__)
 
 

examples/question-answering/run_squad.py

@@ -29,7 +29,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from transformers import (
     MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     WEIGHTS_NAME,
@@ -46,7 +45,6 @@ from transformers.data.metrics.squad_metrics import (
     squad_evaluate,
 )
 from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -714,11 +712,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 
@@ -736,7 +730,6 @@ def main():
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,
         cache_dir=args.cache_dir if args.cache_dir else None,
-        use_fast=False,  # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
     )
     model = AutoModelForQuestionAnswering.from_pretrained(
         args.model_name_or_path,
@@ -785,10 +778,7 @@ def main():
 
         # Load a trained model and vocabulary that you have fine-tuned
         model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)  # , force_download=True)
-
-        # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
-        # So we use use_fast=False here for now until Fast-tokenizer-compatible-examples are out
-        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case, use_fast=False)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
         model.to(args.device)
 
     # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory

examples/question-answering/run_squad_trainer.py

@@ -22,11 +22,9 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional
 
-import transformers
 from transformers import AutoConfig, AutoModelForQuestionAnswering, AutoTokenizer, HfArgumentParser, SquadDataset
 from transformers import SquadDataTrainingArguments as DataTrainingArguments
 from transformers import Trainer, TrainingArguments
-from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -93,11 +91,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Prepare Question-Answering task
@@ -114,7 +107,6 @@ def main():
     tokenizer = AutoTokenizer.from_pretrained(
         model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
         cache_dir=model_args.cache_dir,
-        use_fast=False,  # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
     )
     model = AutoModelForQuestionAnswering.from_pretrained(
         model_args.model_name_or_path,

examples/question-answering/run_tf_squad.py

@@ -33,12 +33,6 @@ from transformers import (
     squad_convert_examples_to_features,
 )
 from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor
-from transformers.utils import logging as hf_logging
-
-
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()
 
 
 logger = logging.getLogger(__name__)

examples/rag/test_distributed_retriever.py

@@ -16,7 +16,7 @@ from transformers.configuration_dpr import DPRConfig
 from transformers.configuration_rag import RagConfig
 from transformers.file_utils import is_datasets_available, is_faiss_available, is_psutil_available, is_torch_available
 from transformers.retrieval_rag import CustomHFIndex
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me
+from transformers.testing_utils import require_torch_non_multigpu_but_fix_me
 from transformers.tokenization_bart import BartTokenizer
 from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
 from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer
@@ -179,7 +179,7 @@ class RagRetrieverTest(TestCase):
             retriever.init_retrieval(port)
         return retriever
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_pytorch_distributed_retriever_retrieve(self):
         n_docs = 1
         retriever = self.get_dummy_pytorch_distributed_retriever(init_retrieval=True)
@@ -195,7 +195,7 @@ class RagRetrieverTest(TestCase):
         self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
         self.assertListEqual(doc_ids.tolist(), [[1], [0]])
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_custom_hf_index_retriever_retrieve(self):
         n_docs = 1
         retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=False)
@@ -211,7 +211,7 @@ class RagRetrieverTest(TestCase):
         self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
         self.assertListEqual(doc_ids.tolist(), [[1], [0]])
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_custom_pytorch_distributed_retriever_retrieve_from_disk(self):
         n_docs = 1
         retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=True)

examples/requirements.txt

@@ -18,4 +18,3 @@ fire
 pytest
 conllu
 sentencepiece != 0.1.92
-protobuf

examples/seq2seq/README.md

@@ -380,7 +380,7 @@ cp xsum/test* all_pl
 then use `all_pl` as DATA in the command above.
 
 #### Direct Knowledge Distillation (KD)
-+ In this method, we use try to enforce that the student and teacher produce similar encoder_outputs, logits, and hidden_states using `SummarizationDistiller`.
++ In this method, we use try to enforce that the student and teacher produce similar encoder_outputs, logits, and hidden_states using `BartSummarizationDistiller`.
 + This method was used for `sshleifer/distilbart-xsum-12-6`, `6-6`, and `9-6` checkpoints were produced.
 + You must use [`distillation.py`](./distillation.py). Note that this command initializes the student for you.
 

examples/seq2seq/distillation.py

@@ -25,8 +25,8 @@ sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
 from lightning_base import generic_train  # noqa
 
 
-class SummarizationDistiller(SummarizationModule):
-    """Supports T5, Bart, Pegasus and other models that inherit from Bart."""
+class BartSummarizationDistiller(SummarizationModule):
+    """Supports Bart, Pegasus and other models that inherit from Bart."""
 
     loss_names = ["loss", "ce_loss", "mlm_loss", "hid_loss_enc", "hid_loss_dec"]
 
@@ -40,38 +40,25 @@ class SummarizationDistiller(SummarizationModule):
         hparams.model_name_or_path = str(save_dir)  # Tell lightning we are training the student
         teacher = AutoModelForSeq2SeqLM.from_pretrained(hparams.teacher).eval()
         use_task_specific_params(teacher, hparams.task)  # We copy good generation parameters to student by default
-        if hparams.student is not None:
-            student = AutoModelForSeq2SeqLM.from_pretrained(hparams.student)
-            use_task_specific_params(student, hparams.task)
-            e_layer_ids, d_layer_ids = None, None
-        else:
-            student, e_layer_ids, d_layer_ids = create_student_by_copying_alternating_layers(
-                teacher, e=hparams.student_encoder_layers, d=hparams.student_decoder_layers, save_path=save_dir
-            )
-
+        student, e_layer_ids, d_layer_ids = create_student_by_copying_alternating_layers(
+            teacher, e=hparams.student_encoder_layers, d=hparams.student_decoder_layers, save_path=save_dir
+        )
         if hparams.length_penalty != -1:
             student.config.length_penalty = hparams.length_penalty
-        hparams.tokenizer_name = hparams.teacher  # Use teacher's tokenizer
         super().__init__(hparams, model=student, config=student.config)
-        assert (
-            student.config.model_type == teacher.config.model_type
-        ), f"teacher, student model types should be the same, got {student.config.model_type} != {teacher.config.model_type}"
+        model_type = student.config.model_type
+        self.e_layer_ids, self.d_layer_ids = e_layer_ids, d_layer_ids  # type: List[int], List[int]
 
-        if student.config.model_type == "t5":
-            student_encoder_layers = len(student.get_encoder().block)
-            student_decoder_layers = len(student.get_decoder().block)
+        if model_type == "t5":
             teacher_encoder_layers = len(teacher.get_encoder().block)
             teacher_decoder_layers = len(teacher.get_decoder().block)
         else:
-            student_encoder_layers = student.config.encoder_layers
-            student_decoder_layers = student.config.decoder_layers
             teacher_encoder_layers = teacher.config.encoder_layers
             teacher_decoder_layers = teacher.config.decoder_layers
 
-        self.different_base_models = not (hparams.student is None or hparams.teacher == hparams.student)
-        self.do_calc_hidden_loss = (not self.different_base_models) and hparams.alpha_hid > 0
-        self.different_encoder = self.different_base_models or (student_encoder_layers != teacher_encoder_layers)
-        # self.different_encoder determines whether we need to run the teacher encoder
+        self.different_encoder = hparams.student_encoder_layers != teacher_encoder_layers
+        self.different_decoder = hparams.student_decoder_layers != teacher_decoder_layers
+
         self.teacher = teacher
         freeze_params(self.teacher)
 
@@ -80,28 +67,13 @@ class SummarizationDistiller(SummarizationModule):
                 del self.teacher.model.encoder
             except AttributeError:  # T5
                 del self.teacher.encoder
-
-        if e_layer_ids is None:
-            e_layer_ids = list(range(student_encoder_layers))
-        if d_layer_ids is None:
-            d_layer_ids = list(range(student_decoder_layers))
-
-        self.e_layer_ids, self.d_layer_ids = e_layer_ids, d_layer_ids  # type: List[int], List[int]
-
-        if self.do_calc_hidden_loss:  # Intermediate supervision: Decide which layers to supervise
-            if hparams.supervise_forward:
-                self.e_matches = get_layers_to_supervise(
-                    n_student=len(self.e_layer_ids), n_teacher=teacher_encoder_layers
-                )
-                self.d_matches = get_layers_to_supervise(
-                    n_student=len(self.d_layer_ids), n_teacher=teacher_decoder_layers
-                )
-            else:  # student layer should emulate hidden states of the teacher layer it was copied from
-                self.e_matches = self.e_layer_ids
-                self.d_matches = self.d_layer_ids
-        else:
-            self.e_matches = None
-            self.d_matches = None
+        # Intermediate supervision: Decide which layers to supervise
+        if hparams.supervise_forward:
+            self.e_matches = get_layers_to_supervise(n_student=len(self.e_layer_ids), n_teacher=teacher_encoder_layers)
+            self.d_matches = get_layers_to_supervise(n_student=len(self.d_layer_ids), n_teacher=teacher_decoder_layers)
+        else:  # student layer should emulate hidden states of the teacher layer it was copied from
+            self.e_matches = self.e_layer_ids
+            self.d_matches = self.d_layer_ids
 
         self.ce_loss_fct = nn.KLDivLoss(reduction="batchmean")
         self.temperature = 2.0
@@ -111,8 +83,22 @@ class SummarizationDistiller(SummarizationModule):
         gc.collect()
         torch.cuda.empty_cache()
 
+    def calc_mse_loss(self, teacher_outputs: torch.Tensor, student_outputs: torch.Tensor, mask) -> torch.FloatTensor:
+        """Supervise MSE(teacher.encoder_outputs, student.encoder_outputs)."""
+        # raise NotImplementedError()
+        if mask is not None:
+            # mask has False at padding_idx
+            sel_mask = mask[:, :, None].expand_as(student_outputs).bool()
+            s_logits_slct = torch.masked_select(student_outputs, sel_mask)
+            t_logits_slct = torch.masked_select(teacher_outputs, sel_mask)
+        else:
+            t_logits_slct = teacher_outputs
+            s_logits_slct = student_outputs
+        return F.mse_loss(s_logits_slct, t_logits_slct)
+
     def calc_ce_loss(self, mask, s_logits, t_logits):
         """Copy pasted from distillbert (transformers/examples/distillation/)"""
+
         # mask has False at padding_idx
         sel_mask = mask[:, :, None].expand_as(s_logits)
         vocab_size = s_logits.size(-1)
@@ -136,8 +122,8 @@ class SummarizationDistiller(SummarizationModule):
         add_distill_args(parser)
         return parser
 
-    def _step(self, batch: dict) -> tuple:
-        """Compute the loss for a batch"""
+    def _step(self, batch):
+        # assert is_frozen(self.teacher) copied_decoder_layers
         pad_token_id = self.tokenizer.pad_token_id
         input_ids, src_mask, labels = batch["input_ids"], batch["attention_mask"], batch["labels"]
         if isinstance(self.model, T5ForConditionalGeneration):
@@ -146,16 +132,14 @@ class SummarizationDistiller(SummarizationModule):
             decoder_input_ids = shift_tokens_right(labels, pad_token_id)
 
         # noinspection PyCallingNonCallable
-        student_outputs = self(
+        lm_logits, dec_hidden, enc_outputs, enc_hidden_state = self(
             input_ids,
             attention_mask=src_mask,
             decoder_input_ids=decoder_input_ids,
-            output_hidden_states=self.do_calc_hidden_loss,
+            output_hidden_states=True,
             output_attentions=False,
             use_cache=False,
-            return_dict=True,
         )
-        lm_logits = student_outputs.logits
 
         # Same cross entropy vs. label smoothing logic as finetune.py
         assert lm_logits.shape[-1] == self.model.config.vocab_size
@@ -164,7 +148,7 @@ class SummarizationDistiller(SummarizationModule):
             loss_fct = torch.nn.CrossEntropyLoss(ignore_index=pad_token_id)
             student_lm_loss = loss_fct(lm_logits.view(-1, lm_logits.shape[-1]), labels.view(-1))
         else:
-            lprobs = F.log_softmax(lm_logits, dim=-1)
+            lprobs = torch.nn.functional.log_softmax(lm_logits, dim=-1)
             student_lm_loss, _ = label_smoothed_nll_loss(
                 lprobs, labels, self.hparams.label_smoothing, ignore_index=pad_token_id
             )
@@ -172,44 +156,37 @@ class SummarizationDistiller(SummarizationModule):
         def zero_tensor():
             return torch.tensor(0.0).type_as(student_lm_loss)
 
-        teacher_enc_outputs = student_outputs.encoder_last_hidden_state  # use this unless self.different_base_models
         hid_loss_enc, hid_loss_dec = zero_tensor(), zero_tensor()
         if self.different_encoder:  # compute encoder hidden state loss
-            all_teacher_encoder_outputs = self.teacher.get_encoder()(
+            with torch.no_grad():
+                teacher_enc_hid = self.teacher.get_encoder()(
+                    input_ids, attention_mask=src_mask, output_hidden_states=True, return_dict=True
+                ).hidden_states
+
+            hid_loss_enc = self.calc_hidden_loss(
+                src_mask,
+                enc_hidden_state,
+                teacher_enc_hid,
+                self.e_matches,
+                normalize_hidden=self.hparams.normalize_hidden,
+            )
+
+        with torch.no_grad():
+            outputs = self.teacher(
                 input_ids,
                 attention_mask=src_mask,
-                output_hidden_states=self.do_calc_hidden_loss,
+                encoder_outputs=(enc_outputs,),
+                decoder_input_ids=decoder_input_ids,
+                lm_labels=labels,
+                output_hidden_states=True,
                 return_dict=True,
             )
-            if self.different_base_models:
-                teacher_enc_outputs = all_teacher_encoder_outputs.last_hidden_state
-            elif self.do_calc_hidden_loss:
-                hid_loss_enc = self.calc_hidden_loss(
-                    src_mask,
-                    student_outputs.encoder_hidden_states,
-                    all_teacher_encoder_outputs.hidden_states,
-                    self.e_matches,
-                    normalize_hidden=self.hparams.normalize_hidden,
-                )
-
-        teacher_outputs = self.teacher(
-            input_ids,
-            attention_mask=src_mask,
-            encoder_outputs=(teacher_enc_outputs,),
-            decoder_input_ids=decoder_input_ids,
-            output_hidden_states=self.do_calc_hidden_loss,
-            use_cache=False,  # since we are not passing labels, never let this default to True
-            return_dict=True,
-        )
+            tlogits, tdec_hidden = outputs.logits, outputs.decoder_hidden_states
         dec_mask = decoder_input_ids.ne(pad_token_id)
-        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, teacher_outputs.logits)
-        if self.do_calc_hidden_loss:  # Intermediate supervision of decoder hidden states
+        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, tlogits)
+        if self.alpha_hid > 0:  # Intermediate supervision of decoder hidden states
             hid_loss_dec = self.calc_hidden_loss(
-                dec_mask,
-                student_outputs.decoder_hidden_states,
-                teacher_outputs.decoder_hidden_states,
-                self.d_matches,
-                normalize_hidden=self.hparams.normalize_hidden,
+                dec_mask, dec_hidden, tdec_hidden, self.d_matches, normalize_hidden=self.hparams.normalize_hidden
             )
 
         blended_loss = (
@@ -229,7 +206,6 @@ class SummarizationDistiller(SummarizationModule):
         valid_count = mask.sum() * hidden_states[0].size(-1)
         student_states = torch.stack([hidden_states[i] for i in range(len(matches))])
         teacher_states = torch.stack([hidden_states_T[j] for j in matches])
-        assert student_states.shape == teacher_states.shape, f"{student_states.shape} != {teacher_states.shape}"
         if normalize_hidden:
             student_states = F.layer_norm(student_states, student_states.shape[1:])
             teacher_states = F.layer_norm(teacher_states, teacher_states.shape[1:])
@@ -239,16 +215,10 @@ class SummarizationDistiller(SummarizationModule):
 
 
 def add_distill_args(parser):
-    # NOTE: if --student argument was specified and the teacher and student base models
-    # are different, the models still have to have the same tokenizer, specified by
-    # --tokenizer_name. So, for example, you can distill from t5_large to t5_small but not
-    # from bart to t5. This s because if the tokenizers are different, the output space
-    # for the two models is also different and their logits are not comparable.
     parser.add_argument("--teacher", type=str)
     parser.add_argument("--alpha_ce", default=0.8, type=float)
     parser.add_argument("--alpha_mlm", default=0.2, type=float)
     parser.add_argument("--alpha_hid", default=0.0, type=float, required=False)
-    parser.add_argument("--student", type=str, required=False)
     parser.add_argument("--student_decoder_layers", default=12, type=int, required=False)
     parser.add_argument("--student_encoder_layers", default=12, type=int, required=False)
     parser.add_argument("--no_teacher", action="store_true", default=False)
@@ -257,8 +227,8 @@ def add_distill_args(parser):
     parser.add_argument("--normalize_hidden", action="store_true", default=False)
 
 
-class TranslationDistiller(SummarizationDistiller):
-    """Supports T5, mBART, Marian, other models that inherit from Bart."""
+class BartTranslationDistiller(BartSummarizationDistiller):
+    """Supports Mbart, Marian, other models that inherit from Bart."""
 
     mode = "translation"
     metric_names = ["bleu"]
@@ -287,7 +257,7 @@ def create_module(args):
     if args.no_teacher:
         module_cls = TranslationModule if "translation" in args.task else SummarizationModule
     else:  # DISTILL WITH TEACHER
-        module_cls = TranslationDistiller if "translation" in args.task else SummarizationDistiller
+        module_cls = BartTranslationDistiller if "translation" in args.task else BartSummarizationDistiller
     args.setup_cls: str = module_cls.__name__
     print(f"using module {args.setup_cls}")
     model = module_cls(args)
@@ -305,7 +275,7 @@ def distill_main(args):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser = pl.Trainer.add_argparse_args(parser)
-    parser = SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+    parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
     args = parser.parse_args()
 
     distill_main(args)

examples/seq2seq/finetune_trainer.py

@@ -4,11 +4,10 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional
 
-import transformers
 from seq2seq_trainer import Seq2SeqTrainer
 from seq2seq_training_args import Seq2SeqTrainingArguments
 from transformers import AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer, HfArgumentParser, MBartTokenizer, set_seed
-from transformers.trainer_utils import EvaluationStrategy, is_main_process
+from transformers.trainer_utils import EvaluationStrategy
 from utils import (
     Seq2SeqDataCollator,
     Seq2SeqDataset,
@@ -132,11 +131,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/seq2seq/test_bash_script.py

@@ -9,11 +9,11 @@ import pytorch_lightning as pl
 import timeout_decorator
 import torch
 
-from distillation import SummarizationDistiller, distill_main
+from distillation import BartSummarizationDistiller, distill_main
 from finetune import SummarizationModule, main
 from transformers import MarianMTModel
 from transformers.file_utils import cached_path
-from transformers.testing_utils import TestCasePlus, require_torch_gpu, require_torch_non_multi_gpu_but_fix_me, slow
+from transformers.testing_utils import TestCasePlus, require_torch_gpu, require_torch_non_multigpu_but_fix_me, slow
 from utils import load_json
 
 
@@ -32,7 +32,7 @@ class TestMbartCc25Enro(TestCasePlus):
 
     @slow
     @require_torch_gpu
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_model_download(self):
         """This warms up the cache so that we can time the next test without including download time, which varies between machines."""
         MarianMTModel.from_pretrained(MARIAN_MODEL)
@@ -40,7 +40,7 @@ class TestMbartCc25Enro(TestCasePlus):
     # @timeout_decorator.timeout(1200)
     @slow
     @require_torch_gpu
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_train_mbart_cc25_enro_script(self):
         env_vars_to_replace = {
             "$MAX_LEN": 64,
@@ -75,7 +75,7 @@ class TestMbartCc25Enro(TestCasePlus):
             --num_sanity_val_steps 0
             --eval_beams 2
         """.split()
-        # XXX: args.gpus > 1 : handle multi_gpu in the future
+        # XXX: args.gpus > 1 : handle multigpu in the future
 
         testargs = ["finetune.py"] + bash_script.split() + args
         with patch.object(sys, "argv", testargs):
@@ -129,7 +129,7 @@ class TestDistilMarianNoTeacher(TestCasePlus):
     @timeout_decorator.timeout(600)
     @slow
     @require_torch_gpu
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_opus_mt_distill_script(self):
         data_dir = f"{self.test_file_dir_str}/test_data/wmt_en_ro"
         env_vars_to_replace = {
@@ -170,9 +170,9 @@ class TestDistilMarianNoTeacher(TestCasePlus):
         with patch.object(sys, "argv", testargs):
             parser = argparse.ArgumentParser()
             parser = pl.Trainer.add_argparse_args(parser)
-            parser = SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+            parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
             args = parser.parse_args()
-            # assert args.gpus == gpus THIS BREAKS for multi_gpu
+            # assert args.gpus == gpus THIS BREAKS for multigpu
 
             model = distill_main(args)
 

examples/seq2seq/test_datasets.py

@@ -11,7 +11,7 @@ from save_len_file import save_len_file
 from test_seq2seq_examples import ARTICLES, BART_TINY, MARIAN_TINY, MBART_TINY, SUMMARIES, T5_TINY, make_test_data_dir
 from transformers import AutoTokenizer
 from transformers.modeling_bart import shift_tokens_right
-from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me, slow
+from transformers.testing_utils import TestCasePlus, require_torch_non_multigpu_but_fix_me, slow
 from utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeq2SeqDataset, Seq2SeqDataset
 
 
@@ -30,7 +30,7 @@ class TestAll(TestCasePlus):
         ],
     )
     @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_seq2seq_dataset_truncation(self, tok_name):
         tokenizer = AutoTokenizer.from_pretrained(tok_name)
         tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
@@ -70,7 +70,7 @@ class TestAll(TestCasePlus):
             break  # No need to test every batch
 
     @parameterized.expand([BART_TINY, BERT_BASE_CASED])
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_legacy_dataset_truncation(self, tok):
         tokenizer = AutoTokenizer.from_pretrained(tok)
         tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
@@ -95,7 +95,7 @@ class TestAll(TestCasePlus):
             assert max_len_target > trunc_target  # Truncated
             break  # No need to test every batch
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_pack_dataset(self):
         tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25")
 
@@ -114,7 +114,7 @@ class TestAll(TestCasePlus):
         assert orig_paths == new_paths
 
     @pytest.mark.skipif(not FAIRSEQ_AVAILABLE, reason="This test requires fairseq")
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_dynamic_batch_size(self):
         if not FAIRSEQ_AVAILABLE:
             return
@@ -139,7 +139,7 @@ class TestAll(TestCasePlus):
         if failures:
             raise AssertionError(f"too many tokens in {len(failures)} batches")
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_sortish_sampler_reduces_padding(self):
         ds, _, tokenizer = self._get_dataset(max_len=512)
         bs = 2
@@ -179,7 +179,7 @@ class TestAll(TestCasePlus):
         )
         return ds, max_tokens, tokenizer
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_distributed_sortish_sampler_splits_indices_between_procs(self):
         ds, max_tokens, tokenizer = self._get_dataset()
         ids1 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=0, add_extra_examples=False))
@@ -195,9 +195,9 @@ class TestAll(TestCasePlus):
             PEGASUS_XSUM,
         ],
     )
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_dataset_kwargs(self, tok_name):
-        tokenizer = AutoTokenizer.from_pretrained(tok_name, use_fast=False)
+        tokenizer = AutoTokenizer.from_pretrained(tok_name)
         if tok_name == MBART_TINY:
             train_dataset = Seq2SeqDataset(
                 tokenizer,

examples/seq2seq/test_fsmt_bleu_score.py

@@ -22,7 +22,7 @@ from transformers import FSMTForConditionalGeneration, FSMTTokenizer
 from transformers.testing_utils import (
     get_tests_dir,
     require_torch,
-    require_torch_non_multi_gpu_but_fix_me,
+    require_torch_non_multigpu_but_fix_me,
     slow,
     torch_device,
 )
@@ -54,7 +54,7 @@ class ModelEvalTester(unittest.TestCase):
         ]
     )
     @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_bleu_scores(self, pair, min_bleu_score):
         # note: this test is not testing the best performance since it only evals a small batch
         # but it should be enough to detect a regression in the output quality

examples/seq2seq/test_make_student.py

@@ -4,7 +4,7 @@ import unittest
 from make_student import create_student_by_copying_alternating_layers
 from transformers import AutoConfig
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_torch, require_torch_non_multi_gpu_but_fix_me
+from transformers.testing_utils import require_torch, require_torch_non_multigpu_but_fix_me
 
 
 TINY_BART = "sshleifer/bart-tiny-random"
@@ -17,28 +17,28 @@ class MakeStudentTester(unittest.TestCase):
     def teacher_config(self):
         return AutoConfig.from_pretrained(TINY_BART)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_valid_t5(self):
         student, *_ = create_student_by_copying_alternating_layers(TINY_T5, tempfile.mkdtemp(), e=1, d=1)
         self.assertEqual(student.config.num_hidden_layers, 1)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_asymmetric_t5(self):
         student, *_ = create_student_by_copying_alternating_layers(TINY_T5, tempfile.mkdtemp(), e=1, d=None)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_same_decoder_small_encoder(self):
         student, *_ = create_student_by_copying_alternating_layers(TINY_BART, tempfile.mkdtemp(), e=1, d=None)
         self.assertEqual(student.config.encoder_layers, 1)
         self.assertEqual(student.config.decoder_layers, self.teacher_config.encoder_layers)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_small_enc_small_dec(self):
         student, *_ = create_student_by_copying_alternating_layers(TINY_BART, tempfile.mkdtemp(), e=1, d=1)
         self.assertEqual(student.config.encoder_layers, 1)
         self.assertEqual(student.config.decoder_layers, 1)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_raises_assert(self):
         with self.assertRaises(AssertionError):
             create_student_by_copying_alternating_layers(TINY_BART, tempfile.mkdtemp(), e=None, d=None)

examples/seq2seq/test_seq2seq_examples.py

@@ -24,7 +24,7 @@ from transformers.testing_utils import (
     CaptureStdout,
     TestCasePlus,
     require_torch_gpu,
-    require_torch_non_multi_gpu_but_fix_me,
+    require_torch_non_multigpu_but_fix_me,
     slow,
 )
 from utils import ROUGE_KEYS, label_smoothed_nll_loss, lmap, load_json
@@ -96,7 +96,6 @@ CHEAP_ARGS = {
     "freeze_encoder": False,
     "auto_scale_batch_size": False,
     "overwrite_output_dir": False,
-    "student": None,
 }
 
 
@@ -108,7 +107,6 @@ def _dump_articles(path: Path, articles: list):
 ARTICLES = [" Sam ate lunch today.", "Sams lunch ingredients."]
 SUMMARIES = ["A very interesting story about what I ate for lunch.", "Avocado, celery, turkey, coffee"]
 T5_TINY = "patrickvonplaten/t5-tiny-random"
-T5_TINIER = "sshleifer/t5-tinier-random"
 BART_TINY = "sshleifer/bart-tiny-random"
 MBART_TINY = "sshleifer/tiny-mbart"
 MARIAN_TINY = "sshleifer/tiny-marian-en-de"
@@ -135,7 +133,7 @@ class TestSummarizationDistiller(TestCasePlus):
 
     @slow
     @require_torch_gpu
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_hub_configs(self):
         """I put require_torch_gpu cause I only want this to run with self-scheduled."""
 
@@ -153,12 +151,12 @@ class TestSummarizationDistiller(TestCasePlus):
                 failures.append(m)
         assert not failures, f"The following models could not be loaded through AutoConfig: {failures}"
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_distill_no_teacher(self):
         updates = dict(student_encoder_layers=2, student_decoder_layers=1, no_teacher=True)
         self._test_distiller_cli(updates)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_distill_checkpointing_with_teacher(self):
         updates = dict(
             student_encoder_layers=2,
@@ -183,7 +181,7 @@ class TestSummarizationDistiller(TestCasePlus):
         convert_pl_to_hf(ckpts[0], transformer_ckpts[0].parent, out_path_new)
         assert os.path.exists(os.path.join(out_path_new, "pytorch_model.bin"))
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_loss_fn(self):
         model = AutoModelForSeq2SeqLM.from_pretrained(BART_TINY, return_dict=True)
         input_ids, mask = model.dummy_inputs["input_ids"], model.dummy_inputs["attention_mask"]
@@ -204,7 +202,7 @@ class TestSummarizationDistiller(TestCasePlus):
             # TODO: understand why this breaks
             self.assertEqual(nll_loss, model_computed_loss)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_distill_mbart(self):
         updates = dict(
             student_encoder_layers=2,
@@ -229,7 +227,7 @@ class TestSummarizationDistiller(TestCasePlus):
         assert len(all_files) > 2
         self.assertEqual(len(transformer_ckpts), 2)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_distill_t5(self):
         updates = dict(
             student_encoder_layers=1,
@@ -241,16 +239,6 @@ class TestSummarizationDistiller(TestCasePlus):
         )
         self._test_distiller_cli(updates)
 
-    @require_torch_non_multi_gpu_but_fix_me
-    def test_distill_different_base_models(self):
-        updates = dict(
-            teacher=T5_TINY,
-            student=T5_TINIER,
-            model_name_or_path=T5_TINIER,
-            tokenizer_name=T5_TINIER,
-        )
-        self._test_distiller_cli(updates)
-
     def _test_distiller_cli(self, updates, check_contents=True):
         default_updates = dict(
             label_smoothing=0.0,
@@ -321,21 +309,21 @@ class TestTheRest(TestCasePlus):
 
     # test one model to quickly (no-@slow) catch simple problems and do an
     # extensive testing of functionality with multiple models as @slow separately
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_eval(self):
         self.run_eval_tester(T5_TINY)
 
     # any extra models should go into the list here - can be slow
     @parameterized.expand([BART_TINY, MBART_TINY])
     @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_eval_slow(self, model):
         self.run_eval_tester(model)
 
     # testing with 2 models to validate: 1. translation (t5) 2. summarization (mbart)
     @parameterized.expand([T5_TINY, MBART_TINY])
     @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_eval_search(self, model):
         input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
         output_file_name = input_file_name.parent / "utest_output.txt"
@@ -386,7 +374,7 @@ class TestTheRest(TestCasePlus):
     @parameterized.expand(
         [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY],
     )
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_finetune(self, model):
         args_d: dict = CHEAP_ARGS.copy()
         task = "translation" if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else "summarization"
@@ -438,7 +426,7 @@ class TestTheRest(TestCasePlus):
         assert isinstance(example_batch, dict)
         assert len(example_batch) >= 4
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_finetune_extra_model_args(self):
         args_d: dict = CHEAP_ARGS.copy()
 
@@ -489,7 +477,7 @@ class TestTheRest(TestCasePlus):
             model = main(args)
         assert str(excinfo.value) == f"model config doesn't have a `{unsupported_param}` attribute"
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_finetune_lr_schedulers(self):
         args_d: dict = CHEAP_ARGS.copy()
 

examples/seq2seq/test_seq2seq_examples_multi_gpu.py

@@ -8,7 +8,7 @@ from transformers.testing_utils import (
     execute_subprocess_async,
     get_gpu_count,
     require_torch_gpu,
-    require_torch_multi_gpu,
+    require_torch_multigpu,
     slow,
 )
 
@@ -21,8 +21,8 @@ class TestSummarizationDistillerMultiGPU(TestCasePlus):
     def setUpClass(cls):
         return cls
 
-    @require_torch_multi_gpu
-    def test_multi_gpu(self):
+    @require_torch_multigpu
+    def test_multigpu(self):
 
         updates = dict(
             no_teacher=True,

examples/seq2seq/test_tatoeba_conversion.py

@@ -4,7 +4,7 @@ import unittest
 
 from transformers.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow
+from transformers.testing_utils import require_torch_non_multigpu_but_fix_me, slow
 
 
 @unittest.skipUnless(os.path.exists(DEFAULT_REPO), "Tatoeba directory does not exist.")
@@ -15,12 +15,12 @@ class TatoebaConversionTester(unittest.TestCase):
         return TatoebaConverter(save_dir=tmp_dir)
 
     @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_resolver(self):
         self.resolver.convert_models(["heb-eng"])
 
     @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_model_card(self):
         content, mmeta = self.resolver.write_model_card("opus-mt-he-en", dry_run=True)
         assert mmeta["long_pair"] == "heb-eng"

examples/test_examples.py

@@ -23,7 +23,7 @@ from unittest.mock import patch
 import torch
 
 from transformers.file_utils import is_apex_available
-from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me, torch_device
+from transformers.testing_utils import TestCasePlus, require_torch_non_multigpu_but_fix_me, torch_device
 
 
 SRC_DIRS = [
@@ -67,7 +67,7 @@ def is_cuda_and_apex_available():
 
 
 class ExamplesTests(TestCasePlus):
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -100,7 +100,7 @@ class ExamplesTests(TestCasePlus):
             for value in result.values():
                 self.assertGreaterEqual(value, 0.75)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_pl_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -138,7 +138,7 @@ class ExamplesTests(TestCasePlus):
             #         self.assertGreaterEqual(v, 0.75, f"({k})")
             #
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_clm(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -170,7 +170,7 @@ class ExamplesTests(TestCasePlus):
             result = run_clm.main()
             self.assertLess(result["perplexity"], 100)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_mlm(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -196,7 +196,7 @@ class ExamplesTests(TestCasePlus):
             result = run_mlm.main()
             self.assertLess(result["perplexity"], 42)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_ner(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -227,7 +227,7 @@ class ExamplesTests(TestCasePlus):
             self.assertGreaterEqual(result["eval_precision"], 0.75)
             self.assertLess(result["eval_loss"], 0.5)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_squad(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -256,7 +256,7 @@ class ExamplesTests(TestCasePlus):
             self.assertGreaterEqual(result["f1"], 25)
             self.assertGreaterEqual(result["exact"], 21)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_generation(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)

examples/test_xla_examples.py

@@ -20,7 +20,7 @@ import unittest
 from time import time
 from unittest.mock import patch
 
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, require_torch_tpu
+from transformers.testing_utils import require_torch_non_multigpu_but_fix_me, require_torch_tpu
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -30,7 +30,7 @@ logger = logging.getLogger()
 
 @require_torch_tpu
 class TorchXLAExamplesTests(unittest.TestCase):
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_glue(self):
         import xla_spawn
 
@@ -82,7 +82,7 @@ class TorchXLAExamplesTests(unittest.TestCase):
             # Assert that the script takes less than 300 seconds to make sure it doesn't hang.
             self.assertLess(end - start, 500)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_trainer_tpu(self):
         import xla_spawn
 

examples/text-classification/run_glue.py

@@ -171,8 +171,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info(f"Training/evaluation parameters {training_args}")
 
     # Set seed before initializing model.

examples/text-classification/run_tf_glue.py

@@ -27,12 +27,6 @@ from transformers import (
     glue_processors,
     glue_tasks_num_labels,
 )
-from transformers.utils import logging as hf_logging
-
-
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()
 
 
 class Split(Enum):

examples/text-classification/run_tf_text_classification.py

@@ -21,12 +21,6 @@ from transformers import (
     TFTrainer,
     TFTrainingArguments,
 )
-from transformers.utils import logging as hf_logging
-
-
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()
 
 
 def get_tfds(

examples/text-classification/run_xnli.py

@@ -29,7 +29,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from transformers import (
     WEIGHTS_NAME,
     AdamW,
@@ -42,7 +41,6 @@ from transformers import glue_convert_examples_to_features as convert_examples_t
 from transformers import xnli_compute_metrics as compute_metrics
 from transformers import xnli_output_modes as output_modes
 from transformers import xnli_processors as processors
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -528,11 +526,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 

examples/token-classification/run_ner.py

@@ -163,8 +163,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -346,7 +344,7 @@ def main():
     if training_args.do_predict:
         logger.info("*** Predict ***")
 
-        test_dataset = tokenized_datasets["test"]
+        test_dataset = datasets["test"]
         predictions, labels, metrics = trainer.predict(test_dataset)
         predictions = np.argmax(predictions, axis=2)
 

examples/token-classification/run_ner_old.py

@@ -25,7 +25,6 @@ import numpy as np
 from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
 from torch import nn
 
-import transformers
 from transformers import (
     AutoConfig,
     AutoModelForTokenClassification,
@@ -36,7 +35,6 @@ from transformers import (
     TrainingArguments,
     set_seed,
 )
-from transformers.trainer_utils import is_main_process
 from utils_ner import Split, TokenClassificationDataset, TokenClassificationTask
 
 
@@ -141,11 +139,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/token-classification/run_tf_ner.py

@@ -33,15 +33,9 @@ from transformers import (
     TFTrainer,
     TFTrainingArguments,
 )
-from transformers.utils import logging as hf_logging
 from utils_ner import Split, TFTokenClassificationDataset, TokenClassificationTask
 
 
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()
-
-
 logger = logging.getLogger(__name__)
 
 

examples/token-classification/test_ner_examples.py

@@ -4,7 +4,7 @@ import unittest
 from unittest.mock import patch
 
 import run_ner_old as run_ner
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow
+from transformers.testing_utils import require_torch_non_multigpu_but_fix_me, slow
 
 
 logging.basicConfig(level=logging.INFO)
@@ -14,7 +14,7 @@ logger = logging.getLogger()
 
 class ExamplesTests(unittest.TestCase):
     @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_ner(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -35,7 +35,7 @@ class ExamplesTests(unittest.TestCase):
             result = run_ner.main()
             self.assertLess(result["eval_loss"], 1.5)
 
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
     def test_run_ner_pl(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)

model_cards/deepset/gbert-large/README.md

@@ -5,7 +5,7 @@ datasets:
 - wikipedia
 - OPUS
 - OpenLegalData
-- oscar
+- OSCAR
 ---
 
 # German BERT large

model_cards/deepset/gelectra-large-generator/README.md

@@ -5,7 +5,7 @@ datasets:
 - wikipedia
 - OPUS
 - OpenLegalData
-- oscar
+- OSCAR
 ---
 
 # German ELECTRA large generator

model_cards/deepset/gelectra-large/README.md

@@ -5,7 +5,7 @@ datasets:
 - wikipedia
 - OPUS
 - OpenLegalData
-- oscar
+- OSCAR
 ---
 
 # German ELECTRA large

model_cards/deepset/roberta-base-squad2-v2/README.md

@@ -90,9 +90,9 @@ tokenizer = Tokenizer.load(model_name)
 ### In haystack
 For doing QA at scale (i.e. many docs instead of single paragraph), you can load the model also in [haystack](https://github.com/deepset-ai/haystack/):
 ```python
-reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2-v2")
+reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2")
 # or 
-reader = TransformersReader(model_name_or_path="deepset/roberta-base-squad2-v2",tokenizer="deepset/roberta-base-squad2-v2")
+reader = TransformersReader(model="deepset/roberta-base-squad2",tokenizer="deepset/roberta-base-squad2")
 ```
 
 

model_cards/deepset/roberta-base-squad2/README.md

@@ -5,7 +5,7 @@ datasets:
 
 # roberta-base for QA 
 
-NOTE: This is version 2 of the model. See [this github issue](https://github.com/deepset-ai/FARM/issues/552) from the FARM repository for an explanation of why we updated. If you'd like to use version 1, specify `revision="v1.0"` when loading the model in Transformers 3.5.
+NOTE: This model has been superseded by deepset/roberta-base-squad2-v2. For an explanation of why, see [this github issue](https://github.com/deepset-ai/FARM/issues/552) from the FARM repository.
 
 ## Overview
 **Language model:** roberta-base  
@@ -19,10 +19,10 @@ NOTE: This is version 2 of the model. See [this github issue](https://github.com
 ## Hyperparameters
 
 ```
-batch_size = 96
-n_epochs = 2
+batch_size = 50
+n_epochs = 3
 base_LM_model = "roberta-base"
-max_seq_len = 386
+max_seq_len = 384
 learning_rate = 3e-5
 lr_schedule = LinearWarmup
 warmup_proportion = 0.2
@@ -32,18 +32,9 @@ max_query_length=64
 
 ## Performance
 Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/).
-
 ```
-"exact": 79.97136359807968
-"f1": 83.00449234495325
-
-"total": 11873
-"HasAns_exact": 78.03643724696356
-"HasAns_f1": 84.11139298441825
-"HasAns_total": 5928
-"NoAns_exact": 81.90075693860386
-"NoAns_f1": 81.90075693860386
-"NoAns_total": 5945
+"exact": 78.49743114629833,
+"f1": 81.73092721240889
 ```
 
 ## Usage
@@ -94,7 +85,7 @@ For doing QA at scale (i.e. many docs instead of single paragraph), you can load
 ```python
 reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2")
 # or 
-reader = TransformersReader(model_name_or_path="deepset/roberta-base-squad2",tokenizer="deepset/roberta-base-squad2")
+reader = TransformersReader(model="deepset/roberta-base-squad2",tokenizer="deepset/roberta-base-squad2")
 ```
 
 

model_cards/digitalepidemiologylab/covid-twitter-bert/README.md

@@ -4,7 +4,7 @@ thumbnail: "https://raw.githubusercontent.com/digitalepidemiologylab/covid-twitt
 tags:
 - Twitter
 - COVID-19
-license: mit
+license: "MIT"
 ---
 
 # COVID-Twitter-BERT (CT-BERT) v1

model_cards/facebook/bart-large-mnli/README.md

@@ -2,80 +2,7 @@
 license: mit
 thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 pipeline_tag: zero-shot-classification
-datasets:
-- multi_nli
+widget:
+- text: "Last week I upgraded my iOS version and ever since then my phone has been overheating whenever I use your app."
+  labels: "mobile, website, billing, account access"
 ---
-
-# bart-large-mnli
-
-This is the checkpoint for [bart-large](https://huggingface.co/facebook/bart-large) after being trained on the [MultiNLI (MNLI)](https://huggingface.co/datasets/multi_nli) dataset.
-
-Additional information about this model:
-- The [bart-large](https://huggingface.co/facebook/bart-large) model page
-- [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension
-](https://arxiv.org/abs/1910.13461)
-- [BART fairseq implementation](https://github.com/pytorch/fairseq/tree/master/fairseq/models/bart)
-
-## NLI-based Zero Shot Text Classification
-
-[Yin et al.](https://arxiv.org/abs/1909.00161) proposed a method for using pre-trained NLI models as a ready-made zero-shot sequence classifiers. The method works by posing the sequence to be classified as the NLI premise and to construct a hypothesis from each candidate label. For example, if we want to evaluate whether a sequence belongs to the class "politics", we could construct a hypothesis of `This text is about politics.`. The probabilities for entailment and contradiction are then converted to label probabilities.
-
-This method is surprisingly effective in many cases, particularly when used with larger pre-trained models like BART and Roberta. See [this blog post](https://joeddav.github.io/blog/2020/05/29/ZSL.html) for a more expansive introduction to this and other zero shot methods, and see the code snippets below for examples of using this model for zero-shot classification both with Hugging Face's built-in pipeline and with native Transformers/PyTorch code.
-
-#### With the zero-shot classification pipeline
-
-The model can be loaded with the `zero-shot-classification` pipeline like so:
-
-```python
-from transformers import pipeline
-classifier = pipeline("zero-shot-classification",
-                      model="facebook/bart-large-mnli")
-```
-
-You can then use this pipeline to classify sequences into any of the class names you specify.
-
-```python
-sequence_to_classify = "one day I will see the world"
-candidate_labels = ['travel', 'cooking', 'dancing']
-classifier(sequence_to_classify, candidate_labels)
-#{'labels': ['travel', 'dancing', 'cooking'],
-# 'scores': [0.9938651323318481, 0.0032737774308770895, 0.002861034357920289],
-# 'sequence': 'one day I will see the world'}
-```
-
-If more than one candidate label can be correct, pass `multi_class=True` to calculate each class independently:
-
-```python
-candidate_labels = ['travel', 'cooking', 'dancing', 'exploration']
-classifier(sequence_to_classify, candidate_labels, multi_class=True)
-#{'labels': ['travel', 'exploration', 'dancing', 'cooking'],
-# 'scores': [0.9945111274719238,
-#  0.9383890628814697,
-#  0.0057061901316046715,
-#  0.0018193122232332826],
-# 'sequence': 'one day I will see the world'}
-```
-
-
-#### With manual PyTorch
-
-```python
-# pose sequence as a NLI premise and label as a hypothesis
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
-nli_model = AutoModelForSequenceClassification.from_pretrained('joeddav/xlm-roberta-large-xnli')
-tokenizer = AutoTokenizer.from_pretrained('joeddav/xlm-roberta-large-xnli')
-
-premise = sequence
-hypothesis = f'This example is {label}.'
-
-# run through model pre-trained on MNLI
-x = tokenizer.encode(premise, hypothesis, return_tensors='pt',
-                     truncation_strategy='only_first')
-logits = nli_model(x.to(device))[0]
-
-# we throw away "neutral" (dim 1) and take the probability of
-# "entailment" (2) as the probability of the label being true 
-entail_contradiction_logits = logits[:,[0,2]]
-probs = entail_contradiction_logits.softmax(dim=1)
-prob_label_is_true = probs[:,1]
-```

model_cards/joeddav/bart-large-mnli-yahoo-answers/README.md

@@ -16,7 +16,7 @@ This model takes [facebook/bart-large-mnli](https://huggingface.co/facebook/bart
 
 You can play with an interactive demo of this zero-shot technique with this model, as well as the non-finetuned [facebook/bart-large-mnli](https://huggingface.co/facebook/bart-large-mnli), [here](https://huggingface.co/zero-shot/).
 
-## Intended Usage
+## Inteded Usage
 
 This model was fine-tuned on topic classification and will perform best at zero-shot topic classification. Use `hypothesis_template="This text is about {}."` as this is the template used during fine-tuning.
 

model_cards/joeddav/xlm-roberta-large-xnli/README.md

@@ -24,7 +24,7 @@ widget:
 
 This model takes [xlm-roberta-large](https://huggingface.co/xlm-roberta-large) and fine-tunes it on a combination of NLI data in 15 languages. It is intended to be used for zero-shot text classification, such as with the Hugging Face [ZeroShotClassificationPipeline](https://huggingface.co/transformers/master/main_classes/pipelines.html#transformers.ZeroShotClassificationPipeline).
 
-## Intended Usage
+## Inteded Usage
 
 This model is intended to be used for zero-shot text classification, especially in languages other than English. It is fine-tuned on XNLI, which is a multilingual NLI dataset. The model can therefore be used with any of the languages in the XNLI corpus:
 

model_cards/m3hrdadfi/bert2bert-fa-news-headline/README.md

@@ -1,24 +0,0 @@
----
-language: fa
-license: apache-2.0
-tags:
-- summarization
----
-
-A Bert2Bert model on VoA Persian Corpus (a medium-sized corpus of 7.9 million words, 2003-2008) generates headlines. The model achieved a 25.30 ROUGE-2 score. 
- 
-For more detail, please follow the [News Headline Generation](https://github.com/m3hrdadfi/news-headline-generation) repo. 
-
-
-## Eval results
-The following table summarizes the ROUGE scores obtained by the Bert2Bert model.
-
-|    %    | Precision | Recall | FMeasure |
-|:-------:|:---------:|:------:|:--------:|
-| ROUGE-1 |   43.78   |  45.52 |   43.54  |
-| ROUGE-2 |   24.50   | 25.30* |   24.24  |
-| ROUGE-L |   41.20   |  42.22 |   40.76  |
-
-
-## Questions?
-Post a Github issue on the [News Headline Generation](https://github.com/hooshvare/news-headline-generation/issues) repo.

model_cards/m3hrdadfi/bert2bert-fa-wiki-summary/README.md

@@ -1,24 +0,0 @@
----
-language: fa
-license: apache-2.0
-tags:
-- summarization
----
-
-A Bert2Bert model on the Wiki Summary dataset to summarize articles. The model achieved an 8.47 ROUGE-2 score. 
- 
-For more detail, please follow the [Wiki Summary](https://github.com/m3hrdadfi/wiki-summary) repo. 
-
-
-## Eval results
-The following table summarizes the ROUGE scores obtained by the Bert2Bert model.
-
-|    %    | Precision | Recall | FMeasure |
-|:-------:|:---------:|:------:|:--------:|
-| ROUGE-1 |   28.14   |  30.86 |   27.34  |
-| ROUGE-2 |   07.12   | 08.47* |   07.10  |
-| ROUGE-L |   28.49   |  25.87 |   25.50  |
-
-
-## Questions?
-Post a Github issue on the [Wiki Summary](https://github.com/m3hrdadfi/wiki-summary/issues) repo.

model_cards/pdelobelle/robbert-v2-dutch-base/README.md

@@ -5,7 +5,7 @@ tags:
 - Dutch
 - RoBERTa
 - RobBERT
-license: mit
+license: "MIT"
 datasets:
 - oscar
 - Shuffled Dutch section of the OSCAR corpus (https://oscar-corpus.com/)

model_cards/sagorsarker/codeswitch-hineng-lid-lince/README.md

@@ -4,7 +4,7 @@ language:
 - en
 datasets:
 - lince
-license: mit
+license: "MIT"
 tags:
 - codeswitching
 - hindi-english

model_cards/sagorsarker/codeswitch-hineng-ner-lince/README.md

@@ -4,7 +4,7 @@ language:
 - en
 datasets:
 - lince
-license: mit
+license: "MIT"
 tags:
 - codeswitching
 - hindi-english

model_cards/sagorsarker/codeswitch-hineng-pos-lince/README.md

@@ -4,7 +4,7 @@ language:
 - en
 datasets:
 - lince
-license: mit
+license: "MIT"
 tags:
 - codeswitching
 - hindi-english

model_cards/sagorsarker/codeswitch-nepeng-lid-lince/README.md

@@ -4,7 +4,7 @@ language:
 - en
 datasets:
 - lince
-license: mit
+license: "MIT"
 tags:
 - codeswitching
 - nepali-english

model_cards/sagorsarker/codeswitch-spaeng-lid-lince/README.md

@@ -4,7 +4,7 @@ language:
 - en
 datasets:
 - lince
-license: mit
+license: "MIT"
 tags:
 - codeswitching
 - spanish-english

model_cards/sagorsarker/codeswitch-spaeng-ner-lince/README.md

@@ -4,7 +4,7 @@ language:
 - en
 datasets:
 - lince
-license: mit
+license: "MIT"
 tags:
 - codeswitching
 - spanish-english

model_cards/sagorsarker/codeswitch-spaeng-pos-lince/README.md

@@ -4,7 +4,7 @@ language:
 - en
 datasets:
 - lince
-license: mit
+license: "MIT"
 tags:
 - codeswitching
 - spanish-english

model_cards/sagorsarker/codeswitch-spaeng-sentiment-analysis-lince/README.md

@@ -4,7 +4,7 @@ language:
 - en
 datasets:
 - lince
-license: mit
+license: "MIT"
 tags:
 - codeswitching
 - spanish-english

model_cards/squeezebert/squeezebert-uncased/README.md

@@ -22,7 +22,7 @@ The authors found that SqueezeBERT is 4.3x faster than `bert-base-uncased` on a
 The model is pretrained using the Masked Language Model (MLM) and Sentence Order Prediction (SOP) tasks.
 (Author's note: If you decide to pretrain your own model, and you prefer to train with MLM only, that should work too.)
 
-From the SqueezeBERT paper:
+The SqueezeBERT paper presents 2 approaches to finetuning the model:
 > We pretrain SqueezeBERT from scratch (without distillation) using the [LAMB](https://arxiv.org/abs/1904.00962) optimizer, and we employ the hyperparameters recommended by the LAMB authors: a global batch size of 8192, a learning rate of 2.5e-3, and a warmup proportion of 0.28. Following the LAMB paper's recommendations, we pretrain for 56k steps with a maximum sequence length of 128 and then for 6k steps with a maximum sequence length of 512.
 
 ## Finetuning

model_cards/uer/chinese_roberta_L-2_H-128/README.md

@@ -1,44 +0,0 @@
----
-language: zh
-datasets: 
-- CLUECorpus
----
-
-# Chinese RoBERTa Miniatures
-
-## Model description
-
-This is the set of 24 Chinese RoBERTa models pre-trained by [UER-py](https://www.aclweb.org/anthology/D19-3041.pdf).
-
-You can download the 24 Chinese RoBERTa miniatures either from the [UER-py Github page](https://github.com/dbiir/UER-py/), or via HuggingFace from the links below:
-
-|   |H=128|H=256|H=512|H=768|
-|---|:---:|:---:|:---:|:---:|
-| **L=2**  |[**2/128 (BERT-Tiny)**][2_128]|[2/256]|[2/512]|[2/768]|
-| **L=4**  |[4/128]|[**4/256 (BERT-Mini)**]|[**4/512 (BERT-Small)**]|[4/768]|
-| **L=6**  |[6/128]|[6/256]|[6/512]|[6/768]|
-| **L=8**  |[8/128]|[8/256]|[**8/512 (BERT-Medium)**]|[8/768]|
-| **L=10** |[10/128]|[10/256]|[10/512]|[10/768]|
-| **L=12** |[12/128]|[12/256]|[12/512]|[**12/768 (BERT-Base)**]|
-
-## Training data
-
-CLUECorpus2020 and CLUECorpusSmall are used as training corpus.
-
-## Training procedure
-
-Training details can be found in [UER-py](https://github.com/dbiir/UER-py/).
-
-### BibTeX entry and citation info
-
-```
-@article{zhao2019uer,
-  title={UER: An Open-Source Toolkit for Pre-training Models},
-  author={Zhao, Zhe and Chen, Hui and Zhang, Jinbin and Zhao, Xin and Liu, Tao and Lu, Wei and Chen, Xi and Deng, Haotang and Ju, Qi and Du, Xiaoyong},
-  journal={EMNLP-IJCNLP 2019},
-  pages={241},
-  year={2019}
-}
-```
-
-[2_128]: https://huggingface.co/uer/chinese_roberta_L-2_H-128

model_cards/valhalla/t5-base-e2e-qg/README.md

@@ -5,7 +5,7 @@ tags:
 - question-generation
 widget:
 - text: "Python is a programming language. It is developed by Guido Van Rossum and released in 1991. </s>"
-license: mit
+license: "MIT"
 ---
 
 ## T5 for question-generation

model_cards/valhalla/t5-base-qa-qg-hl/README.md

@@ -6,7 +6,7 @@ tags:
 widget:
 - text: "generate question: <hl> 42 <hl> is the answer to life, the universe and everything. </s>"
 - text: "question: What is 42 context: 42 is the answer to life, the universe and everything. </s>"
-license: mit
+license: "MIT"
 ---
 
 ## T5 for multi-task QA and QG

model_cards/valhalla/t5-base-qg-hl/README.md

@@ -7,7 +7,7 @@ widget:
 - text: "<hl> 42 <hl> is the answer to life, the universe and everything. </s>"
 - text: "Python is a programming language. It is developed by <hl> Guido Van Rossum <hl>. </s>"
 - text: "Although <hl> practicality <hl> beats purity </s>"
-license: mit
+license: "MIT"
 ---
 
 ## T5 for question-generation

model_cards/valhalla/t5-samll-qg-prepend/README.md

@@ -7,7 +7,7 @@ widget:
 - text: "answer: 42  context: 42 is the answer to life, the universe and everything. </s>"
 - text: "answer: Guido Van Rossum context: Python is a programming language. It is developed by Guido Van Rossum. </s>"
 - text: "answer: Explicit context: Explicit is better than implicit </s>"
-license: mit
+license: "MIT"
 ---
 
 ## T5 for question-generation

model_cards/valhalla/t5-small-e2e-qg/README.md

@@ -5,7 +5,7 @@ tags:
 - question-generation
 widget:
 - text: "Python is developed by Guido Van Rossum and released in 1991. </s>"
-license: mit
+license: "MIT"
 ---
 
 ## T5 for question-generation

model_cards/valhalla/t5-small-qa-qg-hl/README.md

@@ -6,7 +6,7 @@ tags:
 widget:
 - text: "generate question: <hl> 42 <hl> is the answer to life, the universe and everything. </s>"
 - text: "question: What is 42 context: 42 is the answer to life, the universe and everything. </s>"
-license: mit
+license: "MIT"
 ---
 
 ## T5 for multi-task QA and QG

model_cards/valhalla/t5-small-qg-hl/README.md

@@ -7,7 +7,7 @@ widget:
 - text: "<hl> 42 <hl> is the answer to life, the universe and everything. </s>"
 - text: "Python is a programming language. It is developed by <hl> Guido Van Rossum <hl>. </s>"
 - text: "Simple is better than <hl> complex <hl>. </s>"
-license: mit
+license: "MIT"
 ---
 
 ## T5 for question-generation

setup.py

@@ -96,15 +96,14 @@ else:
     extras["retrieval"] = ["faiss-cpu", "datasets"]
     extras["flax"] = ["jaxlib==0.1.55", "jax>=0.2.0", "flax==0.2.2"]
 
-extras["tokenizers"] = ["tokenizers==0.9.4"]
+extras["tokenizers"] = ["tokenizers==0.9.2"]
 extras["onnxruntime"] = ["onnxruntime>=1.4.0", "onnxruntime-tools>=1.4.2"]
-extras["modelcreation"] = ["cookiecutter==1.7.2"]
 
 extras["serving"] = ["pydantic", "uvicorn", "fastapi", "starlette"]
 
-extras["sentencepiece"] = ["sentencepiece==0.1.91", "protobuf"]
+extras["sentencepiece"] = ["sentencepiece==0.1.91"]
 extras["retrieval"] = ["faiss-cpu", "datasets"]
-extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil"] + extras["retrieval"] + extras["modelcreation"]
+extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil"] + extras["retrieval"]
 # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
 extras["docs"] = ["recommonmark", "sphinx==3.2.1", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
 extras["quality"] = ["black >= 20.8b1", "isort >= 5.5.4", "flake8 >= 3.8.3"]
@@ -112,12 +111,12 @@ extras["quality"] = ["black >= 20.8b1", "isort >= 5.5.4", "flake8 >= 3.8.3"]
 
 extras["all"] = extras["tf"] + extras["torch"] + extras["flax"] + extras["sentencepiece"] + extras["tokenizers"]
 
-extras["dev"] = extras["all"] + extras["testing"] + extras["quality"] + extras["ja"] + extras["docs"] + extras["sklearn"] + extras["modelcreation"]
+extras["dev"] = extras["all"] + extras["testing"] + extras["quality"] + extras["ja"] + extras["docs"] + extras["sklearn"]
 
 
 setup(
     name="transformers",
-    version="3.5.0",
+    version="3.5.1",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Sylvain Gugger, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
@@ -130,7 +129,7 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.9.4",
+        "tokenizers == 0.9.3",
         # dataclasses for Python versions that don't have it
         "dataclasses;python_version<'3.7'",
         # utilities from PyPA to e.g. compare versions
@@ -143,6 +142,9 @@ setup(
         "tqdm >= 4.27",
         # for OpenAI GPT
         "regex != 2019.12.17",
+        # for SentencePiece models
+        "sentencepiece == 0.1.91",
+        "protobuf",
         # for XLM
         "sacremoses",
     ],

src/transformers/__init__.py

@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-__version__ = "3.5.0"
+__version__ = "3.5.1"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -406,9 +406,6 @@ if is_torch_available():
         DistilBertPreTrainedModel,
     )
     from .modeling_dpr import (
-        DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
-        DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
-        DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
         DPRContextEncoder,
         DPRPretrainedContextEncoder,
         DPRPretrainedQuestionEncoder,
@@ -654,7 +651,6 @@ if is_tf_available():
         TF_MODEL_FOR_CAUSAL_LM_MAPPING,
         TF_MODEL_FOR_MASKED_LM_MAPPING,
         TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
-        TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
         TF_MODEL_FOR_PRETRAINING_MAPPING,
         TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
         TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
@@ -716,17 +712,6 @@ if is_tf_available():
         TFDistilBertModel,
         TFDistilBertPreTrainedModel,
     )
-    from .modeling_tf_dpr import (
-        TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
-        TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
-        TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
-        TFDPRContextEncoder,
-        TFDPRPretrainedContextEncoder,
-        TFDPRPretrainedQuestionEncoder,
-        TFDPRPretrainedReader,
-        TFDPRQuestionEncoder,
-        TFDPRReader,
-    )
     from .modeling_tf_electra import (
         TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFElectraForMaskedLM,

src/transformers/commands/add_new_model.py

@@ -1,192 +0,0 @@
-import json
-import os
-import shutil
-from argparse import ArgumentParser, Namespace
-from pathlib import Path
-from typing import List
-
-from cookiecutter.main import cookiecutter
-from transformers.commands import BaseTransformersCLICommand
-
-from ..utils import logging
-
-
-logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
-
-
-def add_new_model_command_factory(args: Namespace):
-    return AddNewModelCommand(args.testing, args.testing_file, path=args.path)
-
-
-class AddNewModelCommand(BaseTransformersCLICommand):
-    @staticmethod
-    def register_subcommand(parser: ArgumentParser):
-        add_new_model_parser = parser.add_parser("add-new-model")
-        add_new_model_parser.add_argument("--testing", action="store_true", help="If in testing mode.")
-        add_new_model_parser.add_argument("--testing_file", type=str, help="Configuration file on which to run.")
-        add_new_model_parser.add_argument(
-            "--path", type=str, help="Path to cookiecutter. Should only be used for testing purposes."
-        )
-        add_new_model_parser.set_defaults(func=add_new_model_command_factory)
-
-    def __init__(self, testing: bool, testing_file: str, path=None, *args):
-        self._testing = testing
-        self._testing_file = testing_file
-        self._path = path
-
-    def run(self):
-        # Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
-        directories = [directory for directory in os.listdir() if "cookiecutter-template-" == directory[:22]]
-        if len(directories) > 0:
-            raise ValueError(
-                "Several directories starting with `cookiecutter-template-` in current working directory. "
-                "Please clean your directory by removing all folders startign with `cookiecutter-template-` or "
-                "change your working directory."
-            )
-
-        path_to_transformer_root = (
-            Path(__file__).parent.parent.parent.parent if self._path is None else Path(self._path).parent.parent
-        )
-        path_to_cookiecutter = path_to_transformer_root / "templates" / "cookiecutter"
-
-        # Execute cookiecutter
-        if not self._testing:
-            cookiecutter(str(path_to_cookiecutter))
-        else:
-            with open(self._testing_file, "r") as configuration_file:
-                testing_configuration = json.load(configuration_file)
-
-            cookiecutter(
-                str(path_to_cookiecutter if self._path is None else self._path),
-                no_input=True,
-                extra_context=testing_configuration,
-            )
-
-        directory = [directory for directory in os.listdir() if "cookiecutter-template-" in directory[:22]][0]
-
-        # Retrieve configuration
-        with open(directory + "/configuration.json", "r") as configuration_file:
-            configuration = json.load(configuration_file)
-
-        lowercase_model_name = configuration["lowercase_modelname"]
-        pytorch_or_tensorflow = configuration["generate_tensorflow_and_pytorch"]
-        os.remove(f"{directory}/configuration.json")
-
-        output_pytorch = "PyTorch" in pytorch_or_tensorflow
-        output_tensorflow = "TensorFlow" in pytorch_or_tensorflow
-
-        shutil.move(
-            f"{directory}/configuration_{lowercase_model_name}.py",
-            f"{path_to_transformer_root}/src/transformers/configuration_{lowercase_model_name}.py",
-        )
-
-        def remove_copy_lines(path):
-            with open(path, "r") as f:
-                lines = f.readlines()
-            with open(path, "w") as f:
-                for line in lines:
-                    if "# Copied from transformers." not in line:
-                        f.write(line)
-
-        if output_pytorch:
-            if not self._testing:
-                remove_copy_lines(f"{directory}/modeling_{lowercase_model_name}.py")
-
-            shutil.move(
-                f"{directory}/modeling_{lowercase_model_name}.py",
-                f"{path_to_transformer_root}/src/transformers/modeling_{lowercase_model_name}.py",
-            )
-
-            shutil.move(
-                f"{directory}/test_modeling_{lowercase_model_name}.py",
-                f"{path_to_transformer_root}/tests/test_modeling_{lowercase_model_name}.py",
-            )
-        else:
-            os.remove(f"{directory}/modeling_{lowercase_model_name}.py")
-            os.remove(f"{directory}/test_modeling_{lowercase_model_name}.py")
-
-        if output_tensorflow:
-            if not self._testing:
-                remove_copy_lines(f"{directory}/modeling_tf_{lowercase_model_name}.py")
-
-            shutil.move(
-                f"{directory}/modeling_tf_{lowercase_model_name}.py",
-                f"{path_to_transformer_root}/src/transformers/modeling_tf_{lowercase_model_name}.py",
-            )
-
-            shutil.move(
-                f"{directory}/test_modeling_tf_{lowercase_model_name}.py",
-                f"{path_to_transformer_root}/tests/test_modeling_tf_{lowercase_model_name}.py",
-            )
-        else:
-            os.remove(f"{directory}/modeling_tf_{lowercase_model_name}.py")
-            os.remove(f"{directory}/test_modeling_tf_{lowercase_model_name}.py")
-
-        shutil.move(
-            f"{directory}/{lowercase_model_name}.rst",
-            f"{path_to_transformer_root}/docs/source/model_doc/{lowercase_model_name}.rst",
-        )
-
-        shutil.move(
-            f"{directory}/tokenization_{lowercase_model_name}.py",
-            f"{path_to_transformer_root}/src/transformers/tokenization_{lowercase_model_name}.py",
-        )
-
-        from os import fdopen, remove
-        from shutil import copymode, move
-        from tempfile import mkstemp
-
-        def replace(original_file: str, line_to_copy_below: str, lines_to_copy: List[str]):
-            # Create temp file
-            fh, abs_path = mkstemp()
-            line_found = False
-            with fdopen(fh, "w") as new_file:
-                with open(original_file) as old_file:
-                    for line in old_file:
-                        new_file.write(line)
-                        if line_to_copy_below in line:
-                            line_found = True
-                            for line_to_copy in lines_to_copy:
-                                new_file.write(line_to_copy)
-
-            if not line_found:
-                raise ValueError(f"Line {line_to_copy_below} was not found in file.")
-
-            # Copy the file permissions from the old file to the new file
-            copymode(original_file, abs_path)
-            # Remove original file
-            remove(original_file)
-            # Move new file
-            move(abs_path, original_file)
-
-        def skip_units(line):
-            return ("generating PyTorch" in line and not output_pytorch) or (
-                "generating TensorFlow" in line and not output_tensorflow
-            )
-
-        def replace_in_files(path_to_datafile):
-            with open(path_to_datafile) as datafile:
-                lines_to_copy = []
-                skip_file = False
-                skip_snippet = False
-                for line in datafile:
-                    if "# To replace in: " in line and "##" not in line:
-                        file_to_replace_in = line.split('"')[1]
-                        skip_file = skip_units(line)
-                    elif "# Below: " in line and "##" not in line:
-                        line_to_copy_below = line.split('"')[1]
-                        skip_snippet = skip_units(line)
-                    elif "# End." in line and "##" not in line:
-                        if not skip_file and not skip_snippet:
-                            replace(file_to_replace_in, line_to_copy_below, lines_to_copy)
-
-                        lines_to_copy = []
-                    elif "# Replace with" in line and "##" not in line:
-                        lines_to_copy = []
-                    elif "##" not in line:
-                        lines_to_copy.append(line)
-
-            remove(path_to_datafile)
-
-        replace_in_files(f"{directory}/to_replace_{lowercase_model_name}.py")
-        os.rmdir(directory)

src/transformers/commands/transformers_cli.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 from argparse import ArgumentParser
 
-from transformers.commands.add_new_model import AddNewModelCommand
 from transformers.commands.convert import ConvertCommand
 from transformers.commands.download import DownloadCommand
 from transformers.commands.env import EnvironmentCommand
@@ -21,7 +20,6 @@ def main():
     RunCommand.register_subcommand(commands_parser)
     ServeCommand.register_subcommand(commands_parser)
     UserCommands.register_subcommand(commands_parser)
-    AddNewModelCommand.register_subcommand(commands_parser)
 
     # Let's go
     args = parser.parse_args()