Release: v3.5.1

Fix typo
2020-11-13 10:03:08 -05:00 · 2020-11-13 10:00:29 -05:00
597 changed files with 6353 additions and 12501 deletions
@@ -281,7 +281,7 @@ jobs:
                      - v0.4-build_doc-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
-            - run: pip install ."[all, docs]"
+            - run: pip install .[tf,torch,sentencepiece,docs]
            - save_cache:
                  key: v0.4-build_doc-{{ checksum "setup.py" }}
                  paths:
@@ -303,7 +303,7 @@ jobs:
                  keys:
                      - v0.4-deploy_doc-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
-            - run: pip install ."[all,docs]"
+            - run: pip install .[tf,torch,sentencepiece,docs]
            - save_cache:
                  key: v0.4-deploy_doc-{{ checksum "setup.py" }}
                  paths:
@@ -51,5 +51,4 @@ deploy_doc "7fb8bdf" v3.0.2
 deploy_doc "4b3ee9c" v3.1.0
 deploy_doc "3ebb1b3" v3.2.0
 deploy_doc "0613f05" v3.3.1
-deploy_doc "eb0e0ce" v3.4.0
-deploy_doc "818878d" # v3.5.1 Latest stable release
+deploy_doc "eb0e0ce" # v3.4.0 Latest stable release
@@ -1,6 +1,6 @@
 ---
 name: "❓ Questions & Help"
-about: Post your general questions on the Hugging Face forum: https://discuss.huggingface.co/
+about: Post your general questions on the Hugging Face forum or Stack Overflow tagged huggingface-transformers
 title: ''
 labels: ''
 assignees: ''
@@ -10,17 +10,18 @@ assignees: ''
 # ❓ Questions & Help

 <!-- The GitHub issue tracker is primarly intended for bugs, feature requests,
-     new models, benchmarks, and migration questions. For all other questions,
+     new models and benchmarks, and migration questions. For all other questions,
     we direct you to the Hugging Face forum: https://discuss.huggingface.co/ .
+     You can also try Stack Overflow (SO) where a whole community of PyTorch and
+     Tensorflow enthusiast can help you out. In this case, make sure to tag your
+     question with the right deep learning framework as well as the
+     huggingface-transformers tag: 
+     https://stackoverflow.com/questions/tagged/huggingface-transformers 
     -->

 ## Details
-
 <!-- Description of your issue -->

-<!-- You should first ask your question on the forum, and only if
-     you didn't get an answer after a few days ask it here on GitHub. -->
-
-**A link to original question on the forum**:
-
-<!-- Your issue will be closed if you don't fill this part. -->
+<!-- You should first ask your question on the forum or SO, and only if
+     you didn't get an answer ask it here on GitHub. -->
+**A link to original question on the forum/Stack Overflow**:
@@ -8,9 +8,6 @@ on:
 jobs:
  torch_hub_integration:
    runs-on: ubuntu-latest
-    env:
-      # TODO quickfix but may need more investigation
-      ACTIONS_ALLOW_UNSECURE_COMMANDS: True
    steps:
    # no checkout necessary here.
    - name: Extract branch name
@@ -4,19 +4,17 @@ on:
  push:
    branches:
      - master
-      - model-templates
    paths:
      - "src/**"
      - "tests/**"
      - ".github/**"
-      - "templates/**"
  # pull_request:
  repository_dispatch:


 jobs:
  run_tests_torch_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2
      - name: Python version
@@ -57,14 +55,6 @@ jobs:
          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"

-      - name: Create model files
-        run: |
-          source .env/bin/activate
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-
      - name: Run all non-slow tests on GPU
        env:
          OMP_NUM_THREADS: 1
@@ -86,7 +76,7 @@ jobs:
                  

  run_tests_tf_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2
      - name: Python version
@@ -126,14 +116,6 @@ jobs:
          TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
          TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"

-      - name: Create model files
-        run: |
-          source .env/bin/activate
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-
      - name: Run all non-slow tests on GPU
        env:
          OMP_NUM_THREADS: 1
@@ -153,8 +135,8 @@ jobs:
          name: run_all_tests_tf_gpu_test_reports
          path: reports

-  run_tests_torch_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+  run_tests_torch_multiple_gpu:
+    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2
      - name: Python version
@@ -172,7 +154,7 @@ jobs:
        id: cache
        with:
          path: .env
-          key: v1.1-tests_torch_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v1.1-tests_torch_multiple_gpu-${{ hashFiles('setup.py') }}

      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
        run: |
@@ -199,11 +181,11 @@ jobs:
          OMP_NUM_THREADS: 1
        run: |
          source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_multi_gpu tests
+          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_multiple_gpu tests

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_torch_multi_gpu_failures_short.txt          
+        run: cat reports/tests_torch_multiple_gpu_failures_short.txt          

      - name: Test suite reports artifacts
        if: ${{ always() }}
@@ -212,8 +194,8 @@ jobs:
          name: run_all_tests_torch_multi_gpu_test_reports
          path: reports

-  run_tests_tf_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+  run_tests_tf_multiple_gpu:
+    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2
      - name: Python version
@@ -231,7 +213,7 @@ jobs:
        id: cache
        with:
          path: .env
-          key: v1.1-tests_tf_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v1.1-tests_tf_multiple_gpu-${{ hashFiles('setup.py') }}

      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
        run: |
@@ -258,11 +240,11 @@ jobs:
          OMP_NUM_THREADS: 1
        run: |
          source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_multi_gpu tests
+          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_multiple_gpu tests

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_failures_short.txt
+        run: cat reports/tests_tf_multiple_gpu_failures_short.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
@@ -9,14 +9,13 @@ on:
  push:
    branches:
      - ci_*
-      - framework-agnostic-tokenizers
  repository_dispatch:
  schedule:
    - cron: "0 0 * * *"

 jobs:
  run_all_tests_torch_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2

@@ -110,7 +109,7 @@ jobs:


  run_all_tests_tf_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2

@@ -188,8 +187,8 @@ jobs:
          name: run_all_tests_tf_gpu_test_reports
          path: reports
          
-  run_all_tests_torch_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+  run_all_tests_torch_multiple_gpu:
+    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2

@@ -239,11 +238,11 @@ jobs:
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_multiple_gpu tests

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_torch_multi_gpu_failures_short.txt
+        run: cat reports/tests_torch_multiple_gpu_failures_short.txt

      - name: Run examples tests on multi-GPU
        env:
@@ -251,11 +250,11 @@ jobs:
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_multi_gpu examples
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_multiple_gpu examples

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/examples_torch_multi_gpu_failures_short.txt
+        run: cat reports/examples_torch_multiple_gpu_failures_short.txt

      - name: Run all pipeline tests on multi-GPU
        if: ${{ always() }}
@@ -266,11 +265,11 @@ jobs:
          RUN_PIPELINE_TESTS: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_multiple_gpu tests

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_torch_pipeline_multi_gpu_failures_short.txt
+        run: cat reports/tests_torch_pipeline_multiple_gpu_failures_short.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
@@ -279,8 +278,8 @@ jobs:
          name: run_all_tests_torch_multi_gpu_test_reports
          path: reports

-  run_all_tests_tf_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+  run_all_tests_tf_multiple_gpu:
+    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2

@@ -330,11 +329,11 @@ jobs:
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_multiple_gpu tests

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_failures_short.txt
+        run: cat reports/tests_tf_multiple_gpu_failures_short.txt

      - name: Run all pipeline tests on multi-GPU
        if: ${{ always() }}
@@ -345,11 +344,11 @@ jobs:
          RUN_PIPELINE_TESTS: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multiple_gpu tests
          
      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_pipelines_failures_short.txt
+        run: cat reports/tests_tf_multiple_gpu_pipelines_failures_short.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
@@ -133,6 +133,7 @@ dmypy.json
 tensorflow_code

 # Models
+models
 proc_data

 # examples
@@ -308,12 +308,3 @@ Check our [documentation writing guide](https://github.com/huggingface/transform
 for more information.

 #### This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/master/CONTRIBUTING.md)
-
-
-### Develop on Windows
-
-One way one can run the make command on Window is to pass by MSYS2:
-
-1. [Download MSYS2](https://www.msys2.org/), we assume to have it installed in C:\msys64
-2. Open the command line C:\msys64\msys2.exe (it should be available from the start menu)
-3. Run in the shell: `pacman -Syu` and install make with `pacman -S make`
@@ -181,7 +181,6 @@ Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
 1. **[LXMERT](https://huggingface.co/transformers/model_doc/lxmert.html)** (from UNC Chapel Hill) released with the paper [LXMERT: Learning Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal.
 1. **[MarianMT](https://huggingface.co/transformers/model_doc/marian.html)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team.
 1. **[MBart](https://huggingface.co/transformers/model_doc/mbart.html)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-1. **[MT5](https://huggingface.co/transformers/model_doc/mt5.html)** (from Google AI) released with the paper [mT5: A massively multilingual pre-trained text-to-text transformer](https://arxiv.org/abs/2010.11934) by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
 1. **[Pegasus](https://huggingface.co/transformers/model_doc/pegasus.html)** (from Google) released with the paper [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization](https://arxiv.org/abs/1912.08777)> by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
 1. **[ProphetNet](https://huggingface.co/transformers/model_doc/prophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
 1. **[Reformer](https://huggingface.co/transformers/model_doc/reformer.html)** (from Google Research) released with the paper [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
@@ -1,11 +1,10 @@
 // These two things need to be updated at each release for the version selector.
 // Last stable version
-const stableVersion = "v3.5.0"
+const stableVersion = "v3.4.0"
 // Dictionary doc folder to label
 const versionMapping = {
    "master": "master",
-    "": "v3.5.0/v3.5.1",
-    "v3.4.0": "v3.4.0",
+    "": "v3.4.0",
    "v3.3.1": "v3.3.0/v3.3.1",
    "v3.2.0": "v3.2.0",
    "v3.1.0": "v3.1.0 (stable)",
@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'3.5.0'
+release = u'3.5.1'


 # -- General configuration ---------------------------------------------------
@@ -126,44 +126,41 @@ conversion utilities for the following models:
 21. :doc:`MBart <model_doc/mbart>` (from Facebook) released with the paper `Multilingual Denoising Pre-training for
    Neural Machine Translation <https://arxiv.org/abs/2001.08210>`__ by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li,
    Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-22. :doc:`MT5 <model_doc/mt5>` (from Google AI) released with the paper `mT5: A massively multilingual pre-trained
-    text-to-text transformer <https://arxiv.org/abs/2010.11934>`__ by Linting Xue, Noah Constant, Adam Roberts, Mihir
-    Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
-23. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
+22. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
    Gap-sentences for Abstractive Summarization <https://arxiv.org/abs/1912.08777>`__> by Jingqing Zhang, Yao Zhao,
    Mohammad Saleh and Peter J. Liu.
-24. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
+23. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
    Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi,
    Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-25. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
+24. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
    Transformer <https://arxiv.org/abs/2001.04451>`__ by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
-26. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
+25. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
    Pretraining Approach <https://arxiv.org/abs/1907.11692>`__ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar
    Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. ultilingual BERT into `DistilmBERT
    <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German version of
    DistilBERT.
-27. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
+26. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
    about efficient neural networks? <https://arxiv.org/abs/2006.11316>`__ by Forrest N. Iandola, Albert E. Shaw, Ravi
    Krishna, and Kurt W. Keutzer.
-28. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
+27. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
    Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by Colin Raffel and Noam Shazeer and Adam
    Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
-29. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
+28. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
    Attentive Language Models Beyond a Fixed-Length Context <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*,
    Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
-30. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
+29. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
    Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis Conneau.
-31. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
+30. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
    Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan,
    Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-32. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
+31. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
    Cross-lingual Representation Learning at Scale <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay
    Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke
    Zettlemoyer and Veselin Stoyanov.
-33. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `XLNet: Generalized Autoregressive
+32. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `XLNet: Generalized Autoregressive
    Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by Zhilin Yang*, Zihang Dai*, Yiming
    Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
-34. `Other community models <https://huggingface.co/models>`__, contributed by the `community
+33. `Other community models <https://huggingface.co/models>`__, contributed by the `community
    <https://huggingface.co/users>`__.

 .. toctree::
@@ -251,7 +248,6 @@ conversion utilities for the following models:
    model_doc/marian
    model_doc/mbart
    model_doc/mobilebert
-    model_doc/mt5
    model_doc/gpt
    model_doc/gpt2
    model_doc/pegasus
@@ -35,7 +35,7 @@ Here is an example of how to customize :class:`~transformers.Trainer` using a cu
    class MyTrainer(Trainer):
        def compute_loss(self, model, inputs):
            labels = inputs.pop("labels")
-            outputs = model(**inputs)
+            outputs = models(**inputs)
            logits = outputs[0]
            return my_custom_loss(logits, labels)

@@ -51,10 +51,10 @@ AlbertTokenizer
 Albert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.albert.modeling_albert.AlbertForPreTrainingOutput
+.. autoclass:: transformers.modeling_albert.AlbertForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.albert.modeling_tf_albert.TFAlbertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_albert.TFAlbertForPreTrainingOutput
    :members:


@@ -34,8 +34,6 @@ ________________________________________________________________________________
 - An example of how to train :class:`~transformers.BartForConditionalGeneration` with a Hugging Face :obj:`datasets`
  object can be found in this `forum discussion
  <https://discuss.huggingface.co/t/train-bart-for-conditional-generation-e-g-summarization/1904>`__.
- `Distilled checkpoints <https://huggingface.co/models?search=distilbart>`__ are described in this `paper
-  <https://arxiv.org/abs/2010.13002>`__.


 Implementation Notes
@@ -44,33 +42,16 @@ Implementation Notes
 - Bart doesn't use :obj:`token_type_ids` for sequence classification. Use :class:`~transformers.BartTokenizer` or
  :meth:`~transformers.BartTokenizer.encode` to get the proper splitting.
 - The forward pass of :class:`~transformers.BartModel` will create decoder inputs (using the helper function
-  :func:`transformers.models.bart.modeling_bart._prepare_bart_decoder_inputs`) if they are not passed. This is
-  different than some other modeling APIs.
- Model predictions are intended to be identical to the original implementation when
-  :obj:`force_bos_token_to_be_generated=True`. This only works, however, if the string you pass to
-  :func:`fairseq.encode` starts with a space.
+  :func:`transformers.modeling_bart._prepare_bart_decoder_inputs`) if they are not passed. This is different than some
+  other modeling APIs.
+- Model predictions are intended to be identical to the original implementation. This only works, however, if the
+  string you pass to :func:`fairseq.encode` starts with a space.
 - :meth:`~transformers.BartForConditionalGeneration.generate` should be used for conditional generation tasks like
  summarization, see the example in that docstrings.
 - Models that load the `facebook/bart-large-cnn` weights will not have a :obj:`mask_token_id`, or be able to perform
  mask-filling tasks.
 - For training/forward passes that don't involve beam search, pass :obj:`use_cache=False`.

-Mask Filling
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The :obj:`facebook/bart-base` and :obj:`facebook/bart-large` checkpoints can be used to fill multi-token masks.
-
-.. code-block::
-
-    from transformers import BartForConditionalGeneration, BartTokenizer
-    model = BartForConditionalGeneration.from_pretrained("facebook/bart-large", force_bos_token_to_be_generated=True)
-    tok = BartTokenizer.from_pretrained("facebook/bart-large")
-    example_english_phrase = "UN Chief Says There Is No <mask> in Syria"
-    batch = tok(example_english_phrase, return_tensors='pt')
-    generated_ids = model.generate(batch['input_ids'])
-    assert tok.batch_decode(generated_ids, skip_special_tokens=True) == ['UN Chief Says There Is No Plan to Stop Chemical Weapons in Syria']
-
-

 BartConfig
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -93,7 +74,7 @@ BartModel
 .. autoclass:: transformers.BartModel
    :members: forward

-.. autofunction:: transformers.models.bart.modeling_bart._prepare_bart_decoder_inputs
+.. autofunction:: transformers.modeling_bart._prepare_bart_decoder_inputs


 BartForConditionalGeneration
@@ -57,10 +57,10 @@ BertTokenizerFast
 Bert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.bert.modeling_bert.BertForPreTrainingOutput
+.. autoclass:: transformers.modeling_bert.BertForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.bert.modeling_tf_bert.TFBertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_bert.TFBertForPreTrainingOutput
    :members:


@@ -188,10 +188,3 @@ TFBertForQuestionAnswering

 .. autoclass:: transformers.TFBertForQuestionAnswering
    :members: call
-
-
-FlaxBertModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxBertModel
-    :members: __call__
@@ -40,7 +40,7 @@ Usage:
  labels = tokenizer('This is a short summary', return_tensors="pt").input_ids

  # train...
-  loss = bert2bert(input_ids=input_ids, decoder_input_ids=labels, labels=labels).loss
+  loss = bert2bert(input_ids=input_ids, decoder_input_ids=labels, labels=labels, return_dict=True).loss
  loss.backward()


@@ -71,13 +71,13 @@ DPRReaderTokenizerFast
 DPR specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.dpr.modeling_dpr.DPRContextEncoderOutput
+.. autoclass:: transformers.modeling_dpr.DPRContextEncoderOutput
    :members:

-.. autoclass:: transformers.models.dpr.modeling_dpr.DPRQuestionEncoderOutput
+.. autoclass:: transformers.modeling_dpr.DPRQuestionEncoderOutput
    :members:

-.. autoclass:: transformers.models.dpr.modeling_dpr.DPRReaderOutput
+.. autoclass:: transformers.modeling_dpr.DPRReaderOutput
    :members:


@@ -99,22 +99,3 @@ DPRReader

 .. autoclass:: transformers.DPRReader
    :members: forward
-
-TFDPRContextEncoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFDPRContextEncoder
-    :members: call
-
-TFDPRQuestionEncoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFDPRQuestionEncoder
-    :members: call
-
-
-TFDPRReader
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFDPRReader
-    :members: call
@@ -69,10 +69,10 @@ ElectraTokenizerFast
 Electra specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput
+.. autoclass:: transformers.modeling_electra.ElectraForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.electra.modeling_tf_electra.TFElectraForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_electra.TFElectraForPreTrainingOutput
    :members:


@@ -65,10 +65,10 @@ FunnelTokenizerFast
 Funnel specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.funnel.modeling_funnel.FunnelForPreTrainingOutput
+.. autoclass:: transformers.modeling_funnel.FunnelForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.funnel.modeling_tf_funnel.TFFunnelForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_funnel.TFFunnelForPreTrainingOutput
    :members:


@@ -72,10 +72,10 @@ OpenAIGPTTokenizerFast
 OpenAI specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.openai.modeling_openai.OpenAIGPTDoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_openai.OpenAIGPTDoubleHeadsModelOutput
    :members:

-.. autoclass:: transformers.models.openai.modeling_tf_openai.TFOpenAIGPTDoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_tf_openai.TFOpenAIGPTDoubleHeadsModelOutput
    :members:


@@ -60,10 +60,10 @@ GPT2TokenizerFast
 GPT2 specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.gpt2.modeling_gpt2.GPT2DoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_gpt2.GPT2DoubleHeadsModelOutput
    :members:

-.. autoclass:: transformers.models.gpt2.modeling_tf_gpt2.TFGPT2DoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_tf_gpt2.TFGPT2DoubleHeadsModelOutput
    :members:


@@ -93,27 +93,29 @@ LongformerTokenizerFast
 Longformer specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerBaseModelOutput
+.. autoclass:: transformers.modeling_longformer.LongformerBaseModelOutput
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerBaseModelOutputWithPooling
+.. autoclass:: transformers.modeling_longformer.LongformerBaseModelOutputWithPooling
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMultipleChoiceModelOutput
+.. autoclass:: transformers.modeling_longformer.LongformerMultipleChoiceModelOutput
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerQuestionAnsweringModelOutput
+.. autoclass:: transformers.modeling_longformer.LongformerQuestionAnsweringModelOutput
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutput
+.. autoclass:: transformers.modeling_tf_longformer.TFLongformerBaseModelOutput
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutputWithPooling
+.. autoclass:: transformers.modeling_tf_longformer.TFLongformerBaseModelOutputWithPooling
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerQuestionAnsweringModelOutput
+.. autoclass:: transformers.modeling_tf_longformer.TFLongformerQuestionAnsweringModelOutput
    :members: 

+LongformerModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 LongformerModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -67,19 +67,19 @@ LxmertTokenizerFast
 Lxmert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.lxmert.modeling_lxmert.LxmertModelOutput
+.. autoclass:: transformers.modeling_lxmert.LxmertModelOutput
    :members:

-.. autoclass:: transformers.models.lxmert.modeling_lxmert.LxmertForPreTrainingOutput
+.. autoclass:: transformers.modeling_lxmert.LxmertForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.lxmert.modeling_lxmert.LxmertForQuestionAnsweringOutput
+.. autoclass:: transformers.modeling_lxmert.LxmertForQuestionAnsweringOutput
    :members:

-.. autoclass:: transformers.models.lxmert.modeling_tf_lxmert.TFLxmertModelOutput
+.. autoclass:: transformers.modeling_tf_lxmert.TFLxmertModelOutput
    :members:

-.. autoclass:: transformers.models.lxmert.modeling_tf_lxmert.TFLxmertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_lxmert.TFLxmertForPreTrainingOutput
    :members:


@@ -5,7 +5,7 @@ MarianMT
 <https://github.com/huggingface/transformers/issues/new?assignees=sshleifer&labels=&template=bug-report.md&title>`__
 and assign @patrickvonplaten.

-Translations should be similar, but not identical to output in the test set linked to in each model card.
+Translations should be similar, but not identical to, output in the test set linked to in each model card.

 Implementation Notes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -35,109 +35,24 @@ Naming
  <https://developers.google.com/admin-sdk/directory/v1/languages>`__, three digit codes require googling "language
  code {code}".
 - Codes formatted like :obj:`es_AR` are usually :obj:`code_{region}`. That one is Spanish from Argentina.
- The models were converted in two stages. The first 1000 models use ISO-639-2 codes to identify languages, the second
-  group use a combination of ISO-639-5 codes and ISO-639-2 codes.


-Examples
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
- Since Marian models are smaller than many other translation models available in the library, they can be useful for
-  fine-tuning experiments and integration tests.
- `Fine-tune on TPU
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/builtin_trainer/train_distil_marian_enro_tpu.sh>`__
- `Fine-tune on GPU
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/builtin_trainer/train_distil_marian_enro.sh>`__
- `Fine-tune on GPU with pytorch-lightning
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/distil_marian_no_teacher.sh>`__
-
 Multilingual Models
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

- All model names use the following format: :obj:`Helsinki-NLP/opus-mt-{src}-{tgt}`:
- If a model can output multiple languages, and you should specify a language code by prepending the desired output
-  language to the :obj:`src_text`.
- You can see a models's supported language codes in its model card, under target constituents, like in `opus-mt-en-roa
-  <https://huggingface.co/Helsinki-NLP/opus-mt-en-roa>`__.
- Note that if a model is only multilingual on the source side, like :obj:`Helsinki-NLP/opus-mt-roa-en`, no language
-  codes are required.
+All model names use the following format: :obj:`Helsinki-NLP/opus-mt-{src}-{tgt}`:

-New multi-lingual models from the `Tatoeba-Challenge repo <https://github.com/Helsinki-NLP/Tatoeba-Challenge>`__
-require 3 character language codes:
+    - If :obj:`src` is in all caps, the model supports multiple input languages, you can figure out which ones by
+      looking at the model card, or the Group Members `mapping
+      <https://gist.github.com/sshleifer/6d20e7761931b08e73c3219027b97b8a>`_ .
+    - If :obj:`tgt` is in all caps, the model can output multiple languages, and you should specify a language code by
+      prepending the desired output language to the :obj:`src_text`.
+    - You can see a tokenizer's supported language codes in ``tokenizer.supported_language_codes``
+
+Example of translating english to many romance languages, using language codes:

 .. code-block:: python

-    from transformers import MarianMTModel, MarianTokenizer
-    src_text = [
-        '>>fra<< this is a sentence in english that we want to translate to french',
-        '>>por<< This should go to portuguese',
-        '>>esp<< And this to Spanish'
-    ]
-
-    model_name = 'Helsinki-NLP/opus-mt-en-roa'
-    tokenizer = MarianTokenizer.from_pretrained(model_name)
-    print(tokenizer.supported_language_codes)
-    model = MarianMTModel.from_pretrained(model_name)
-    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text, return_tensors="pt"))
-    tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
-    # ["c'est une phrase en anglais que nous voulons traduire en français",
-    # 'Isto deve ir para o português.',
-    # 'Y esto al español']
-
-
-
-
-Code to see available pretrained models:
-
-.. code-block:: python
-
-    from transformers.hf_api import HfApi
-    model_list = HfApi().model_list()
-    org = "Helsinki-NLP"
-    model_ids = [x.modelId for x in model_list if x.modelId.startswith(org)]
-    suffix = [x.split('/')[1] for x in model_ids]
-    old_style_multi_models = [f'{org}/{s}' for s in suffix if s != s.lower()]
-
-
-
-Old Style Multi-Lingual Models
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-These are the old style multi-lingual models ported from the OPUS-MT-Train repo: and the members of each language
-group:
-
-.. code-block:: python
-
-    ['Helsinki-NLP/opus-mt-NORTH_EU-NORTH_EU',
-     'Helsinki-NLP/opus-mt-ROMANCE-en',
-     'Helsinki-NLP/opus-mt-SCANDINAVIA-SCANDINAVIA',
-     'Helsinki-NLP/opus-mt-de-ZH',
-     'Helsinki-NLP/opus-mt-en-CELTIC',
-     'Helsinki-NLP/opus-mt-en-ROMANCE',
-     'Helsinki-NLP/opus-mt-es-NORWAY',
-     'Helsinki-NLP/opus-mt-fi-NORWAY',
-     'Helsinki-NLP/opus-mt-fi-ZH',
-     'Helsinki-NLP/opus-mt-fi_nb_no_nn_ru_sv_en-SAMI',
-     'Helsinki-NLP/opus-mt-sv-NORWAY',
-     'Helsinki-NLP/opus-mt-sv-ZH']
-    GROUP_MEMBERS = {
-     'ZH': ['cmn', 'cn', 'yue', 'ze_zh', 'zh_cn', 'zh_CN', 'zh_HK', 'zh_tw', 'zh_TW', 'zh_yue', 'zhs', 'zht', 'zh'],
-     'ROMANCE': ['fr', 'fr_BE', 'fr_CA', 'fr_FR', 'wa', 'frp', 'oc', 'ca', 'rm', 'lld', 'fur', 'lij', 'lmo', 'es', 'es_AR', 'es_CL', 'es_CO', 'es_CR', 'es_DO', 'es_EC', 'es_ES', 'es_GT', 'es_HN', 'es_MX', 'es_NI', 'es_PA', 'es_PE', 'es_PR', 'es_SV', 'es_UY', 'es_VE', 'pt', 'pt_br', 'pt_BR', 'pt_PT', 'gl', 'lad', 'an', 'mwl', 'it', 'it_IT', 'co', 'nap', 'scn', 'vec', 'sc', 'ro', 'la'],
-     'NORTH_EU': ['de', 'nl', 'fy', 'af', 'da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
-     'SCANDINAVIA': ['da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
-     'SAMI': ['se', 'sma', 'smj', 'smn', 'sms'],
-     'NORWAY': ['nb_NO', 'nb', 'nn_NO', 'nn', 'nog', 'no_nb', 'no'],
-     'CELTIC': ['ga', 'cy', 'br', 'gd', 'kw', 'gv']
-    }
-
-
-
-
-Example of translating english to many romance languages, using old-style 2 character language codes
-
-
-.. code-block::python
-
    from transformers import MarianMTModel, MarianTokenizer
    src_text = [
        '>>fr<< this is a sentence in english that we want to translate to french',
@@ -148,12 +63,52 @@ Example of translating english to many romance languages, using old-style 2 char
    model_name = 'Helsinki-NLP/opus-mt-en-ROMANCE'
    tokenizer = MarianTokenizer.from_pretrained(model_name)
    print(tokenizer.supported_language_codes)
-
    model = MarianMTModel.from_pretrained(model_name)
-    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text, return_tensors="pt"))
+    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text))
    tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
-    # ["c'est une phrase en anglais que nous voulons traduire en français", 'Isto deve ir para o português.',  'Y esto al español']
+    # ["c'est une phrase en anglais que nous voulons traduire en français",
+    # 'Isto deve ir para o português.',
+    # 'Y esto al español']

+Sometimes, models were trained on collections of languages that do not resolve to a group. In this case, _ is used as a
+separator for src or tgt, as in :obj:`Helsinki-NLP/opus-mt-en_el_es_fi-en_el_es_fi`. These still require language
+codes.
+
+There are many supported regional language codes, like :obj:`>>es_ES<<` (Spain) and :obj:`>>es_AR<<` (Argentina), that
+do not seem to change translations. I have not found these to provide different results than just using :obj:`>>es<<`.
+
+For example:
+
+    - `Helsinki-NLP/opus-mt-NORTH_EU-NORTH_EU`: translates from all NORTH_EU languages (see `mapping
+      <https://gist.github.com/sshleifer/6d20e7761931b08e73c3219027b97b8a>`_) to all NORTH_EU languages. Use a special
+      language code like :obj:`>>de<<` to specify output language.
+    - `Helsinki-NLP/opus-mt-ROMANCE-en`: translates from many romance languages to english, no codes needed since there
+      is only one target language.
+
+
+
+.. code-block:: python
+
+    GROUP_MEMBERS = {
+     'ZH': ['cmn', 'cn', 'yue', 'ze_zh', 'zh_cn', 'zh_CN', 'zh_HK', 'zh_tw', 'zh_TW', 'zh_yue', 'zhs', 'zht', 'zh'],
+     'ROMANCE': ['fr', 'fr_BE', 'fr_CA', 'fr_FR', 'wa', 'frp', 'oc', 'ca', 'rm', 'lld', 'fur', 'lij', 'lmo', 'es', 'es_AR', 'es_CL', 'es_CO', 'es_CR', 'es_DO', 'es_EC', 'es_ES', 'es_GT', 'es_HN', 'es_MX', 'es_NI', 'es_PA', 'es_PE', 'es_PR', 'es_SV', 'es_UY', 'es_VE', 'pt', 'pt_br', 'pt_BR', 'pt_PT', 'gl', 'lad', 'an', 'mwl', 'it', 'it_IT', 'co', 'nap', 'scn', 'vec', 'sc', 'ro', 'la'],
+     'NORTH_EU': ['de', 'nl', 'fy', 'af', 'da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
+     'SCANDINAVIA': ['da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
+     'SAMI': ['se', 'sma', 'smj', 'smn', 'sms'],
+     'NORWAY': ['nb_NO', 'nb', 'nn_NO', 'nn', 'nog', 'no_nb', 'no'],
+     'CELTIC': ['ga', 'cy', 'br', 'gd', 'kw', 'gv']
+    }
+
+Code to see available pretrained models:
+
+.. code-block:: python
+
+    from transformers.hf_api import HfApi
+    model_list = HfApi().model_list()
+    org = "Helsinki-NLP"
+    model_ids = [x.modelId for x in model_list if x.modelId.startswith(org)]
+    suffix = [x.split('/')[1] for x in model_ids]
+    multi_models = [f'{org}/{s}' for s in suffix if s != s.lower()]


 MarianConfig
@@ -19,13 +19,6 @@ on the encoder, decoder, or reconstructing parts of the text.

 The Authors' code can be found `here <https://github.com/pytorch/fairseq/tree/master/examples/mbart>`__

-Examples
-_______________________________________________________________________________________________________________________
-
- Examples and scripts for fine-tuning mBART and other models for sequence to sequence tasks can be found in
-  `examples/seq2seq/ <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
- Given the large embeddings table, mBART consumes a large amount of GPU RAM, especially for fine-tuning.
-  :class:`MarianMTModel` is usually a better choice for bilingual machine translation.

 Training
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -44,8 +37,12 @@ the sequences for sequence-to-sequence fine-tuning.

    example_english_phrase = "UN Chief Says There Is No Military Solution in Syria"
    expected_translation_romanian = "Şeful ONU declară că nu există o soluţie militară în Siria"
-    batch = tokenizer.prepare_seq2seq_batch(example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian, return_tensors="pt")
-    model(input_ids=batch['input_ids'], labels=batch['labels']) # forward pass
+    batch = tokenizer.prepare_seq2seq_batch(example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian)
+    input_ids = batch["input_ids"]
+    target_ids = batch["decoder_input_ids"]
+    decoder_input_ids = target_ids[:, :-1].contiguous()
+    labels = target_ids[:, 1:].clone()
+    model(input_ids=input_ids, decoder_input_ids=decoder_input_ids, labels=labels) #forward

 - Generation

@@ -58,7 +55,7 @@ the sequences for sequence-to-sequence fine-tuning.
    model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-en-ro")
    tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-en-ro")
    article = "UN Chief Says There Is No Military Solution in Syria"
-    batch = tokenizer.prepare_seq2seq_batch(src_texts=[article], src_lang="en_XX", return_tensors="pt")
+    batch = tokenizer.prepare_seq2seq_batch(src_texts=[article], src_lang="en_XX")
    translated_tokens = model.generate(**batch, decoder_start_token_id=tokenizer.lang_code_to_id["ro_RO"])
    translation = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0]
    assert translation == "Şeful ONU declară că nu există o soluţie militară în Siria"
@@ -58,10 +58,10 @@ MobileBertTokenizerFast
 MobileBert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.mobilebert.modeling_mobilebert.MobileBertForPreTrainingOutput
+.. autoclass:: transformers.modeling_mobilebert.MobileBertForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.mobilebert.modeling_tf_mobilebert.TFMobileBertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_mobilebert.TFMobileBertForPreTrainingOutput
    :members:


@@ -1,53 +0,0 @@
-MT5
-----------------------------------------------------------------------------------------------------------------------
-
-Overview
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The mT5 model was presented in `mT5: A massively multilingual pre-trained text-to-text transformer
-<https://arxiv.org/abs/2010.11934>`_ by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya
-Siddhant, Aditya Barua, Colin Raffel.
-
-The abstract from the paper is the following:
-
-*The recent "Text-to-Text Transfer Transformer" (T5) leveraged a unified text-to-text format and scale to attain
-state-of-the-art results on a wide variety of English-language NLP tasks. In this paper, we introduce mT5, a
-multilingual variant of T5 that was pre-trained on a new Common Crawl-based dataset covering 101 languages. We describe
-the design and modified training of mT5 and demonstrate its state-of-the-art performance on many multilingual
-benchmarks. All of the code and model checkpoints*
-
-The original code can be found `here <https://github.com/google-research/multilingual-t5>`__.
-
-MT5Config
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5Config
-    :members:
-
-
-MT5Model
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5Model
-    :members:
-
-
-MT5ForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5ForConditionalGeneration
-    :members:
-
-
-TFMT5Model
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFMT5Model
-    :members:
-
-
-TFMT5ForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFMT5ForConditionalGeneration
-    :members:
@@ -31,19 +31,10 @@ All the `checkpoints <https://huggingface.co/models?search=pegasus>`__ are fine-
 - Each checkpoint is 2.2 GB on disk and 568M parameters.
 - FP16 is not supported (help/ideas on this appreciated!).
 - Summarizing xsum in fp32 takes about 400ms/sample, with default parameters on a v100 GPU.
- Full replication results and correctly pre-processed data can be found in this `Issue
-  <https://github.com/huggingface/transformers/issues/6844#issue-689259666>`__.
- `Distilled checkpoints <https://huggingface.co/models?search=distill-pegasus>`__ are described in this `paper
-  <https://arxiv.org/abs/2010.13002>`__.
+- For XSUM, The paper reports rouge1,rouge2, rougeL of paper: 47.21/24.56/39.25. As of Aug 9, this port scores
+  46.91/24.34/39.1.

-Examples
-_______________________________________________________________________________________________________________________
-
- `Script <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/finetune_pegasus_xsum.sh>`__ to
-  fine-tune pegasus on the XSUM dataset. Data download instructions at `examples/seq2seq/
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
- FP16 is not supported (help/ideas on this appreciated!).
- The adafactor optimizer is recommended for pegasus fine-tuning.
+The gap is likely because of different alpha/length_penalty implementations in beam search.


 Implementation Notes
@@ -54,7 +45,7 @@ Implementation Notes
 - Some key configuration differences:

    - static, sinusoidal position embeddings
-    - no :obj:`layernorm_embedding` (:obj:`PegasusConfig.normalize_embedding=False`)
+    - no :obj:`layernorm_embedding` (:obj`PegasusConfig.normalize_embedding=False`)
    - the model starts generating with pad_token_id (which has 0 token_embedding) as the prefix.
    - more beams are used (:obj:`num_beams=8`)
 - All pretrained pegasus checkpoints are the same besides three attributes: :obj:`tokenizer.model_max_length` (maximum
@@ -78,7 +69,7 @@ Usage Example
    torch_device = 'cuda' if torch.cuda.is_available() else 'cpu'
    tokenizer = PegasusTokenizer.from_pretrained(model_name)
    model = PegasusForConditionalGeneration.from_pretrained(model_name).to(torch_device)
-    batch = tokenizer.prepare_seq2seq_batch(src_text, truncation=True, padding='longest', return_tensors="pt").to(torch_device)
+    batch = tokenizer.prepare_seq2seq_batch(src_text, truncation=True, padding='longest').to(torch_device)
    translated = model.generate(**batch)
    tgt_text = tokenizer.batch_decode(translated, skip_special_tokens=True)
    assert tgt_text[0] == "California's largest electricity provider has turned off power to hundreds of thousands of customers."
@@ -47,16 +47,16 @@ ProphetNetTokenizer
 ProphetNet specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetSeq2SeqLMOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqLMOutput
    :members:

-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetSeq2SeqModelOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqModelOutput
    :members:

-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetDecoderModelOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderModelOutput
    :members:

-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetDecoderLMOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderLMOutput
    :members:

 ProphetNetModel
@@ -50,10 +50,10 @@ RagTokenizer
 Rag specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.rag.modeling_rag.RetrievAugLMMarginOutput
+.. autoclass:: transformers.modeling_rag.RetrievAugLMMarginOutput
    :members:

-.. autoclass:: transformers.models.rag.modeling_rag.RetrievAugLMOutput
+.. autoclass:: transformers.modeling_rag.RetrievAugLMOutput
    :members:

 RagRetriever
@@ -146,10 +146,3 @@ TFRobertaForQuestionAnswering

 .. autoclass:: transformers.TFRobertaForQuestionAnswering
    :members: call
-
-
-FlaxRobertaModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxRobertaModel
-    :members: __call__
@@ -64,7 +64,7 @@ token. T5 can be trained / fine-tuned both in a supervised and unsupervised fash
  input_ids = tokenizer('The <extra_id_0> walks in <extra_id_1> park', return_tensors='pt').input_ids
  labels = tokenizer('<extra_id_0> cute dog <extra_id_1> the <extra_id_2>', return_tensors='pt').input_ids
  # the forward function automatically creates the correct decoder_input_ids
-  loss = model(input_ids=input_ids, labels=labels).loss
+  loss = model(input_ids=input_ids, labels=labels, return_dict=True).loss

 - Supervised training

@@ -77,7 +77,7 @@ token. T5 can be trained / fine-tuned both in a supervised and unsupervised fash
  input_ids = tokenizer('translate English to German: The house is wonderful.', return_tensors='pt').input_ids
  labels = tokenizer('Das Haus ist wunderbar.', return_tensors='pt').input_ids
  # the forward function automatically creates the correct decoder_input_ids
-  loss = model(input_ids=input_ids, labels=labels).loss
+  loss = model(input_ids=input_ids, labels=labels, return_dict=True).loss


 T5Config
@@ -49,16 +49,16 @@ TransfoXLTokenizer
 TransfoXL specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.transfo_xl.modeling_transfo_xl.TransfoXLModelOutput
+.. autoclass:: transformers.modeling_transfo_xl.TransfoXLModelOutput
    :members:

-.. autoclass:: transformers.models.transfo_xl.modeling_transfo_xl.TransfoXLLMHeadModelOutput
+.. autoclass:: transformers.modeling_transfo_xl.TransfoXLLMHeadModelOutput
    :members:

-.. autoclass:: transformers.models.transfo_xl.modeling_tf_transfo_xl.TFTransfoXLModelOutput
+.. autoclass:: transformers.modeling_tf_transfo_xl.TFTransfoXLModelOutput
    :members:

-.. autoclass:: transformers.models.transfo_xl.modeling_tf_transfo_xl.TFTransfoXLLMHeadModelOutput
+.. autoclass:: transformers.modeling_tf_transfo_xl.TFTransfoXLLMHeadModelOutput
    :members:


@@ -50,7 +50,7 @@ XLMTokenizer
 XLM specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.xlm.modeling_xlm.XLMForQuestionAnsweringOutput
+.. autoclass:: transformers.modeling_xlm.XLMForQuestionAnsweringOutput
    :members:


@@ -53,43 +53,43 @@ XLNetTokenizer
 XLNet specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetModelOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetModelOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetLMHeadModelOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetLMHeadModelOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForSequenceClassificationOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForSequenceClassificationOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForMultipleChoiceOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForMultipleChoiceOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForTokenClassificationOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForTokenClassificationOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForQuestionAnsweringSimpleOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForQuestionAnsweringSimpleOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForQuestionAnsweringOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForQuestionAnsweringOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetModelOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetModelOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetLMHeadModelOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetLMHeadModelOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForSequenceClassificationOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForSequenceClassificationOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForMultipleChoiceOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForMultipleChoiceOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForTokenClassificationOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForTokenClassificationOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForQuestionAnsweringSimpleOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForQuestionAnsweringSimpleOutput
    :members:


@@ -37,7 +37,7 @@ For instance:

 .. code-block::

-    >>> model = AutoModel.from_pretrained(
+    >>> tokenizer = AutoTokenizer.from_pretrained(
    >>>   "julien-c/EsperBERTo-small",
    >>>   revision="v2.0.1" # tag name, or branch name, or commit hash
    >>> )
@@ -46,42 +46,31 @@ Basic steps
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 In order to upload a model, you'll need to first create a git repo. This repo will live on the model hub, allowing
-users to clone it and you (and your organization members) to push to it.
+users to clone it and you (and your organization members) to push to it. First, you should ensure you are logged in the
+``transformers-cli``:

-You can create a model repo directly from the website, `here <https://huggingface.co/new>`.
-
-Alternatively, you can use the ``transformers-cli``. The next steps describe that process:
-
-Go to a terminal and run the following command. It should be in the virtual environment where you installed 🤗
+Go in a terminal and run the following command. It should be in the virtual environment where you installed 🤗
 Transformers, since that command :obj:`transformers-cli` comes from the library.

-.. code-block:: bash
+.. code-block::

    transformers-cli login


 Once you are logged in with your model hub credentials, you can start building your repositories. To create a repo:

-.. code-block:: bash
+.. code-block::

    transformers-cli repo create your-model-name

-This creates a repo on the model hub, which can be cloned.
+This creates a repo on the model hub, which can be cloned. You can then add/remove from that repo as you would with any
+other git repo.

-.. code-block:: bash
+.. code-block::

    git clone https://huggingface.co/username/your-model-name

-    # Make sure you have git-lfs installed
-    # (https://git-lfs.github.com/)
-    git lfs install
-
-When you have your local clone of your repo and lfs installed, you can then add/remove from that clone as you would
-with any other git repo.
-
-.. code-block:: bash
-
-    # Commit as usual
+    # Then commit as usual
    cd your-model-name
    echo "hello" >> README.md
    git add . && git commit -m "Update from $USER"
@@ -170,25 +159,24 @@ Or, if you're using the Trainer API
 .. code-block::

    >>> trainer.save_model("path/to/awesome-name-you-picked")
-    >>> tokenizer.save_pretrained("path/to/repo/clone/your-model-name")

 You can then add these files to the staging environment and verify that they have been correctly staged with the ``git
 status`` command:

-.. code-block:: bash
+.. code-block::

    git add --all
    git status

 Finally, the files should be comitted:

-.. code-block:: bash
+.. code-block::

    git commit -m "First version of the your-model-name model and tokenizer."

 And pushed to the remote:

-.. code-block:: bash
+.. code-block::

    git push

@@ -211,7 +199,7 @@ don't forget to link to its model card so that people can fully trace how your m
 If you have never made a pull request to the 🤗 Transformers repo, look at the :doc:`contributing guide <contributing>`
 to see the steps to follow.

-.. note::
+.. Note::

    You can also send your model card in the folder you uploaded with the CLI by placing it in a `README.md` file
    inside `path/to/awesome-name-you-picked/`.
@@ -237,47 +225,3 @@ You may specify a revision by using the ``revision`` flag in the ``from_pretrain
    >>>   "julien-c/EsperBERTo-small",
    >>>   revision="v2.0.1" # tag name, or branch name, or commit hash
    >>> )
-
-Workflow in a Colab notebook
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you're in a Colab notebook (or similar) with no direct access to a terminal, here is the workflow you can use to
-upload your model. You can execute each one of them in a cell by adding a ! at the beginning.
-
-First you need to install `git-lfs` in the environment used by the notebook:
-
-.. code-block:: bash
-
-    sudo apt-get install git-lfs
-
-Then you can use the :obj:`transformers-cli` to create your new repo:
-
-
-.. code-block:: bash
-
-    transformers-cli login
-    transformers-cli repo create your-model-name
-
-Once it's created, you can clone it and configure it (replace username by your username on huggingface.co):
-
-.. code-block:: bash
-
-    git clone https://username:password@huggingface.co/username/your-model-name
-    # Alternatively if you have a token,
-    # you can use it instead of your password
-    git clone https://username:token@huggingface.co/username/your-model-name
-
-    cd your-model-name
-    git lfs install
-    git config --global user.email "email@example.com"
-    # Tip: using the same email than for your huggingface.co account will link your commits to your profile
-    git config --global user.name "Your name"
-
-Once you've saved your model inside, and your clone is setup with the right remote URL, you can add it and push it with
-usual git commands.
-
-.. code-block:: bash
-
-    git add .
-    git commit -m "Initial commit"
-    git push
@@ -560,7 +560,6 @@ A framework for translation models, using the same models as BART

 The library provides a version of this model for conditional generation.

-
 T5
 -----------------------------------------------------------------------------------------------------------------------

@@ -593,28 +592,6 @@ For instance, if we have the sentence “My dog is very cute .”, and we decide

 The library provides a version of this model for conditional generation.

-
-MT5
-----------------------------------------------------------------------------------------------------------------------
-
-.. raw:: html
-
-   <a href="https://huggingface.co/models?filter=mt5">
-       <img alt="Models" src="https://img.shields.io/badge/All_model_pages-mt5-blueviolet">
-   </a>
-   <a href="model_doc/mt5.html">
-       <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-mt5-blueviolet">
-   </a>
-
-`mT5: A massively multilingual pre-trained text-to-text transformer <https://arxiv.org/abs/2010.11934>`_, Linting Xue
-et al.
-
-The model architecture is same as T5. mT5's pre-training objective includes T5's self-supervised training, but not T5's
-supervised training. mT5 is trained on 101 languages.
-
-The library provides a version of this model for conditional generation.
-
-
 MBart
 -----------------------------------------------------------------------------------------------------------------------

@@ -3,11 +3,11 @@ Pretrained models

 Here is the full list of the currently provided pretrained models together with a short presentation of each model.

-For a list that includes all community-uploaded models, refer to `https://huggingface.co/models
+For a list that includes community-uploaded models, refer to `https://huggingface.co/models
 <https://huggingface.co/models>`__.

 +--------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
-| Architecture       | Model id                                                   | Details of the model                                                                                                                  |
+| Architecture       | Shortcut name                                              | Details of the model                                                                                                                  |
 +====================+============================================================+=======================================================================================================================================+
 | BERT               | ``bert-base-uncased``                                      | | 12-layer, 768-hidden, 12-heads, 110M parameters.                                                                                    |
 |                    |                                                            | | Trained on lower-cased English text.                                                                                                |
@@ -89,7 +89,7 @@ each other. The process is the following:
    >>> import torch

    >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased-finetuned-mrpc")
-    >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc")
+    >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc", return_dict=True)

    >>> classes = ["not paraphrase", "is paraphrase"]

@@ -122,7 +122,7 @@ each other. The process is the following:
    >>> import tensorflow as tf

    >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased-finetuned-mrpc")
-    >>> model = TFAutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc")
+    >>> model = TFAutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc", return_dict=True)

    >>> classes = ["not paraphrase", "is paraphrase"]

@@ -211,7 +211,7 @@ Here is an example of question answering using a model and a tokenizer. The proc
    >>> import torch

    >>> tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
-    >>> model = AutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
+    >>> model = AutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad", return_dict=True)

    >>> text = r"""
    ... 🤗 Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides general-purpose
@@ -253,7 +253,7 @@ Here is an example of question answering using a model and a tokenizer. The proc
    >>> import tensorflow as tf

    >>> tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
-    >>> model = TFAutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
+    >>> model = TFAutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad", return_dict=True)

    >>> text = r"""
    ... 🤗 Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides general-purpose
@@ -373,7 +373,7 @@ Here is an example of doing masked language modeling using a model and a tokeniz
    >>> import torch

    >>> tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased")
-    >>> model = AutoModelWithLMHead.from_pretrained("distilbert-base-cased")
+    >>> model = AutoModelWithLMHead.from_pretrained("distilbert-base-cased", return_dict=True)

    >>> sequence = f"Distilled models are smaller than the models they mimic. Using them instead of the large versions would help {tokenizer.mask_token} our carbon footprint."

@@ -389,7 +389,7 @@ Here is an example of doing masked language modeling using a model and a tokeniz
    >>> import tensorflow as tf

    >>> tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased")
-    >>> model = TFAutoModelWithLMHead.from_pretrained("distilbert-base-cased")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("distilbert-base-cased", return_dict=True)

    >>> sequence = f"Distilled models are smaller than the models they mimic. Using them instead of the large versions would help {tokenizer.mask_token} our carbon footprint."

@@ -437,7 +437,7 @@ of tokens.
    >>> from torch.nn import functional as F

    >>> tokenizer = AutoTokenizer.from_pretrained("gpt2")
-    >>> model = AutoModelWithLMHead.from_pretrained("gpt2")
+    >>> model = AutoModelWithLMHead.from_pretrained("gpt2", return_dict=True)

    >>> sequence = f"Hugging Face is based in DUMBO, New York City, and "

@@ -461,7 +461,7 @@ of tokens.
    >>> import tensorflow as tf

    >>> tokenizer = AutoTokenizer.from_pretrained("gpt2")
-    >>> model = TFAutoModelWithLMHead.from_pretrained("gpt2")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("gpt2", return_dict=True)

    >>> sequence = f"Hugging Face is based in DUMBO, New York City, and "

@@ -513,14 +513,14 @@ Here, the model generates a random text with a total maximal length of *50* toke
 concerned, I will"*. The default arguments of ``PreTrainedModel.generate()`` can be directly overridden in the
 pipeline, as is shown above for the argument ``max_length``.

-Here is an example of text generation using ``XLNet`` and its tokenizer.
+Here is an example of text generation using ``XLNet`` and its tokenzier.

 .. code-block::

    >>> ## PYTORCH CODE
    >>> from transformers import AutoModelWithLMHead, AutoTokenizer

-    >>> model = AutoModelWithLMHead.from_pretrained("xlnet-base-cased")
+    >>> model = AutoModelWithLMHead.from_pretrained("xlnet-base-cased", return_dict=True)
    >>> tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")

    >>> # Padding text helps XLNet with short prompts - proposed by Aman Rusia in https://github.com/rusiaaman/XLNet-gen#methodology
@@ -545,7 +545,7 @@ Here is an example of text generation using ``XLNet`` and its tokenizer.
    >>> ## TENSORFLOW CODE
    >>> from transformers import TFAutoModelWithLMHead, AutoTokenizer

-    >>> model = TFAutoModelWithLMHead.from_pretrained("xlnet-base-cased")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("xlnet-base-cased", return_dict=True)
    >>> tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")

    >>> # Padding text helps XLNet with short prompts - proposed by Aman Rusia in https://github.com/rusiaaman/XLNet-gen#methodology
@@ -664,7 +664,7 @@ Here is an example of doing named entity recognition, using a model and a tokeni
    >>> from transformers import AutoModelForTokenClassification, AutoTokenizer
    >>> import torch

-    >>> model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english")
+    >>> model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english", return_dict=True)
    >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

    >>> label_list = [
@@ -692,7 +692,7 @@ Here is an example of doing named entity recognition, using a model and a tokeni
    >>> from transformers import TFAutoModelForTokenClassification, AutoTokenizer
    >>> import tensorflow as tf

-    >>> model = TFAutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english")
+    >>> model = TFAutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english", return_dict=True)
    >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

    >>> label_list = [
@@ -790,7 +790,7 @@ CNN / Daily Mail), it yields very good results.
    >>> ## PYTORCH CODE
    >>> from transformers import AutoModelWithLMHead, AutoTokenizer

-    >>> model = AutoModelWithLMHead.from_pretrained("t5-base")
+    >>> model = AutoModelWithLMHead.from_pretrained("t5-base", return_dict=True)
    >>> tokenizer = AutoTokenizer.from_pretrained("t5-base")

    >>> # T5 uses a max_length of 512 so we cut the article to 512 tokens.
@@ -799,7 +799,7 @@ CNN / Daily Mail), it yields very good results.
    >>> ## TENSORFLOW CODE
    >>> from transformers import TFAutoModelWithLMHead, AutoTokenizer

-    >>> model = TFAutoModelWithLMHead.from_pretrained("t5-base")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("t5-base", return_dict=True)
    >>> tokenizer = AutoTokenizer.from_pretrained("t5-base")

    >>> # T5 uses a max_length of 512 so we cut the article to 512 tokens.
@@ -834,7 +834,7 @@ Here is an example of doing translation using a model and a tokenizer. The proce

 1. Instantiate a tokenizer and a model from the checkpoint name. Summarization is usually done using an encoder-decoder
   model, such as ``Bart`` or ``T5``.
-2. Define the article that should be summarized.
+2. Define the article that should be summarizaed.
 3. Add the T5 specific prefix "translate English to German: "
 4. Use the ``PreTrainedModel.generate()`` method to perform the translation.

@@ -843,7 +843,7 @@ Here is an example of doing translation using a model and a tokenizer. The proce
    >>> ## PYTORCH CODE
    >>> from transformers import AutoModelWithLMHead, AutoTokenizer

-    >>> model = AutoModelWithLMHead.from_pretrained("t5-base")
+    >>> model = AutoModelWithLMHead.from_pretrained("t5-base", return_dict=True)
    >>> tokenizer = AutoTokenizer.from_pretrained("t5-base")

    >>> inputs = tokenizer.encode("translate English to German: Hugging Face is a technology company based in New York and Paris", return_tensors="pt")
@@ -851,7 +851,7 @@ Here is an example of doing translation using a model and a tokenizer. The proce
    >>> ## TENSORFLOW CODE
    >>> from transformers import TFAutoModelWithLMHead, AutoTokenizer

-    >>> model = TFAutoModelWithLMHead.from_pretrained("t5-base")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("t5-base", return_dict=True)
    >>> tokenizer = AutoTokenizer.from_pretrained("t5-base")

    >>> inputs = tokenizer.encode("translate English to German: Hugging Face is a technology company based in New York and Paris", return_tensors="tf")
@@ -405,32 +405,32 @@ decorators are used to set the requirements of tests CPU/GPU/TPU-wise:

 * ``require_torch`` - this test will run only under torch
 * ``require_torch_gpu`` - as ``require_torch`` plus requires at least 1 GPU
-* ``require_torch_multi_gpu`` - as ``require_torch`` plus requires at least 2 GPUs
-* ``require_torch_non_multi_gpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
+* ``require_torch_multigpu`` - as ``require_torch`` plus requires at least 2 GPUs
+* ``require_torch_non_multigpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
 * ``require_torch_tpu`` - as ``require_torch`` plus requires at least 1 TPU

 Let's depict the GPU requirements in the following table:


-+----------+----------------------------------+
-| n gpus   |  decorator                       |
-+==========+==================================+
-| ``>= 0`` | ``@require_torch``               |
-+----------+----------------------------------+
-| ``>= 1`` | ``@require_torch_gpu``           |
-+----------+----------------------------------+
-| ``>= 2`` | ``@require_torch_multi_gpu``     |
-+----------+----------------------------------+
-| ``< 2``  | ``@require_torch_non_multi_gpu`` |
-+----------+----------------------------------+
+----------+---------------------------------+
+| n gpus   |  decorator                      |
+==========+=================================+
+| ``>= 0`` | ``@require_torch``              |
+----------+---------------------------------+
+| ``>= 1`` | ``@require_torch_gpu``          |
+----------+---------------------------------+
+| ``>= 2`` | ``@require_torch_multigpu``     |
+----------+---------------------------------+
+| ``< 2``  | ``@require_torch_non_multigpu`` |
+----------+---------------------------------+


 For example, here is a test that must be run only when there are 2 or more GPUs available and pytorch is installed:

 .. code-block:: python

-    @require_torch_multi_gpu
-    def test_example_with_multi_gpu():
+    @require_torch_multigpu
+    def test_example_with_multigpu():

 If a test requires ``tensorflow`` use the ``require_tf`` decorator. For example:

@@ -454,7 +454,7 @@ last for them to work correctly. Here is an example of the correct usage:
 .. code-block:: python

    @parameterized.expand(...)
-    @require_torch_multi_gpu
+    @require_torch_multigpu
    def test_integration_foo():

 This order problem doesn't exist with ``@pytest.mark.parametrize``, you can put it first or last and it will still
@@ -716,11 +716,11 @@ Temporary files and directories
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 Using unique temporary files and directories are essential for parallel test running, so that the tests won't overwrite
-each other's data. Also we want to get the temporary files and directories removed at the end of each test that created
+each other's data. Also we want to get the temp files and directories removed at the end of each test that created
 them. Therefore, using packages like ``tempfile``, which address these needs is essential.

-However, when debugging tests, you need to be able to see what goes into the temporary file or directory and you want
-to know it's exact path and not having it randomized on every test re-run.
+However, when debugging tests, you need to be able to see what goes into the temp file or directory and you want to
+know it's exact path and not having it randomized on every test re-run.

 A helper class :obj:`transformers.test_utils.TestCasePlus` is best used for such purposes. It's a sub-class of
 :obj:`unittest.TestCase`, so we can easily inherit from it in the test modules.
@@ -736,33 +736,32 @@ Here is an example of its usage:

 This code creates a unique temporary directory, and sets :obj:`tmp_dir` to its location.

-* Create a unique temporary dir:
+In this and all the following scenarios the temporary directory will be auto-removed at the end of test, unless
+``after=False`` is passed to the helper function.
+
+* Create a temporary directory of my choice and delete it at the end - useful for debugging when you want to monitor a
+  specific directory:

 .. code-block:: python

    def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir()
+        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test")

-``tmp_dir`` will contain the path to the created temporary dir. It will be automatically removed at the end of the
-test.
-
-* Create a temporary dir of my choice, ensure it's empty before the test starts and don't empty it after the test.
+* Create a temporary directory of my choice and do not delete it at the end---useful for when you want to look at the
+  temp results:

 .. code-block:: python

    def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir("./xxx")
+        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", after=False)

-This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn't
-leave any data in there.
+* Create a temporary directory of my choice and ensure to delete it right away---useful for when you disabled deletion
+  in the previous test run and want to make sure the that temporary directory is empty before the new test is run:

-* You can override the default behavior by directly overriding the ``before`` and ``after`` args, leading to one of the
-  following behaviors:
+.. code-block:: python

-    - ``before=True``: the temporary dir will always be cleared at the beginning of the test.
-    - ``before=False``: if the temporary dir already existed, any existing files will remain there.
-    - ``after=True``: the temporary dir will always be deleted at the end of the test.
-    - ``after=False``: the temporary dir will always be left intact at the end of the test.
+   def test_whatever(self):
+        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", before=True)

 .. note::
   In order to run the equivalent of ``rm -r`` safely, only subdirs of the project repository checkout are allowed if
@@ -816,7 +815,7 @@ or the ``xfail`` way:
    @pytest.mark.xfail
    def test_feature_x():

- Here is how to skip a test based on some internal check inside the test:
+Here is how to skip a test based on some internal check inside the test:

 .. code-block:: python

@@ -839,7 +838,7 @@ or the ``xfail`` way:
    def test_feature_x():
        pytest.xfail("expected to fail until bug XYZ is fixed")

- Here is how to skip all tests in a module if some import is missing:
+Here is how to skip all tests in a module if some import is missing:

 .. code-block:: python

@@ -1,243 +1,223 @@
-Summary of the tokenizers
+Tokenizer summary
 -----------------------------------------------------------------------------------------------------------------------

-On this page, we will have a closer look at tokenization. As we saw in :doc:`the preprocessing tutorial
-<preprocessing>`, tokenizing a text is splitting it into words or subwords, which then are converted to ids through a
-look-up table. Converting words or subwords to ids is straightforward, so in this summary, we will focus on splitting a
-text into words or subwords (i.e. tokenizing a text). More specifically, we will look at the three main types of
-tokenizers used in 🤗 Transformers: :ref:`Byte-Pair Encoding (BPE) <byte-pair-encoding>`, :ref:`WordPiece <wordpiece>`,
-and :ref:`SentencePiece <sentencepiece>`, and show exemplary which tokenizer type is used by which model.
+In this page, we will have a closer look at tokenization. As we saw in :doc:`the preprocessing tutorial
+<preprocessing>`, tokenizing a text is splitting it into words or subwords, which then are converted to ids. The second
+part is pretty straightforward, here we will focus on the first part. More specifically, we will look at the three main
+different kinds of tokenizers used in 🤗 Transformers: :ref:`Byte-Pair Encoding (BPE) <byte-pair-encoding>`,
+:ref:`WordPiece <wordpiece>` and :ref:`SentencePiece <sentencepiece>`, and provide examples of models using each of
+those.

-Note that on each model page, you can look at the documentation of the associated tokenizer to know which tokenizer
-type was used by the pretrained model. For instance, if we look at :class:`~transformers.BertTokenizer`, we can see
-that the model uses :ref:`WordPiece <wordpiece>`.
+Note that on each model page, you can look at the documentation of the associated tokenizer to know which of those
+algorithms the pretrained model used. For instance, if we look at :class:`~transformers.BertTokenizer`, we can see it's
+using :ref:`WordPiece <wordpiece>`.

-Introduction
+Introduction to tokenization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-Splitting a text into smaller chunks is a task that is harder than it looks, and there are multiple ways of doing so.
-For instance, let's look at the sentence ``"Don't you love 🤗 Transformers? We sure do."`` A simple way of tokenizing
-this text is to split it by spaces, which would give:
+Splitting a text in smaller chunks is a task that's harder than it looks, and there are multiple ways of doing it. For
+instance, let's look at the sentence "Don't you love 🤗 Transformers? We sure do." A first simple way of tokenizing this
+text is just to split it by spaces, which would give:

 .. code-block::

    ["Don't", "you", "love", "🤗", "Transformers?", "We", "sure", "do."]

-This is a sensible first step, but if we look at the tokens ``"Transformers?"`` and ``"do."``, we notice that the
-punctuation is attached to the words ``"Transformer"`` and ``"do"``, which is suboptimal. We should take the
-punctuation into account so that a model does not have to learn a different representation of a word and every possible
-punctuation symbol that could follow it, which would explode the number of representations the model has to learn.
-Taking punctuation into account, tokenizing our exemplary text would give:
+This is a nice first step, but if we look at the tokens "Transformers?" or "do.", we can see we can do better. Those
+will be different than the tokens "Transformers" and "do" for our model, so we should probably take the punctuation
+into account. This would give:

 .. code-block::

    ["Don", "'", "t", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]

-Better. However, it is disadvantageous, how the tokenization dealt with the word ``"Don't"``. ``"Don't"`` stands for
-``"do not"``, so it would be better tokenized as ``["Do", "n't"]``. This is where things start getting complicated, and
-part of the reason each model has its own tokenizer type. Depending on the rules we apply for tokenizing a text, a
-different tokenized output is generated for the same text. A pretrained model only performs properly if you feed it an
-input that was tokenized with the same rules that were used to tokenize its training data.
+which is better already. One thing that is annoying though is how it dealt with "Don't". "Don't" stands for do not, so
+it should probably be better tokenized as ``["Do", "n't"]``. This is where things start getting more complicated, and
+part of the reason each kind of model has its own tokenizer class. Depending on the rules we apply to split our texts
+into tokens, we'll get different tokenized versions of the same text. And of course, a given pretrained model won't
+perform properly if you don't use the exact same rules as the persons who pretrained it.

 `spaCy <https://spacy.io/>`__ and `Moses <http://www.statmt.org/moses/?n=Development.GetStarted>`__ are two popular
-rule-based tokenizers. Applying them on our example, *spaCy* and *Moses* would output something like:
+rule-based tokenizers. On the text above, they'd output something like:

 .. code-block::

    ["Do", "n't", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]

-As can be seen space and punctuation tokenization, as well as rule-based tokenization, is used here. Space and
-punctuation tokenization and rule-based tokenization are both examples of word tokenization, which is loosely defined
-as splitting sentences into words. While it's the most intuitive way to split texts into smaller chunks, this
-tokenization method can lead to problems for massive text corpora. In this case, space and punctuation tokenization
-usually generates a very big vocabulary (the set of all unique words and tokens used). *E.g.*, :doc:`Transformer XL
-<model_doc/transformerxl>` uses space and punctuation tokenization, resulting in a vocabulary size of 267,735!
+Space/punctuation-tokenization and rule-based tokenization are both examples of word tokenization, which is splitting a
+sentence into words. While it's the most intuitive way to separate texts in smaller chunks, it can have a problem when
+you have a huge corpus: it usually yields a very big vocabulary (the set of all unique tokens used). :doc:`Transformer
+XL <model_doc/transformerxl>` for instance uses space/punctuation-tokenization, and has a vocabulary size of 267,735!

-Such a big vocabulary size forces the model to have an enormous embedding matrix as the input and output layer, which
-causes both an increased memory and time complexity. In general, transformers models rarely have a vocabulary size
-greater than 50,000, especially if they are pretrained only on a single language.
+A huge vocabulary size means a huge embedding matrix at the start of the model, which will cause memory problems.
+TransformerXL deals with it by using a special kind of embeddings called adaptive embeddings, but in general,
+transformers models rarely have a vocabulary size greater than 50,000, especially if they are trained on a single
+language.

-So if simple space and punctuation tokenization is unsatisfactory, why not simply tokenize on characters? While
-character tokenization is very simple and would greatly reduce memory and time complexity it makes it much harder for
-the model to learn meaningful input representations. *E.g.* learning a meaningful context-independent representation
-for the letter ``"t"`` is much harder as learning a context-independent representation for the word ``"today"``.
-Therefore, character tokenization is often accompanied by a loss of performance. So to get the best of both worlds,
-transformers models use a hybrid between word-level and character-level tokenization called **subword** tokenization.
+So if tokenizing on words is unsatisfactory, we could go on the opposite direction and simply tokenize on characters.
+While it's very simple and would save a lot of memory, this doesn't allow the model to learn representations of texts
+as meaningful as when using a word tokenization, leading to a loss of performance. So to get the best of both worlds,
+all transformers models use a hybrid between word-level and character-level tokenization called subword tokenization.

 Subword tokenization
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-Subword tokenization algorithms rely on the principle that frequently used words should not be split into smaller
-subwords, but rare words should be decomposed into meaningful subwords. For instance ``"annoyingly"`` might be
-considered a rare word and could be decomposed into ``"annoying"`` and ``"ly"``. Both ``"annoying"`` and ``"ly"`` as
-stand-alone subwords would appear more frequently while at the same time the meaning of ``"annoyingly"`` is kept by the
-composite meaning of ``"annoying"`` and ``"ly"``. This is especially useful in agglutinative languages such as Turkish,
-where you can form (almost) arbitrarily long complex words by stringing together subwords.
+Subword tokenization algorithms rely on the principle that most common words should be left as is, but rare words
+should be decomposed in meaningful subword units. For instance "annoyingly" might be considered a rare word and
+decomposed as "annoying" and "ly". This is especially useful in agglutinative languages such as Turkish, where you can
+form (almost) arbitrarily long complex words by stringing together some subwords.

-Subword tokenization allows the model to have a reasonable vocabulary size while being able to learn meaningful
-context-independent representations. In addition, subword tokenization enables the model to process words it has never
-seen before, by decomposing them into known subwords. For instance, the :class:`~transformers.BertTokenizer` tokenizes
-``"I have a new GPU!"`` as follows:
+This allows the model to keep a reasonable vocabulary while still learning useful representations for common words or
+subwords. This also enables the model to process words it has never seen before, by decomposing them into subwords it
+knows. For instance, the base :class:`~transformers.BertTokenizer` will tokenize "I have a new GPU!" like this:

 .. code-block::

    >>> from transformers import BertTokenizer
-    >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+    >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    >>> tokenizer.tokenize("I have a new GPU!")
-    ["i", "have", "a", "new", "gp", "##u", "!"]
+    ['i', 'have', 'a', 'new', 'gp', '##u', '!']

-Because we are considering the uncased model, the sentence was lowercased first. We can see that the words ``["i",
-"have", "a", "new"]`` are present in the tokenizer's vocabulary, but the word ``"gpu"`` is not. Consequently, the
-tokenizer splits ``"gpu"`` into known subwords: ``["gp" and "##u"]``. ``"##"`` means that the rest of the token should
-be attached to the previous one, without space (for decoding or reversal of the tokenization).
+Since we are considering the uncased model, the sentence was lowercased first. Then all the words were present in the
+vocabulary of the tokenizer, except for "gpu", so the tokenizer splits it in subwords it knows: "gp" and "##u". The
+"##" means that the rest of the token should be attached to the previous one, without space (for when we need to decode
+predictions and reverse the tokenization).

-As another example, :class:`~transformers.XLNetTokenizer` tokenizes our previously exemplary text as follows:
+Another example is when we use the base :class:`~transformers.XLNetTokenizer` to tokenize our previous text:

 .. code-block::

    >>> from transformers import XLNetTokenizer
-    >>> tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased")
+    >>> tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
    >>> tokenizer.tokenize("Don't you love 🤗 Transformers? We sure do.")
-    ["▁Don", "'", "t", "▁you", "▁love", "▁", "🤗", "▁", "Transform", "ers", "?", "▁We", "▁sure", "▁do", "."]
+    ['▁Don', "'", 't', '▁you', '▁love', '▁', '🤗', '▁', 'Transform', 'ers', '?', '▁We', '▁sure', '▁do', '.']

-We'll get back to the meaning of those ``"▁"`` when we look at :ref:`SentencePiece <sentencepiece>`. As one can see,
-the rare word ``"Transformers"`` has been split into the more frequent subwords ``"Transform"`` and ``"ers"``.
+We'll get back to the meaning of those '▁' when we look at :ref:`SentencePiece <sentencepiece>` but you can see
+Transformers has been split into "Transform" and "ers".

-Let's now look at how the different subword tokenization algorithms work. Note that all of those tokenization
-algorithms rely on some form of training which is usually done on the corpus the corresponding model will be trained
-on.
+Let's now look at how the different subword tokenization algorithms work. Note that they all rely on some form of
+training which is usually done on the corpus the corresponding model will be trained on.

 .. _byte-pair-encoding:

-Byte-Pair Encoding (BPE)
+Byte-Pair Encoding
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-Byte-Pair Encoding (BPE) was introduced in `Neural Machine Translation of Rare Words with Subword Units (Sennrich et
-al., 2015) <https://arxiv.org/abs/1508.07909>`__. BPE relies on a pre-tokenizer that splits the training data into
-words. Pretokenization can be as simple as space tokenization, e.g. :doc:`GPT-2 <model_doc/gpt2>`, :doc:`Roberta
-<model_doc/roberta>`. More advanced pre-tokenization include rule-based tokenization, e.g. :doc:`XLM <model_doc/xlm>`,
-:doc:`FlauBERT <model_doc/flaubert>` which uses Moses for most languages, or :doc:`GPT <model_doc/gpt>` which uses
-Spacy and ftfy, to count the frequency of each word in the training corpus.
+Byte-Pair Encoding was introduced in `this paper <https://arxiv.org/abs/1508.07909>`__. It relies on a pretokenizer
+splitting the training data into words, which can be a simple space tokenization (:doc:`GPT-2 <model_doc/gpt2>` and
+:doc:`Roberta <model_doc/roberta>` uses this for instance) or a rule-based tokenizer (:doc:`XLM <model_doc/xlm>` use
+Moses for most languages, as does :doc:`FlauBERT <model_doc/flaubert>`),

-After pre-tokenization, a set of unique words has been created and the frequency of each word it occurred in the
-training data has been determined. Next, BPE creates a base vocabulary consisting of all symbols that occur in the set
-of unique words and learns merge rules to form a new symbol from two symbols of the base vocabulary. It does so until
-the vocabulary has attained the desired vocabulary size. Note that the desired vocabulary size is a hyperparameter to
-define before training the tokenizer.
+:doc:`GPT <model_doc/gpt>` uses Spacy and ftfy, and counts the frequency of each word in the training corpus.

-As an example, let's assume that after pre-tokenization, the following set of words including their frequency has been
-determined:
+It then begins from the list of all characters and will learn merge rules to form a new token from two symbols in the
+vocabulary until it has learned a vocabulary of the desired size (this is a hyperparameter to pick).
+
+Let's say that after the pre-tokenization we have the following words (the number indicating the frequency of each
+word):

 .. code-block::

-    ("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)
+    ('hug', 10), ('pug', 5), ('pun', 12), ('bun', 4), ('hugs', 5)

-Consequently, the base vocabulary is ``["b", "g", "h", "n", "p", "s", "u"]``. Splitting all words into symbols of the
-base vocabulary, we obtain:
+Then the base vocabulary is ['b', 'g', 'h', 'n', 'p', 's', 'u'] and all our words are first split by character:

 .. code-block::

-    ("h" "u" "g", 10), ("p" "u" "g", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "u" "g" "s", 5)
+    ('h' 'u' 'g', 10), ('p' 'u' 'g', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'u' 'g' 's', 5)

-BPE then counts the frequency of each possible symbol pair and picks the symbol pair that occurs most frequently. In
-the example above ``"h"`` followed by ``"u"`` is present `10 + 5 = 15` times (10 times in the 10 occurrences of
-``"hug"``, 5 times in the 5 occurrences of "hugs"). However, the most frequent symbol pair is ``"u"`` followed by "g",
-occurring `10 + 5 + 5 = 20` times in total. Thus, the first merge rule the tokenizer learns is to group all ``"u"``
-symbols followed by a ``"g"`` symbol together. Next, "ug" is added to the vocabulary. The set of words then becomes
+We then take each pair of symbols and look at the most frequent. For instance 'hu' is present `10 + 5 = 15` times (10
+times in the 10 occurrences of 'hug', 5 times in the 5 occurrences of 'hugs'). The most frequent here is 'ug', present
+`10 + 5 + 5 = 20` times in total. So the first merge rule the tokenizer learns is to group all 'u' and 'g' together
+then it adds 'ug' to the vocabulary. Our corpus then becomes

 .. code-block::

-    ("h" "ug", 10), ("p" "ug", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "ug" "s", 5)
+    ('h' 'ug', 10), ('p' 'ug', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'ug' 's', 5)

-BPE then identifies the next most common symbol pair. It's ``"u"`` followed by ``"n"``, which occurs 16 times. ``"u"``,
-``"n"`` is merged to ``"un"`` and added to the vocabulary. The next most frequent symbol pair is ``"h"`` followed by
-``"ug"``, occurring 15 times. Again the pair is merged and ``"hug"`` can be added to the vocabulary.
+and we continue by looking at the next most common pair of symbols. It's 'un', present 16 times, so we merge those two
+and add 'un' to the vocabulary. Then it's 'hug' (as 'h' + 'ug'), present 15 times, so we merge those two and add 'hug'
+to the vocabulary.

-At this stage, the vocabulary is ``["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"]`` and our set of unique words
-is represented as
+At this stage, the vocabulary is ``['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']`` and our corpus is
+represented as

 .. code-block::

-    ("hug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("hug" "s", 5)
+    ('hug', 10), ('p' 'ug', 5), ('p' 'un', 12), ('b' 'un', 4), ('hug' 's', 5)

-Assuming, that the Byte-Pair Encoding training would stop at this point, the learned merge rules would then be applied
-to new words (as long as those new words do not include symbols that were not in the base vocabulary). For instance,
-the word ``"bug"`` would be tokenized to ``["b", "ug"]`` but ``"mug"`` would be tokenized as ``["<unk>", "ug"]`` since
-the symbol ``"m"`` is not in the base vocabulary. In general, single letters such as ``"m"`` are not replaced by the
-``"<unk>"`` symbol because the training data usually includes at least one occurrence of each letter, but it is likely
-to happen for very special characters like emojis.
+If we stop there, the tokenizer can apply the rules it learned to new words (as long as they don't contain characters
+that were not in the base vocabulary). For instance 'bug' would be tokenized as ``['b', 'ug']`` but mug would be
+tokenized as ``['<unk>', 'ug']`` since the 'm' is not in the base vocabulary. This doesn't happen to letters in general
+(since the base corpus uses all of them), but to special characters like emojis.

-As mentioned earlier, the vocabulary size, *i.e.* the base vocabulary size + the number of merges, is a hyperparameter
+As we said before, the vocabulary size (which is the base vocabulary size + the number of merges) is a hyperparameter
 to choose. For instance :doc:`GPT <model_doc/gpt>` has a vocabulary size of 40,478 since they have 478 base characters
-and chose to stop training after 40,000 merges.
+and chose to stop the training of the tokenizer at 40,000 merges.

 Byte-level BPE
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-A base vocabulary that includes all possible base characters can be quite large if *e.g.* all unicode characters are
-considered as base characters. To have a better base vocabulary, `GPT-2
-<https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf>`__ uses bytes
-as the base vocabulary, which is a clever trick to force the base vocabulary to be of size 256 while ensuring that
-every base character is included in the vocabulary. With some additional rules to deal with punctuation, the GPT2's
-tokenizer can tokenize every text without the need for the <unk> symbol. :doc:`GPT-2 <model_doc/gpt>` has a vocabulary
-size of 50,257, which corresponds to the 256 bytes base tokens, a special end-of-text token and the symbols learned
-with 50,000 merges.
+To deal with the fact the base vocabulary needs to get all base characters, which can be quite big if one allows for
+all unicode characters, the `GPT-2 paper
+<https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf>`__ introduces a
+clever trick, which is to use bytes as the base vocabulary (which gives a size of 256). With some additional rules to
+deal with punctuation, this manages to be able to tokenize every text without needing an unknown token. For instance,
+the :doc:`GPT-2 model <model_doc/gpt>` has a vocabulary size of 50,257, which corresponds to the 256 bytes base tokens,
+a special end-of-text token and the symbols learned with 50,000 merges.

 .. _wordpiece:

 WordPiece
 =======================================================================================================================

-WordPiece is the subword tokenization algorithm used for :doc:`BERT <model_doc/bert>`, :doc:`DistilBERT
-<model_doc/distilbert>`, and :doc:`Electra <model_doc/electra>`. The algorithm was outlined in `Japanese and Korean
-Voice Seach (Schuster et al., 2012)
-<https://static.googleusercontent.com/media/research.google.com/ja//pubs/archive/37842.pdf>`__ and is very similar to
-BPE. WordPiece first initializes the vocabulary to include every character present in the training data and
-progressively learn a given number of merge rules. In contrast to BPE, WordPiece does not choose the most frequent
-symbol pair, but the one that maximizes the likelihood of the training data once added to the vocabulary.
+WordPiece is the subword tokenization algorithm used for :doc:`BERT <model_doc/bert>` (as well as :doc:`DistilBERT
+<model_doc/distilbert>` and :doc:`Electra <model_doc/electra>`) and was outlined in `this paper
+<https://static.googleusercontent.com/media/research.google.com/ja//pubs/archive/37842.pdf>`__. It relies on the same
+base as BPE, which is to initialize the vocabulary to every character present in the corpus and progressively learn a
+given number of merge rules, the difference is that it doesn't choose the pair that is the most frequent but the one
+that will maximize the likelihood on the corpus once merged.

-So what does this mean exactly? Referring to the previous example, maximizing the likelihood of the training data is
-equivalent to finding the symbol pair, whose probability divided by the probabilities of its first symbol followed by
-its second symbol is the greatest among all symbol pairs. *E.g.* ``"u"``, followed by ``"g"`` would have only been
-merged if the probability of ``"ug"`` divided by ``"u"``, ``"g"`` would have been greater than for any other symbol
-pair. Intuitively, WordPiece is slightly different to BPE in that it evaluates what it `loses` by merging two symbols
-to make ensure it's `worth it`.
+What does this mean? Well, in the previous example, it means we would only merge 'u' and 'g' if the probability of
+having 'ug' divided by the probability of having 'u' then 'g' is greater than for any other pair of symbols. It's
+subtly different from what BPE does in the sense that it evaluates what it "loses" by merging two symbols and makes
+sure it's `worth it`.

 .. _unigram:

 Unigram
 =======================================================================================================================

-Unigram is a subword tokenization algorithm introduced in `Subword Regularization: Improving Neural Network Translation
-Models with Multiple Subword Candidates (Kudo, 2018) <https://arxiv.org/pdf/1804.10959.pdf>`__. In contrast to BPE or
-WordPiece, Unigram initializes its base vocabulary to a large number of symbols and progressively trims down each
-symbol to obtain a smaller vocabulary. The base vocabulary could for instance correspond to all pre-tokenized words and
-the most common substrings. Unigram is not used directly for any of the models in the transformers, but it's used in
-conjunction with :ref:`SentencePiece <sentencepiece>`.
+Unigram is a subword tokenization algorithm introduced in `this paper <https://arxiv.org/pdf/1804.10959.pdf>`__.
+Instead of starting with a group of base symbols and learning merges with some rule, like BPE or WordPiece, it starts
+from a large vocabulary (for instance, all pretokenized words and the most common substrings) that it will trim down
+progressively. It's not used directly for any of the pretrained models in the library, but it's used in conjunction
+with :ref:`SentencePiece <sentencepiece>`.

-At each training step, the Unigram algorithm defines a loss (often defined as the log-likelihood) over the training
-data given the current vocabulary and a unigram language model. Then, for each symbol in the vocabulary, the algorithm
-computes how much the overall loss would increase if the symbol was to be removed from the vocabulary. Unigram then
-removes p (with p usually being 10% or 20%) percent of the symbols whose loss increase is the lowest, *i.e.* those
-symbols that least affect the overall loss over the training data. This process is repeated until the vocabulary has
-reached the desired size. The Unigram algorithm always keeps the base characters so that any word can be tokenized.
+More specifically, at a given step, unigram computes a loss from the corpus we have and the current vocabulary, then,
+for each subword, evaluate how much the loss would increase if the subword was removed from the vocabulary. It then
+sorts the subwords by this quantity (that represents how much worse the loss becomes if the token is removed) and
+removes all the worst p tokens (for instance p could be 10% or 20%). It then repeats the process until the vocabulary
+has reached the desired size, always keeping the base characters (to be able to tokenize any word written with them,
+like BPE or WordPiece).

-Because Unigram is not based on merge rules (in contrast to BPE and WordPiece), the algorithm has several ways of
-tokenizing new text after training. As an example, if a trained Unigram tokenizer exhibits the vocabulary:
+Contrary to BPE and WordPiece that work out rules in a certain order that you can then apply in the same order when
+tokenizing new text, Unigram will have several ways of tokenizing a new text. For instance, if it ends up with the
+vocabulary

 .. code-block::

-    ["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"],
+    ['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']

-``"hugs"`` could be tokenized both as ``["hug", "s"]``, ``["h", "ug", "s"]`` or ``["h", "u", "g", "s"]``. So which one
-to choose? Unigram saves the probability of each token in the training corpus on top of saving the vocabulary so that
-the probability of each possible tokenization can be computed after training. The algorithm simply picks the most
-likely tokenization in practice, but also offers the possibility to sample a possible tokenization according to their
-probabilities.
+we had before, it could tokenize "hugs" as ``['hug', 's']``, ``['h', 'ug', 's']`` or ``['h', 'u', 'g', 's']``. So which
+one choose? On top of saving the vocabulary, the trained tokenizer will save the probability of each token in the
+training corpus. You can then give a probability to each tokenization (which is the product of the probabilities of the
+tokens forming it) and pick the most likely one (or if you want to apply some data augmentation, you could sample one
+of the tokenization according to their probabilities).

-Those probabilities are defined by the loss the tokenizer is trained on. Assuming that the training data consists of
-the words :math:`x_{1}, \dots, x_{N}` and that the set of all possible tokenizations for a word :math:`x_{i}` is
-defined as :math:`S(x_{i})`, then the overall loss is defined as
+Those probabilities define the loss that trains the tokenizer: if our corpus consists of the words :math:`x_{1}, \dots,
+x_{N}` and if for the word :math:`x_{i}` we note :math:`S(x_{i})` the set of all possible tokenizations of
+:math:`x_{i}` (with the current vocabulary), then the loss is defined as

 .. math::
    \mathcal{L} = -\sum_{i=1}^{N} \log \left ( \sum_{x \in S(x_{i})} p(x) \right )
@@ -247,18 +227,15 @@ defined as :math:`S(x_{i})`, then the overall loss is defined as
 SentencePiece
 =======================================================================================================================

-All tokenization algorithms described so far have the same problem: It is assumed that the input text uses spaces to
-separate words. However, not all languages use spaces to separate words. One possible solution is to use language
-specific pre-tokenizers, *e.g.* :doc:`XLM <model_doc/xlm>` uses a specific Chinese, Japanese, and Thai pre-tokenizer).
-To solve this problem more generally, `SentencePiece: A simple and language independent subword tokenizer and
-detokenizer for Neural Text Processing (Kudo et al., 2018) <https://arxiv.org/pdf/1808.06226.pdf>`__ treats the input
-as a raw input stream, thus including the space in the set of characters to use. It then uses the BPE or unigram
-algorithm to construct the appropriate vocabulary.
+All the methods we have been looking at so far required some form of pretokenization, which has a central problem: not
+all languages use spaces to separate words. This is a problem :doc:`XLM <model_doc/xlm>` solves by using specific
+pretokenizers for each of those languages (in this case, Chinese, Japanese and Thai). To solve this problem,
+SentencePiece (introduced in `this paper <https://arxiv.org/pdf/1808.06226.pdf>`__) treats the input as a raw stream,
+includes the space in the set of characters to use, then uses BPE or unigram to construct the appropriate vocabulary.

-The :class:`~transformers.XLNetTokenizer` uses SentencePiece for example, which is also why in the example earlier the
-``"▁"`` character was included in the vocabulary. Decoding with SentencePiece is very easy since all tokens can just be
-concatenated and ``"▁"`` is replaced by a space.
+That's why in the example we saw before using :class:`~transformers.XLNetTokenizer` (which uses SentencePiece), we had
+the '▁' character, that represents space. Decoding a tokenized text is then super easy: we just have to concatenate all
+of them together and replace '▁' with space.

-All transformers models in the library that use SentencePiece use it in combination with unigram. Examples of models
-using SentencePiece are :doc:`ALBERT <model_doc/albert>`, :doc:`XLNet <model_doc/xlnet>`, :doc:`Marian
-<model_doc/marian>`, and :doc:`T5 <model_doc/t5>`.
+All transformers models in the library that use SentencePiece use it with unigram. Examples of models using it are
+:doc:`ALBERT <model_doc/albert>`, :doc:`XLNet <model_doc/xlnet>` or the :doc:`Marian framework <model_doc/marian>`.
@@ -39,7 +39,7 @@ head on top of the encoder with an output size of 2. Models are initialized in `
 .. code-block:: python

    from transformers import BertForSequenceClassification
-    model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
+    model = BertForSequenceClassification.from_pretrained('bert-base-uncased', return_dict=True)
    model.train()

 This is useful because it allows us to make use of the pre-trained BERT encoder and easily train it on whatever
@@ -23,7 +23,6 @@ from typing import Dict, List, Optional
 import numpy as np
 import torch

-import transformers
 from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
@@ -34,7 +33,6 @@ from transformers import (
    default_data_collator,
    set_seed,
 )
-from transformers.trainer_utils import is_main_process
 from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels


@@ -57,8 +55,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -127,11 +124,6 @@ def main():
        bool(training_args.local_rank != -1),
        training_args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed
@@ -21,7 +21,7 @@ import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss

 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.albert.modeling_albert import (
+from transformers.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
@@ -23,7 +23,7 @@ from torch import nn
 from torch.nn import CrossEntropyLoss, MSELoss

 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.bert.modeling_bert import (
+from transformers.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
@@ -29,7 +29,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

-import transformers
 from pabee.modeling_pabee_albert import AlbertForSequenceClassificationWithPabee
 from pabee.modeling_pabee_bert import BertForSequenceClassificationWithPabee
 from transformers import (
@@ -45,7 +44,6 @@ from transformers import glue_compute_metrics as compute_metrics
 from transformers import glue_convert_examples_to_features as convert_examples_to_features
 from transformers import glue_output_modes as output_modes
 from transformers import glue_processors as processors
-from transformers.trainer_utils import is_main_process


 try:
@@ -476,7 +474,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -632,11 +630,7 @@ def main():
        bool(args.local_rank != -1),
        args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
    # Set seed
    set_seed(args)

@@ -4,7 +4,7 @@ import sys
 from unittest.mock import patch

 import run_glue_with_pabee
-from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me
+from transformers.testing_utils import TestCasePlus, require_torch_non_multigpu_but_fix_me


 logging.basicConfig(level=logging.DEBUG)
@@ -20,7 +20,7 @@ def get_setup_file():


 class PabeeTests(TestCasePlus):
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_run_glue(self):
        stream_handler = logging.StreamHandler(sys.stdout)
        logger.addHandler(stream_handler)
@@ -30,7 +30,6 @@ from torch.utils.data import DataLoader, SequentialSampler, Subset
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm

-import transformers
 from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
@@ -42,7 +41,6 @@ from transformers import (
    glue_processors,
    set_seed,
 )
-from transformers.trainer_utils import is_main_process


 logger = logging.getLogger(__name__)
@@ -298,7 +296,7 @@ def main():
        "--cache_dir",
        default=None,
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--data_subset", type=int, default=-1, help="If > 0: limit the data to a subset of data_subset instances."
@@ -370,11 +368,6 @@ def main():
    # Setup logging
    logging.basicConfig(level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
    logger.info("device: {} n_gpu: {}, distributed: {}".format(args.device, args.n_gpu, bool(args.local_rank != -1)))
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()

    # Set seeds
    set_seed(args.seed)
@@ -29,7 +29,6 @@ from typing import Optional

 from torch.utils.data import ConcatDataset

-import transformers
 from transformers import (
    CONFIG_MAPPING,
    MODEL_WITH_LM_HEAD_MAPPING,
@@ -48,7 +47,6 @@ from transformers import (
    TrainingArguments,
    set_seed,
 )
-from transformers.trainer_utils import is_main_process


 logger = logging.getLogger(__name__)
@@ -81,8 +79,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -222,11 +219,6 @@ def main():
        bool(training_args.local_rank != -1),
        training_args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed
@@ -31,7 +31,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

-import transformers
 from transformers import (
    WEIGHTS_NAME,
    AdamW,
@@ -42,7 +41,6 @@ from transformers import (
    MMBTForClassification,
    get_linear_schedule_with_warmup,
 )
-from transformers.trainer_utils import is_main_process
 from utils_mmimdb import ImageEncoder, JsonlDataset, collate_fn, get_image_transforms, get_mmimdb_labels


@@ -350,7 +348,7 @@ def main():
        "--cache_dir",
        default=None,
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -478,11 +476,7 @@ def main():
        bool(args.local_rank != -1),
        args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
    # Set seed
    set_seed(args)

@@ -1,6 +1,7 @@
 import torch

-from transformers import CamembertForMaskedLM, CamembertTokenizer
+from transformers.modeling_camembert import CamembertForMaskedLM
+from transformers.tokenization_camembert import CamembertTokenizer


 def fill_mask(masked_input, model, tokenizer, topk=5):
@@ -31,16 +31,8 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

-import transformers
-from transformers import (
-    WEIGHTS_NAME,
-    AdamW,
-    AutoConfig,
-    AutoModelForMultipleChoice,
-    AutoTokenizer,
-    get_linear_schedule_with_warmup,
-)
-from transformers.trainer_utils import is_main_process
+from transformers import WEIGHTS_NAME, AdamW, AutoConfig, AutoTokenizer, get_linear_schedule_with_warmup
+from transformers.modeling_auto import AutoModelForMultipleChoice


 try:
@@ -628,11 +620,6 @@ def main():
        bool(args.local_rank != -1),
        args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()

    # Set seed
    set_seed(args)
@@ -13,7 +13,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

-import transformers
 from src.modeling_highway_bert import DeeBertForSequenceClassification
 from src.modeling_highway_roberta import DeeRobertaForSequenceClassification
 from transformers import (
@@ -29,7 +28,6 @@ from transformers import glue_compute_metrics as compute_metrics
 from transformers import glue_convert_examples_to_features as convert_examples_to_features
 from transformers import glue_output_modes as output_modes
 from transformers import glue_processors as processors
-from transformers.trainer_utils import is_main_process


 try:
@@ -452,7 +450,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -582,11 +580,7 @@ def main():
        bool(args.local_rank != -1),
        args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
    # Set seed
    set_seed(args)

@@ -3,7 +3,7 @@ from torch import nn
 from torch.nn import CrossEntropyLoss, MSELoss

 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.bert.modeling_bert import (
+from transformers.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
@@ -3,13 +3,9 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss

-from transformers import RobertaConfig
+from transformers.configuration_roberta import RobertaConfig
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.roberta.modeling_roberta import (
-    ROBERTA_INPUTS_DOCSTRING,
-    ROBERTA_START_DOCSTRING,
-    RobertaEmbeddings,
-)
+from transformers.modeling_roberta import ROBERTA_INPUTS_DOCSTRING, ROBERTA_START_DOCSTRING, RobertaEmbeddings

 from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy

@@ -5,7 +5,7 @@ import unittest
 from unittest.mock import patch

 import run_glue_deebert
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow
+from transformers.testing_utils import require_torch_non_multigpu_but_fix_me, slow


 logging.basicConfig(level=logging.DEBUG)
@@ -26,7 +26,7 @@ class DeeBertTests(unittest.TestCase):
        logger.addHandler(stream_handler)

    @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_glue_deebert_train(self):

        train_args = """
@@ -30,7 +30,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

-import transformers
 from transformers import (
    WEIGHTS_NAME,
    AdamW,
@@ -58,7 +57,6 @@ from transformers.data.metrics.squad_metrics import (
    squad_evaluate,
 )
 from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor
-from transformers.trainer_utils import is_main_process


 try:
@@ -578,7 +576,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )

    parser.add_argument(
@@ -747,11 +745,7 @@ def main():
        bool(args.local_rank != -1),
        args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
    # Set seed
    set_seed(args)

@@ -76,8 +76,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -169,8 +168,6 @@ def main():
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed before initializing model.
@@ -257,7 +254,7 @@ def main():
        tokenize_function,
        batched=True,
        num_proc=data_args.preprocessing_num_workers,
-        remove_columns=column_names,
+        remove_columns=[text_column_name],
        load_from_cache_file=not data_args.overwrite_cache,
    )

@@ -313,12 +310,9 @@ def main():

    # Training
    if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
        )
-        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

    # Evaluation
@@ -74,8 +74,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -180,8 +179,6 @@ def main():
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed before initializing model.
@@ -295,7 +292,7 @@ def main():
            tokenize_function,
            batched=True,
            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=column_names,
+            remove_columns=[text_column_name],
            load_from_cache_file=not data_args.overwrite_cache,
        )

@@ -354,12 +351,9 @@ def main():

    # Training
    if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
        )
-        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

    # Evaluation
@@ -76,8 +76,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -187,8 +186,6 @@ def main():
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed before initializing model.
@@ -302,12 +299,9 @@ def main():

    # Training
    if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
        )
-        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

    # Evaluation
@@ -64,8 +64,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -177,8 +176,6 @@ def main():
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed before initializing model.
@@ -282,7 +279,7 @@ def main():
            tokenize_function,
            batched=True,
            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=column_names,
+            remove_columns=[text_column_name],
            load_from_cache_file=not data_args.overwrite_cache,
        )

@@ -344,12 +341,9 @@ def main():

    # Training
    if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
        )
-        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

    # Evaluation
@@ -4,7 +4,6 @@ import os
 from pathlib import Path
 from typing import Any, Dict

-import packaging
 import pytorch_lightning as pl
 from pytorch_lightning.utilities import rank_zero_info

@@ -34,17 +33,15 @@ from transformers.optimization import (

 logger = logging.getLogger(__name__)

+try:
+    pkg = "pytorch_lightning"
+    min_ver = "1.0.4"
+    pkg_resources.require(f"{pkg}>={min_ver}")
+except pkg_resources.VersionConflict:
+    logger.warning(
+        f"{pkg}>={min_ver} is required for a normal functioning of this module, but found {pkg}=={pkg_resources.get_distribution(pkg).version}. Try pip install -r examples/requirements.txt"
+    )

-def require_min_ver(pkg, min_ver):
-    got_ver = pkg_resources.get_distribution(pkg).version
-    if packaging.version.parse(got_ver) < packaging.version.parse(min_ver):
-        logger.warning(
-            f"{pkg}>={min_ver} is required for a normal functioning of this module, but found {pkg}=={got_ver}. "
-            "Try: pip install -r examples/requirements.txt"
-        )
-
-
-require_min_ver("pytorch_lightning", "1.0.4")

 MODEL_MODES = {
    "base": AutoModel,
@@ -236,7 +233,7 @@ class BaseTransformer(pl.LightningModule):
            "--cache_dir",
            default="",
            type=str,
-            help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+            help="Where do you want to store the pre-trained models downloaded from s3",
        )
        parser.add_argument(
            "--encoder_layerdrop",
@@ -210,6 +210,7 @@
    "        visual_feats=features,\n",
    "        visual_pos=normalized_boxes,\n",
    "        token_type_ids=inputs.token_type_ids,\n",
+    "        return_dict=True,\n",
    "        output_attentions=False,\n",
    "    )\n",
    "    output_vqa = lxmert_vqa(\n",
@@ -218,6 +219,7 @@
    "        visual_feats=features,\n",
    "        visual_pos=normalized_boxes,\n",
    "        token_type_ids=inputs.token_type_ids,\n",
+    "        return_dict=True,\n",
    "        output_attentions=False,\n",
    "    )\n",
    "    # get prediction\n",
@@ -264,4 +266,4 @@
 },
 "nbformat": 4,
 "nbformat_minor": 4
-}
+}
@@ -47,7 +47,7 @@ nbclient==0.5.0
 nbconvert==6.0.1
 nbformat==5.0.7
 nest-asyncio==1.4.0
-notebook==6.1.5
+notebook==6.1.4
 numpy==1.19.2
 opencv-python==4.4.0.42
 packaging==20.3
@@ -16,7 +16,7 @@
 """Masked Version of BERT. It replaces the `torch.nn.Linear` layers with
 :class:`~emmental.MaskedLinear` and add an additional parameters in the forward pass to
 compute the adaptive mask.
-Built on top of `transformers.models.bert.modeling_bert`"""
+Built on top of `transformers.modeling_bert`"""


 import logging
@@ -29,8 +29,8 @@ from torch.nn import CrossEntropyLoss, MSELoss
 from emmental import MaskedBertConfig
 from emmental.modules import MaskedLinear
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
+from transformers.modeling_bert import ACT2FN, BertLayerNorm, load_tf_weights_in_bert
 from transformers.modeling_utils import PreTrainedModel, prune_linear_layer
-from transformers.models.bert.modeling_bert import ACT2FN, BertLayerNorm, load_tf_weights_in_bert


 logger = logging.getLogger(__name__)
@@ -620,7 +620,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -725,7 +725,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )

    parser.add_argument(
@@ -23,7 +23,6 @@ from typing import Dict, Optional

 import numpy as np

-import transformers
 from transformers import (
    AutoConfig,
    AutoModelForMultipleChoice,
@@ -34,7 +33,6 @@ from transformers import (
    TrainingArguments,
    set_seed,
 )
-from transformers.trainer_utils import is_main_process
 from utils_multiple_choice import MultipleChoiceDataset, Split, processors


@@ -61,8 +59,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -118,11 +115,6 @@ def main():
        bool(training_args.local_rank != -1),
        training_args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed
@@ -33,15 +33,9 @@ from transformers import (
    TFTrainingArguments,
    set_seed,
 )
-from transformers.utils import logging as hf_logging
 from utils_multiple_choice import Split, TFMultipleChoiceDataset, processors


-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()
-
-
 logger = logging.getLogger(__name__)


@@ -65,8 +59,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -29,7 +29,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

-import transformers
 from transformers import (
    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
    WEIGHTS_NAME,
@@ -46,7 +45,6 @@ from transformers.data.metrics.squad_metrics import (
    squad_evaluate,
 )
 from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor
-from transformers.trainer_utils import is_main_process


 try:
@@ -321,7 +319,7 @@ def evaluate(args, model, tokenizer, prefix=""):
            eval_feature = features[feature_index.item()]
            unique_id = int(eval_feature.unique_id)

-            output = [to_list(output[i]) for output in outputs.to_tuple()]
+            output = [to_list(output[i]) for output in outputs]

            # Some models (XLNet, XLM) use 5 arguments for their predictions, while the other "simpler"
            # models only use two.
@@ -532,7 +530,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )

    parser.add_argument(
@@ -714,11 +712,7 @@ def main():
        bool(args.local_rank != -1),
        args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
    # Set seed
    set_seed(args)

@@ -736,7 +730,6 @@ def main():
        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
        do_lower_case=args.do_lower_case,
        cache_dir=args.cache_dir if args.cache_dir else None,
-        use_fast=False,  # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
    )
    model = AutoModelForQuestionAnswering.from_pretrained(
        args.model_name_or_path,
@@ -785,10 +778,7 @@ def main():

        # Load a trained model and vocabulary that you have fine-tuned
        model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)  # , force_download=True)
-
-        # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
-        # So we use use_fast=False here for now until Fast-tokenizer-compatible-examples are out
-        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case, use_fast=False)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
        model.to(args.device)

    # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory
@@ -22,11 +22,9 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional

-import transformers
 from transformers import AutoConfig, AutoModelForQuestionAnswering, AutoTokenizer, HfArgumentParser, SquadDataset
 from transformers import SquadDataTrainingArguments as DataTrainingArguments
 from transformers import Trainer, TrainingArguments
-from transformers.trainer_utils import is_main_process


 logger = logging.getLogger(__name__)
@@ -51,8 +49,7 @@ class ModelArguments:
    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
    # or just modify its tokenizer_config.json.
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -94,11 +91,6 @@ def main():
        bool(training_args.local_rank != -1),
        training_args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Prepare Question-Answering task
@@ -115,7 +107,6 @@ def main():
    tokenizer = AutoTokenizer.from_pretrained(
        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
        cache_dir=model_args.cache_dir,
-        use_fast=False,  # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
    )
    model = AutoModelForQuestionAnswering.from_pretrained(
        model_args.model_name_or_path,
@@ -33,12 +33,6 @@ from transformers import (
    squad_convert_examples_to_features,
 )
 from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor
-from transformers.utils import logging as hf_logging
-
-
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()


 logger = logging.getLogger(__name__)
@@ -63,8 +57,7 @@ class ModelArguments:
    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
    # or just modify its tokenizer_config.json.
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -27,7 +27,7 @@ class RagPyTorchDistributedRetriever(RagRetriever):
            It is used to decode the question and then use the generator_tokenizer.
        generator_tokenizer (:class:`~transformers.PretrainedTokenizer`):
            The tokenizer used for the generator part of the RagModel.
-        index (:class:`~transformers.models.rag.retrieval_rag.Index`, optional, defaults to the one defined by the configuration):
+        index (:class:`~transformers.retrieval_rag.Index`, optional, defaults to the one defined by the configuration):
            If specified, use this index instead of the one built using the configuration
    """

@@ -95,7 +95,7 @@ def evaluate_batch_retrieval(args, rag_model, questions):
        truncation=True,
    )["input_ids"].to(args.device)

-    question_enc_outputs = rag_model.rag.question_encoder(retriever_input_ids)
+    question_enc_outputs = rag_model.rag.question_encoder(retriever_input_ids, return_dict=True)
    question_enc_pool_output = question_enc_outputs.pooler_output

    result = rag_model.retriever(
@@ -204,6 +204,7 @@ class GenerativeQAModule(BaseTransformer):
            decoder_input_ids=decoder_input_ids,
            use_cache=False,
            labels=lm_labels,
+            return_dict=True,
            **rag_kwargs,
        )

@@ -7,7 +7,7 @@ export PYTHONPATH="../":"${PYTHONPATH}"
 python examples/rag/finetune.py \
    --data_dir $DATA_DIR \
    --output_dir $OUTPUT_DIR \
-    --model_name_or_path $MODEL_NAME_OR_PATH \
+    --model_name_or_path $MODLE_NAME_OR_PATH \
    --model_type rag_sequence \
    --fp16 \
    --gpus 8 \
@@ -11,12 +11,16 @@ import numpy as np
 from datasets import Dataset

 import faiss
-from transformers import BartConfig, BartTokenizer, DPRConfig, DPRQuestionEncoderTokenizer, RagConfig
+from transformers.configuration_bart import BartConfig
+from transformers.configuration_dpr import DPRConfig
+from transformers.configuration_rag import RagConfig
 from transformers.file_utils import is_datasets_available, is_faiss_available, is_psutil_available, is_torch_available
-from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
-from transformers.models.rag.retrieval_rag import CustomHFIndex
-from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me
+from transformers.retrieval_rag import CustomHFIndex
+from transformers.testing_utils import require_torch_non_multigpu_but_fix_me
+from transformers.tokenization_bart import BartTokenizer
+from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
+from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer
+from transformers.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES


 sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip
@@ -133,7 +137,7 @@ class RagRetrieverTest(TestCase):
            question_encoder=DPRConfig().to_dict(),
            generator=BartConfig().to_dict(),
        )
-        with patch("transformers.models.rag.retrieval_rag.load_dataset") as mock_load_dataset:
+        with patch("transformers.retrieval_rag.load_dataset") as mock_load_dataset:
            mock_load_dataset.return_value = dataset
            retriever = RagPyTorchDistributedRetriever(
                config,
@@ -175,7 +179,7 @@ class RagRetrieverTest(TestCase):
            retriever.init_retrieval(port)
        return retriever

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_pytorch_distributed_retriever_retrieve(self):
        n_docs = 1
        retriever = self.get_dummy_pytorch_distributed_retriever(init_retrieval=True)
@@ -191,7 +195,7 @@ class RagRetrieverTest(TestCase):
        self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
        self.assertListEqual(doc_ids.tolist(), [[1], [0]])

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_custom_hf_index_retriever_retrieve(self):
        n_docs = 1
        retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=False)
@@ -207,7 +211,7 @@ class RagRetrieverTest(TestCase):
        self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
        self.assertListEqual(doc_ids.tolist(), [[1], [0]])

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_custom_pytorch_distributed_retriever_retrieve_from_disk(self):
        n_docs = 1
        retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=True)
@@ -47,7 +47,7 @@ def embed(documents: dict, ctx_encoder: DPRContextEncoder, ctx_tokenizer: DPRCon
    input_ids = ctx_tokenizer(
        documents["title"], documents["text"], truncation=True, padding="longest", return_tensors="pt"
    )["input_ids"]
-    embeddings = ctx_encoder(input_ids.to(device=device)).pooler_output
+    embeddings = ctx_encoder(input_ids.to(device=device), return_dict=True).pooler_output
    return {"embeddings": embeddings.detach().cpu().numpy()}


@@ -18,4 +18,3 @@ fire
 pytest
 conllu
 sentencepiece != 0.1.92
-protobuf
@@ -380,7 +380,7 @@ cp xsum/test* all_pl
 then use `all_pl` as DATA in the command above.

 #### Direct Knowledge Distillation (KD)
-+ In this method, we use try to enforce that the student and teacher produce similar encoder_outputs, logits, and hidden_states using `SummarizationDistiller`.
+ In this method, we use try to enforce that the student and teacher produce similar encoder_outputs, logits, and hidden_states using `BartSummarizationDistiller`.
 + This method was used for `sshleifer/distilbart-xsum-12-6`, `6-6`, and `9-6` checkpoints were produced.
 + You must use [`distillation.py`](./distillation.py). Note that this command initializes the student for you.

@@ -16,7 +16,7 @@ from finetune import SummarizationModule, TranslationModule
 from finetune import main as ft_main
 from make_student import create_student_by_copying_alternating_layers, get_layers_to_supervise
 from transformers import AutoModelForSeq2SeqLM, MBartTokenizer, T5ForConditionalGeneration
-from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.modeling_bart import shift_tokens_right
 from utils import calculate_bleu, check_output_dir, freeze_params, label_smoothed_nll_loss, use_task_specific_params


@@ -25,8 +25,8 @@ sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
 from lightning_base import generic_train  # noqa


-class SummarizationDistiller(SummarizationModule):
-    """Supports T5, Bart, Pegasus and other models that inherit from Bart."""
+class BartSummarizationDistiller(SummarizationModule):
+    """Supports Bart, Pegasus and other models that inherit from Bart."""

    loss_names = ["loss", "ce_loss", "mlm_loss", "hid_loss_enc", "hid_loss_dec"]

@@ -40,38 +40,25 @@ class SummarizationDistiller(SummarizationModule):
        hparams.model_name_or_path = str(save_dir)  # Tell lightning we are training the student
        teacher = AutoModelForSeq2SeqLM.from_pretrained(hparams.teacher).eval()
        use_task_specific_params(teacher, hparams.task)  # We copy good generation parameters to student by default
-        if hparams.student is not None:
-            student = AutoModelForSeq2SeqLM.from_pretrained(hparams.student)
-            use_task_specific_params(student, hparams.task)
-            e_layer_ids, d_layer_ids = None, None
-        else:
-            student, e_layer_ids, d_layer_ids = create_student_by_copying_alternating_layers(
-                teacher, e=hparams.student_encoder_layers, d=hparams.student_decoder_layers, save_path=save_dir
-            )
-
+        student, e_layer_ids, d_layer_ids = create_student_by_copying_alternating_layers(
+            teacher, e=hparams.student_encoder_layers, d=hparams.student_decoder_layers, save_path=save_dir
+        )
        if hparams.length_penalty != -1:
            student.config.length_penalty = hparams.length_penalty
-        hparams.tokenizer_name = hparams.teacher  # Use teacher's tokenizer
        super().__init__(hparams, model=student, config=student.config)
-        assert (
-            student.config.model_type == teacher.config.model_type
-        ), f"teacher, student model types should be the same, got {student.config.model_type} != {teacher.config.model_type}"
+        model_type = student.config.model_type
+        self.e_layer_ids, self.d_layer_ids = e_layer_ids, d_layer_ids  # type: List[int], List[int]

-        if student.config.model_type == "t5":
-            student_encoder_layers = len(student.get_encoder().block)
-            student_decoder_layers = len(student.get_decoder().block)
+        if model_type == "t5":
            teacher_encoder_layers = len(teacher.get_encoder().block)
            teacher_decoder_layers = len(teacher.get_decoder().block)
        else:
-            student_encoder_layers = student.config.encoder_layers
-            student_decoder_layers = student.config.decoder_layers
            teacher_encoder_layers = teacher.config.encoder_layers
            teacher_decoder_layers = teacher.config.decoder_layers

-        self.different_base_models = not (hparams.student is None or hparams.teacher == hparams.student)
-        self.do_calc_hidden_loss = (not self.different_base_models) and hparams.alpha_hid > 0
-        self.different_encoder = self.different_base_models or (student_encoder_layers != teacher_encoder_layers)
-        # self.different_encoder determines whether we need to run the teacher encoder
+        self.different_encoder = hparams.student_encoder_layers != teacher_encoder_layers
+        self.different_decoder = hparams.student_decoder_layers != teacher_decoder_layers
+
        self.teacher = teacher
        freeze_params(self.teacher)

@@ -80,28 +67,13 @@ class SummarizationDistiller(SummarizationModule):
                del self.teacher.model.encoder
            except AttributeError:  # T5
                del self.teacher.encoder
-
-        if e_layer_ids is None:
-            e_layer_ids = list(range(student_encoder_layers))
-        if d_layer_ids is None:
-            d_layer_ids = list(range(student_decoder_layers))
-
-        self.e_layer_ids, self.d_layer_ids = e_layer_ids, d_layer_ids  # type: List[int], List[int]
-
-        if self.do_calc_hidden_loss:  # Intermediate supervision: Decide which layers to supervise
-            if hparams.supervise_forward:
-                self.e_matches = get_layers_to_supervise(
-                    n_student=len(self.e_layer_ids), n_teacher=teacher_encoder_layers
-                )
-                self.d_matches = get_layers_to_supervise(
-                    n_student=len(self.d_layer_ids), n_teacher=teacher_decoder_layers
-                )
-            else:  # student layer should emulate hidden states of the teacher layer it was copied from
-                self.e_matches = self.e_layer_ids
-                self.d_matches = self.d_layer_ids
-        else:
-            self.e_matches = None
-            self.d_matches = None
+        # Intermediate supervision: Decide which layers to supervise
+        if hparams.supervise_forward:
+            self.e_matches = get_layers_to_supervise(n_student=len(self.e_layer_ids), n_teacher=teacher_encoder_layers)
+            self.d_matches = get_layers_to_supervise(n_student=len(self.d_layer_ids), n_teacher=teacher_decoder_layers)
+        else:  # student layer should emulate hidden states of the teacher layer it was copied from
+            self.e_matches = self.e_layer_ids
+            self.d_matches = self.d_layer_ids

        self.ce_loss_fct = nn.KLDivLoss(reduction="batchmean")
        self.temperature = 2.0
@@ -111,8 +83,22 @@ class SummarizationDistiller(SummarizationModule):
        gc.collect()
        torch.cuda.empty_cache()

+    def calc_mse_loss(self, teacher_outputs: torch.Tensor, student_outputs: torch.Tensor, mask) -> torch.FloatTensor:
+        """Supervise MSE(teacher.encoder_outputs, student.encoder_outputs)."""
+        # raise NotImplementedError()
+        if mask is not None:
+            # mask has False at padding_idx
+            sel_mask = mask[:, :, None].expand_as(student_outputs).bool()
+            s_logits_slct = torch.masked_select(student_outputs, sel_mask)
+            t_logits_slct = torch.masked_select(teacher_outputs, sel_mask)
+        else:
+            t_logits_slct = teacher_outputs
+            s_logits_slct = student_outputs
+        return F.mse_loss(s_logits_slct, t_logits_slct)
+
    def calc_ce_loss(self, mask, s_logits, t_logits):
        """Copy pasted from distillbert (transformers/examples/distillation/)"""
+
        # mask has False at padding_idx
        sel_mask = mask[:, :, None].expand_as(s_logits)
        vocab_size = s_logits.size(-1)
@@ -136,8 +122,8 @@ class SummarizationDistiller(SummarizationModule):
        add_distill_args(parser)
        return parser

-    def _step(self, batch: dict) -> tuple:
-        """Compute the loss for a batch"""
+    def _step(self, batch):
+        # assert is_frozen(self.teacher) copied_decoder_layers
        pad_token_id = self.tokenizer.pad_token_id
        input_ids, src_mask, labels = batch["input_ids"], batch["attention_mask"], batch["labels"]
        if isinstance(self.model, T5ForConditionalGeneration):
@@ -146,15 +132,14 @@ class SummarizationDistiller(SummarizationModule):
            decoder_input_ids = shift_tokens_right(labels, pad_token_id)

        # noinspection PyCallingNonCallable
-        student_outputs = self(
+        lm_logits, dec_hidden, enc_outputs, enc_hidden_state = self(
            input_ids,
            attention_mask=src_mask,
            decoder_input_ids=decoder_input_ids,
-            output_hidden_states=self.do_calc_hidden_loss,
+            output_hidden_states=True,
            output_attentions=False,
            use_cache=False,
        )
-        lm_logits = student_outputs["logits"]

        # Same cross entropy vs. label smoothing logic as finetune.py
        assert lm_logits.shape[-1] == self.model.config.vocab_size
@@ -163,7 +148,7 @@ class SummarizationDistiller(SummarizationModule):
            loss_fct = torch.nn.CrossEntropyLoss(ignore_index=pad_token_id)
            student_lm_loss = loss_fct(lm_logits.view(-1, lm_logits.shape[-1]), labels.view(-1))
        else:
-            lprobs = F.log_softmax(lm_logits, dim=-1)
+            lprobs = torch.nn.functional.log_softmax(lm_logits, dim=-1)
            student_lm_loss, _ = label_smoothed_nll_loss(
                lprobs, labels, self.hparams.label_smoothing, ignore_index=pad_token_id
            )
@@ -171,44 +156,37 @@ class SummarizationDistiller(SummarizationModule):
        def zero_tensor():
            return torch.tensor(0.0).type_as(student_lm_loss)

-        teacher_enc_outputs = student_outputs[
-            "encoder_last_hidden_state"
-        ]  # use this unless self.different_base_models
        hid_loss_enc, hid_loss_dec = zero_tensor(), zero_tensor()
        if self.different_encoder:  # compute encoder hidden state loss
-            all_teacher_encoder_outputs = self.teacher.get_encoder()(
+            with torch.no_grad():
+                teacher_enc_hid = self.teacher.get_encoder()(
+                    input_ids, attention_mask=src_mask, output_hidden_states=True, return_dict=True
+                ).hidden_states
+
+            hid_loss_enc = self.calc_hidden_loss(
+                src_mask,
+                enc_hidden_state,
+                teacher_enc_hid,
+                self.e_matches,
+                normalize_hidden=self.hparams.normalize_hidden,
+            )
+
+        with torch.no_grad():
+            outputs = self.teacher(
                input_ids,
                attention_mask=src_mask,
-                output_hidden_states=self.do_calc_hidden_loss,
+                encoder_outputs=(enc_outputs,),
+                decoder_input_ids=decoder_input_ids,
+                lm_labels=labels,
+                output_hidden_states=True,
+                return_dict=True,
            )
-            if self.different_base_models:
-                teacher_enc_outputs = all_teacher_encoder_outputs["last_hidden_state"]
-            elif self.do_calc_hidden_loss:
-                hid_loss_enc = self.calc_hidden_loss(
-                    src_mask,
-                    student_outputs["encoder_hidden_states"],
-                    all_teacher_encoder_outputs["hidden_states"],
-                    self.e_matches,
-                    normalize_hidden=self.hparams.normalize_hidden,
-                )
-
-        teacher_outputs = self.teacher(
-            input_ids,
-            attention_mask=src_mask,
-            encoder_outputs=(teacher_enc_outputs,),
-            decoder_input_ids=decoder_input_ids,
-            output_hidden_states=self.do_calc_hidden_loss,
-            use_cache=False,  # since we are not passing labels, never let this default to True
-        )
+            tlogits, tdec_hidden = outputs.logits, outputs.decoder_hidden_states
        dec_mask = decoder_input_ids.ne(pad_token_id)
-        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, teacher_outputs["logits"])
-        if self.do_calc_hidden_loss:  # Intermediate supervision of decoder hidden states
+        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, tlogits)
+        if self.alpha_hid > 0:  # Intermediate supervision of decoder hidden states
            hid_loss_dec = self.calc_hidden_loss(
-                dec_mask,
-                student_outputs["decoder_hidden_states"],
-                teacher_outputs["decoder_hidden_states"],
-                self.d_matches,
-                normalize_hidden=self.hparams.normalize_hidden,
+                dec_mask, dec_hidden, tdec_hidden, self.d_matches, normalize_hidden=self.hparams.normalize_hidden
            )

        blended_loss = (
@@ -228,7 +206,6 @@ class SummarizationDistiller(SummarizationModule):
        valid_count = mask.sum() * hidden_states[0].size(-1)
        student_states = torch.stack([hidden_states[i] for i in range(len(matches))])
        teacher_states = torch.stack([hidden_states_T[j] for j in matches])
-        assert student_states.shape == teacher_states.shape, f"{student_states.shape} != {teacher_states.shape}"
        if normalize_hidden:
            student_states = F.layer_norm(student_states, student_states.shape[1:])
            teacher_states = F.layer_norm(teacher_states, teacher_states.shape[1:])
@@ -238,16 +215,10 @@ class SummarizationDistiller(SummarizationModule):


 def add_distill_args(parser):
-    # NOTE: if --student argument was specified and the teacher and student base models
-    # are different, the models still have to have the same tokenizer, specified by
-    # --tokenizer_name. So, for example, you can distill from t5_large to t5_small but not
-    # from bart to t5. This s because if the tokenizers are different, the output space
-    # for the two models is also different and their logits are not comparable.
    parser.add_argument("--teacher", type=str)
    parser.add_argument("--alpha_ce", default=0.8, type=float)
    parser.add_argument("--alpha_mlm", default=0.2, type=float)
    parser.add_argument("--alpha_hid", default=0.0, type=float, required=False)
-    parser.add_argument("--student", type=str, required=False)
    parser.add_argument("--student_decoder_layers", default=12, type=int, required=False)
    parser.add_argument("--student_encoder_layers", default=12, type=int, required=False)
    parser.add_argument("--no_teacher", action="store_true", default=False)
@@ -256,8 +227,8 @@ def add_distill_args(parser):
    parser.add_argument("--normalize_hidden", action="store_true", default=False)


-class TranslationDistiller(SummarizationDistiller):
-    """Supports T5, mBART, Marian, other models that inherit from Bart."""
+class BartTranslationDistiller(BartSummarizationDistiller):
+    """Supports Mbart, Marian, other models that inherit from Bart."""

    mode = "translation"
    metric_names = ["bleu"]
@@ -286,7 +257,7 @@ def create_module(args):
    if args.no_teacher:
        module_cls = TranslationModule if "translation" in args.task else SummarizationModule
    else:  # DISTILL WITH TEACHER
-        module_cls = TranslationDistiller if "translation" in args.task else SummarizationDistiller
+        module_cls = BartTranslationDistiller if "translation" in args.task else BartSummarizationDistiller
    args.setup_cls: str = module_cls.__name__
    print(f"using module {args.setup_cls}")
    model = module_cls(args)
@@ -304,7 +275,7 @@ def distill_main(args):
 if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser = pl.Trainer.add_argparse_args(parser)
-    parser = SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+    parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
    args = parser.parse_args()

    distill_main(args)
@@ -17,7 +17,7 @@ from torch.utils.data import DataLoader

 from callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
 from transformers import MBartTokenizer, T5ForConditionalGeneration
-from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.modeling_bart import shift_tokens_right
 from utils import (
    ROUGE_KEYS,
    LegacySeq2SeqDataset,
@@ -148,7 +148,7 @@ class SummarizationModule(BaseTransformer):
            self.save_readable_batch(batch)

        outputs = self(src_ids, attention_mask=src_mask, decoder_input_ids=decoder_input_ids, use_cache=False)
-        lm_logits = outputs["logits"]
+        lm_logits = outputs[0]
        if self.hparams.label_smoothing == 0:
            # Same behavior as modeling_bart.py, besides ignoring pad_token_id
            ce_loss_fct = torch.nn.CrossEntropyLoss(ignore_index=pad_token_id)
@@ -4,11 +4,10 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional

-import transformers
 from seq2seq_trainer import Seq2SeqTrainer
 from seq2seq_training_args import Seq2SeqTrainingArguments
 from transformers import AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer, HfArgumentParser, MBartTokenizer, set_seed
-from transformers.trainer_utils import EvaluationStrategy, is_main_process
+from transformers.trainer_utils import EvaluationStrategy
 from utils import (
    Seq2SeqDataCollator,
    Seq2SeqDataset,
@@ -43,8 +42,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    freeze_encoder: bool = field(default=False, metadata={"help": "Whether tp freeze the encoder."})
    freeze_embeds: bool = field(default=False, metadata={"help": "Whether  to freeze the embeddings."})
@@ -133,11 +131,6 @@ def main():
        bool(training_args.local_rank != -1),
        training_args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed
@@ -5,8 +5,8 @@ from torch import nn
 from torch.utils.data import DistributedSampler, RandomSampler

 from transformers import PreTrainedModel, Trainer, logging
+from transformers.configuration_fsmt import FSMTConfig
 from transformers.file_utils import is_torch_tpu_available
-from transformers.models.fsmt.configuration_fsmt import FSMTConfig
 from transformers.optimization import (
    Adafactor,
    AdamW,
@@ -9,11 +9,11 @@ import pytorch_lightning as pl
 import timeout_decorator
 import torch

-from distillation import SummarizationDistiller, distill_main
+from distillation import BartSummarizationDistiller, distill_main
 from finetune import SummarizationModule, main
 from transformers import MarianMTModel
 from transformers.file_utils import cached_path
-from transformers.testing_utils import TestCasePlus, require_torch_gpu, slow
+from transformers.testing_utils import TestCasePlus, require_torch_gpu, require_torch_non_multigpu_but_fix_me, slow
 from utils import load_json


@@ -32,6 +32,7 @@ class TestMbartCc25Enro(TestCasePlus):

    @slow
    @require_torch_gpu
+    @require_torch_non_multigpu_but_fix_me
    def test_model_download(self):
        """This warms up the cache so that we can time the next test without including download time, which varies between machines."""
        MarianMTModel.from_pretrained(MARIAN_MODEL)
@@ -39,6 +40,7 @@ class TestMbartCc25Enro(TestCasePlus):
    # @timeout_decorator.timeout(1200)
    @slow
    @require_torch_gpu
+    @require_torch_non_multigpu_but_fix_me
    def test_train_mbart_cc25_enro_script(self):
        env_vars_to_replace = {
            "$MAX_LEN": 64,
@@ -73,7 +75,7 @@ class TestMbartCc25Enro(TestCasePlus):
            --num_sanity_val_steps 0
            --eval_beams 2
        """.split()
-        # XXX: args.gpus > 1 : handle multi_gpu in the future
+        # XXX: args.gpus > 1 : handle multigpu in the future

        testargs = ["finetune.py"] + bash_script.split() + args
        with patch.object(sys, "argv", testargs):
@@ -127,6 +129,7 @@ class TestDistilMarianNoTeacher(TestCasePlus):
    @timeout_decorator.timeout(600)
    @slow
    @require_torch_gpu
+    @require_torch_non_multigpu_but_fix_me
    def test_opus_mt_distill_script(self):
        data_dir = f"{self.test_file_dir_str}/test_data/wmt_en_ro"
        env_vars_to_replace = {
@@ -167,9 +170,9 @@ class TestDistilMarianNoTeacher(TestCasePlus):
        with patch.object(sys, "argv", testargs):
            parser = argparse.ArgumentParser()
            parser = pl.Trainer.add_argparse_args(parser)
-            parser = SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+            parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
            args = parser.parse_args()
-            # assert args.gpus == gpus THIS BREAKS for multi_gpu
+            # assert args.gpus == gpus THIS BREAKS for multigpu

            model = distill_main(args)

@@ -10,8 +10,8 @@ from parameterized import parameterized
 from save_len_file import save_len_file
 from test_seq2seq_examples import ARTICLES, BART_TINY, MARIAN_TINY, MBART_TINY, SUMMARIES, T5_TINY, make_test_data_dir
 from transformers import AutoTokenizer
-from transformers.models.bart.modeling_bart import shift_tokens_right
-from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me, slow
+from transformers.modeling_bart import shift_tokens_right
+from transformers.testing_utils import TestCasePlus, require_torch_non_multigpu_but_fix_me, slow
 from utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeq2SeqDataset, Seq2SeqDataset


@@ -30,7 +30,7 @@ class TestAll(TestCasePlus):
        ],
    )
    @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_seq2seq_dataset_truncation(self, tok_name):
        tokenizer = AutoTokenizer.from_pretrained(tok_name)
        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
@@ -70,7 +70,7 @@ class TestAll(TestCasePlus):
            break  # No need to test every batch

    @parameterized.expand([BART_TINY, BERT_BASE_CASED])
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_legacy_dataset_truncation(self, tok):
        tokenizer = AutoTokenizer.from_pretrained(tok)
        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
@@ -95,7 +95,7 @@ class TestAll(TestCasePlus):
            assert max_len_target > trunc_target  # Truncated
            break  # No need to test every batch

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_pack_dataset(self):
        tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25")

@@ -114,7 +114,7 @@ class TestAll(TestCasePlus):
        assert orig_paths == new_paths

    @pytest.mark.skipif(not FAIRSEQ_AVAILABLE, reason="This test requires fairseq")
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_dynamic_batch_size(self):
        if not FAIRSEQ_AVAILABLE:
            return
@@ -139,7 +139,7 @@ class TestAll(TestCasePlus):
        if failures:
            raise AssertionError(f"too many tokens in {len(failures)} batches")

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_sortish_sampler_reduces_padding(self):
        ds, _, tokenizer = self._get_dataset(max_len=512)
        bs = 2
@@ -179,7 +179,7 @@ class TestAll(TestCasePlus):
        )
        return ds, max_tokens, tokenizer

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_distributed_sortish_sampler_splits_indices_between_procs(self):
        ds, max_tokens, tokenizer = self._get_dataset()
        ids1 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=0, add_extra_examples=False))
@@ -195,9 +195,9 @@ class TestAll(TestCasePlus):
            PEGASUS_XSUM,
        ],
    )
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_dataset_kwargs(self, tok_name):
-        tokenizer = AutoTokenizer.from_pretrained(tok_name, use_fast=False)
+        tokenizer = AutoTokenizer.from_pretrained(tok_name)
        if tok_name == MBART_TINY:
            train_dataset = Seq2SeqDataset(
                tokenizer,
@@ -4,13 +4,7 @@ from unittest.mock import patch

 from transformers import BertTokenizer, EncoderDecoderModel
 from transformers.file_utils import is_datasets_available
-from transformers.testing_utils import (
-    TestCasePlus,
-    execute_subprocess_async,
-    get_gpu_count,
-    require_torch_non_multi_gpu_but_fix_me,
-    slow,
-)
+from transformers.testing_utils import TestCasePlus, execute_subprocess_async, get_gpu_count, slow
 from transformers.trainer_callback import TrainerState
 from transformers.trainer_utils import set_seed

@@ -52,7 +46,6 @@ class TestFinetuneTrainer(TestCasePlus):
        assert "test_results.json" in contents

    @slow
-    @require_torch_non_multi_gpu_but_fix_me
    def test_finetune_bert2bert(self):
        if not is_datasets_available():
            return
@@ -145,7 +138,7 @@ class TestFinetuneTrainer(TestCasePlus):
            per_device_train_batch_size=batch_size,
            per_device_eval_batch_size=batch_size,
            predict_with_generate=True,
-            evaluation_strategy="steps",
+            evaluate_during_training=True,
            do_train=True,
            do_eval=True,
            warmup_steps=0,
@@ -186,7 +179,7 @@ class TestFinetuneTrainer(TestCasePlus):
            --per_device_eval_batch_size 4
            --learning_rate 3e-3
            --warmup_steps 8
-            --evaluation_strategy steps
+            --evaluate_during_training
            --predict_with_generate
            --logging_steps 0
            --save_steps {str(eval_steps)}
@@ -19,7 +19,13 @@ import unittest

 from parameterized import parameterized
 from transformers import FSMTForConditionalGeneration, FSMTTokenizer
-from transformers.testing_utils import get_tests_dir, require_torch, slow, torch_device
+from transformers.testing_utils import (
+    get_tests_dir,
+    require_torch,
+    require_torch_non_multigpu_but_fix_me,
+    slow,
+    torch_device,
+)
 from utils import calculate_bleu


@@ -48,6 +54,7 @@ class ModelEvalTester(unittest.TestCase):
        ]
    )
    @slow
+    @require_torch_non_multigpu_but_fix_me
    def test_bleu_scores(self, pair, min_bleu_score):
        # note: this test is not testing the best performance since it only evals a small batch
        # but it should be enough to detect a regression in the output quality
@@ -4,7 +4,7 @@ import unittest
 from make_student import create_student_by_copying_alternating_layers
 from transformers import AutoConfig
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_torch, require_torch_non_multi_gpu_but_fix_me
+from transformers.testing_utils import require_torch, require_torch_non_multigpu_but_fix_me


 TINY_BART = "sshleifer/bart-tiny-random"
@@ -17,28 +17,28 @@ class MakeStudentTester(unittest.TestCase):
    def teacher_config(self):
        return AutoConfig.from_pretrained(TINY_BART)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_valid_t5(self):
        student, *_ = create_student_by_copying_alternating_layers(TINY_T5, tempfile.mkdtemp(), e=1, d=1)
        self.assertEqual(student.config.num_hidden_layers, 1)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_asymmetric_t5(self):
        student, *_ = create_student_by_copying_alternating_layers(TINY_T5, tempfile.mkdtemp(), e=1, d=None)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_same_decoder_small_encoder(self):
        student, *_ = create_student_by_copying_alternating_layers(TINY_BART, tempfile.mkdtemp(), e=1, d=None)
        self.assertEqual(student.config.encoder_layers, 1)
        self.assertEqual(student.config.decoder_layers, self.teacher_config.encoder_layers)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_small_enc_small_dec(self):
        student, *_ = create_student_by_copying_alternating_layers(TINY_BART, tempfile.mkdtemp(), e=1, d=1)
        self.assertEqual(student.config.encoder_layers, 1)
        self.assertEqual(student.config.decoder_layers, 1)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_raises_assert(self):
        with self.assertRaises(AssertionError):
            create_student_by_copying_alternating_layers(TINY_BART, tempfile.mkdtemp(), e=None, d=None)
@@ -19,7 +19,14 @@ from run_eval import generate_summaries_or_translations, run_generate
 from run_eval_search import run_search
 from transformers import AutoConfig, AutoModelForSeq2SeqLM
 from transformers.hf_api import HfApi
-from transformers.testing_utils import CaptureStderr, CaptureStdout, TestCasePlus, require_torch_gpu, slow
+from transformers.testing_utils import (
+    CaptureStderr,
+    CaptureStdout,
+    TestCasePlus,
+    require_torch_gpu,
+    require_torch_non_multigpu_but_fix_me,
+    slow,
+)
 from utils import ROUGE_KEYS, label_smoothed_nll_loss, lmap, load_json


@@ -89,7 +96,6 @@ CHEAP_ARGS = {
    "freeze_encoder": False,
    "auto_scale_batch_size": False,
    "overwrite_output_dir": False,
-    "student": None,
 }


@@ -101,7 +107,6 @@ def _dump_articles(path: Path, articles: list):
 ARTICLES = [" Sam ate lunch today.", "Sams lunch ingredients."]
 SUMMARIES = ["A very interesting story about what I ate for lunch.", "Avocado, celery, turkey, coffee"]
 T5_TINY = "patrickvonplaten/t5-tiny-random"
-T5_TINIER = "sshleifer/t5-tinier-random"
 BART_TINY = "sshleifer/bart-tiny-random"
 MBART_TINY = "sshleifer/tiny-mbart"
 MARIAN_TINY = "sshleifer/tiny-marian-en-de"
@@ -128,6 +133,7 @@ class TestSummarizationDistiller(TestCasePlus):

    @slow
    @require_torch_gpu
+    @require_torch_non_multigpu_but_fix_me
    def test_hub_configs(self):
        """I put require_torch_gpu cause I only want this to run with self-scheduled."""

@@ -145,10 +151,12 @@ class TestSummarizationDistiller(TestCasePlus):
                failures.append(m)
        assert not failures, f"The following models could not be loaded through AutoConfig: {failures}"

+    @require_torch_non_multigpu_but_fix_me
    def test_distill_no_teacher(self):
        updates = dict(student_encoder_layers=2, student_decoder_layers=1, no_teacher=True)
        self._test_distiller_cli(updates)

+    @require_torch_non_multigpu_but_fix_me
    def test_distill_checkpointing_with_teacher(self):
        updates = dict(
            student_encoder_layers=2,
@@ -173,8 +181,9 @@ class TestSummarizationDistiller(TestCasePlus):
        convert_pl_to_hf(ckpts[0], transformer_ckpts[0].parent, out_path_new)
        assert os.path.exists(os.path.join(out_path_new, "pytorch_model.bin"))

+    @require_torch_non_multigpu_but_fix_me
    def test_loss_fn(self):
-        model = AutoModelForSeq2SeqLM.from_pretrained(BART_TINY)
+        model = AutoModelForSeq2SeqLM.from_pretrained(BART_TINY, return_dict=True)
        input_ids, mask = model.dummy_inputs["input_ids"], model.dummy_inputs["attention_mask"]
        target_ids = torch.tensor([[0, 4, 8, 2], [0, 8, 2, 1]], dtype=torch.long, device=model.device)
        decoder_input_ids = target_ids[:, :-1].contiguous()  # Why this line?
@@ -193,6 +202,7 @@ class TestSummarizationDistiller(TestCasePlus):
            # TODO: understand why this breaks
            self.assertEqual(nll_loss, model_computed_loss)

+    @require_torch_non_multigpu_but_fix_me
    def test_distill_mbart(self):
        updates = dict(
            student_encoder_layers=2,
@@ -217,6 +227,7 @@ class TestSummarizationDistiller(TestCasePlus):
        assert len(all_files) > 2
        self.assertEqual(len(transformer_ckpts), 2)

+    @require_torch_non_multigpu_but_fix_me
    def test_distill_t5(self):
        updates = dict(
            student_encoder_layers=1,
@@ -228,15 +239,6 @@ class TestSummarizationDistiller(TestCasePlus):
        )
        self._test_distiller_cli(updates)

-    def test_distill_different_base_models(self):
-        updates = dict(
-            teacher=T5_TINY,
-            student=T5_TINIER,
-            model_name_or_path=T5_TINIER,
-            tokenizer_name=T5_TINIER,
-        )
-        self._test_distiller_cli(updates)
-
    def _test_distiller_cli(self, updates, check_contents=True):
        default_updates = dict(
            label_smoothing=0.0,
@@ -307,18 +309,21 @@ class TestTheRest(TestCasePlus):

    # test one model to quickly (no-@slow) catch simple problems and do an
    # extensive testing of functionality with multiple models as @slow separately
+    @require_torch_non_multigpu_but_fix_me
    def test_run_eval(self):
        self.run_eval_tester(T5_TINY)

    # any extra models should go into the list here - can be slow
    @parameterized.expand([BART_TINY, MBART_TINY])
    @slow
+    @require_torch_non_multigpu_but_fix_me
    def test_run_eval_slow(self, model):
        self.run_eval_tester(model)

    # testing with 2 models to validate: 1. translation (t5) 2. summarization (mbart)
    @parameterized.expand([T5_TINY, MBART_TINY])
    @slow
+    @require_torch_non_multigpu_but_fix_me
    def test_run_eval_search(self, model):
        input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
        output_file_name = input_file_name.parent / "utest_output.txt"
@@ -369,6 +374,7 @@ class TestTheRest(TestCasePlus):
    @parameterized.expand(
        [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY],
    )
+    @require_torch_non_multigpu_but_fix_me
    def test_finetune(self, model):
        args_d: dict = CHEAP_ARGS.copy()
        task = "translation" if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else "summarization"
@@ -420,6 +426,7 @@ class TestTheRest(TestCasePlus):
        assert isinstance(example_batch, dict)
        assert len(example_batch) >= 4

+    @require_torch_non_multigpu_but_fix_me
    def test_finetune_extra_model_args(self):
        args_d: dict = CHEAP_ARGS.copy()

@@ -470,6 +477,7 @@ class TestTheRest(TestCasePlus):
            model = main(args)
        assert str(excinfo.value) == f"model config doesn't have a `{unsupported_param}` attribute"

+    @require_torch_non_multigpu_but_fix_me
    def test_finetune_lr_schedulers(self):
        args_d: dict = CHEAP_ARGS.copy()

@@ -8,7 +8,7 @@ from transformers.testing_utils import (
    execute_subprocess_async,
    get_gpu_count,
    require_torch_gpu,
-    require_torch_multi_gpu,
+    require_torch_multigpu,
    slow,
 )

@@ -21,8 +21,8 @@ class TestSummarizationDistillerMultiGPU(TestCasePlus):
    def setUpClass(cls):
        return cls

-    @require_torch_multi_gpu
-    def test_multi_gpu(self):
+    @require_torch_multigpu
+    def test_multigpu(self):

        updates = dict(
            no_teacher=True,
@@ -2,9 +2,9 @@ import os
 import tempfile
 import unittest

+from transformers.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
 from transformers.file_utils import cached_property
-from transformers.models.marian.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow
+from transformers.testing_utils import require_torch_non_multigpu_but_fix_me, slow


@unittest.skipUnless(os.path.exists(DEFAULT_REPO), "Tatoeba directory does not exist.")
@@ -15,12 +15,12 @@ class TatoebaConversionTester(unittest.TestCase):
        return TatoebaConverter(save_dir=tmp_dir)

    @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_resolver(self):
        self.resolver.convert_models(["heb-eng"])

    @slow
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_model_card(self):
        content, mmeta = self.resolver.write_model_card("opus-mt-he-en", dry_run=True)
        assert mmeta["long_pair"] == "heb-eng"
@@ -21,7 +21,7 @@ from torch.utils.data import Dataset, Sampler
 from sentence_splitter import add_newline_to_end_of_each_sentence
 from transformers import BartTokenizer, EvalPrediction, PreTrainedTokenizer, T5Tokenizer
 from transformers.file_utils import cached_property
-from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.modeling_bart import shift_tokens_right


 try:
@@ -23,7 +23,7 @@ from unittest.mock import patch
 import torch

 from transformers.file_utils import is_apex_available
-from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me, torch_device
+from transformers.testing_utils import TestCasePlus, require_torch_non_multigpu_but_fix_me, torch_device


 SRC_DIRS = [
@@ -67,7 +67,7 @@ def is_cuda_and_apex_available():


 class ExamplesTests(TestCasePlus):
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_run_glue(self):
        stream_handler = logging.StreamHandler(sys.stdout)
        logger.addHandler(stream_handler)
@@ -100,7 +100,7 @@ class ExamplesTests(TestCasePlus):
            for value in result.values():
                self.assertGreaterEqual(value, 0.75)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_run_pl_glue(self):
        stream_handler = logging.StreamHandler(sys.stdout)
        logger.addHandler(stream_handler)
@@ -138,7 +138,7 @@ class ExamplesTests(TestCasePlus):
            #         self.assertGreaterEqual(v, 0.75, f"({k})")
            #

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_run_clm(self):
        stream_handler = logging.StreamHandler(sys.stdout)
        logger.addHandler(stream_handler)
@@ -170,7 +170,7 @@ class ExamplesTests(TestCasePlus):
            result = run_clm.main()
            self.assertLess(result["perplexity"], 100)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_run_mlm(self):
        stream_handler = logging.StreamHandler(sys.stdout)
        logger.addHandler(stream_handler)
@@ -196,7 +196,7 @@ class ExamplesTests(TestCasePlus):
            result = run_mlm.main()
            self.assertLess(result["perplexity"], 42)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_run_ner(self):
        stream_handler = logging.StreamHandler(sys.stdout)
        logger.addHandler(stream_handler)
@@ -227,7 +227,7 @@ class ExamplesTests(TestCasePlus):
            self.assertGreaterEqual(result["eval_precision"], 0.75)
            self.assertLess(result["eval_loss"], 0.5)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_run_squad(self):
        stream_handler = logging.StreamHandler(sys.stdout)
        logger.addHandler(stream_handler)
@@ -256,7 +256,7 @@ class ExamplesTests(TestCasePlus):
            self.assertGreaterEqual(result["f1"], 25)
            self.assertGreaterEqual(result["exact"], 21)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_generation(self):
        stream_handler = logging.StreamHandler(sys.stdout)
        logger.addHandler(stream_handler)
@@ -20,7 +20,7 @@ import unittest
 from time import time
 from unittest.mock import patch

-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, require_torch_tpu
+from transformers.testing_utils import require_torch_non_multigpu_but_fix_me, require_torch_tpu


 logging.basicConfig(level=logging.DEBUG)
@@ -30,7 +30,7 @@ logger = logging.getLogger()

@require_torch_tpu
 class TorchXLAExamplesTests(unittest.TestCase):
-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_run_glue(self):
        import xla_spawn

@@ -82,7 +82,7 @@ class TorchXLAExamplesTests(unittest.TestCase):
            # Assert that the script takes less than 300 seconds to make sure it doesn't hang.
            self.assertLess(end - start, 500)

-    @require_torch_non_multi_gpu_but_fix_me
+    @require_torch_non_multigpu_but_fix_me
    def test_trainer_tpu(self):
        import xla_spawn

@@ -124,8 +124,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -172,8 +171,6 @@ def main():
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
    logger.info(f"Training/evaluation parameters {training_args}")

    # Set seed before initializing model.
@@ -27,12 +27,6 @@ from transformers import (
    glue_processors,
    glue_tasks_num_labels,
 )
-from transformers.utils import logging as hf_logging
-
-
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()


 class Split(Enum):
@@ -117,8 +111,7 @@ class ModelArguments:
    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
    # or just modify its tokenizer_config.json.
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -21,12 +21,6 @@ from transformers import (
    TFTrainer,
    TFTrainingArguments,
 )
-from transformers.utils import logging as hf_logging
-
-
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()


 def get_tfds(
@@ -182,8 +176,7 @@ class ModelArguments:
    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
    # or just modify its tokenizer_config.json.
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -29,7 +29,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

-import transformers
 from transformers import (
    WEIGHTS_NAME,
    AdamW,
@@ -42,7 +41,6 @@ from transformers import glue_convert_examples_to_features as convert_examples_t
 from transformers import xnli_compute_metrics as compute_metrics
 from transformers import xnli_output_modes as output_modes
 from transformers import xnli_processors as processors
-from transformers.trainer_utils import is_main_process


 try:
@@ -406,7 +404,7 @@ def main():
        "--cache_dir",
        default=None,
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -528,11 +526,7 @@ def main():
        bool(args.local_rank != -1),
        args.fp16,
    )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
    # Set seed
    set_seed(args)

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Sylvain Gugger	d5b3e56de5	Release: v3.5.1	2020-11-13 10:03:08 -05:00
Sylvain Gugger	1fb361768d	Fix typo	2020-11-13 10:00:29 -05:00