Sacremoses in TransfoXL

Sacremoses in FSMT
Sacremoses
2020-11-16 16:34:58 -05:00 · 2020-11-16 16:33:31 -05:00 · 2020-11-16 16:30:51 -05:00
581 changed files with 4823 additions and 9644 deletions
@@ -77,7 +77,7 @@ jobs:
                      - v0.4-torch_and_tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece]
+            - run: pip install .[sklearn,tf-cpu,torch,testing]
            - save_cache:
                key: v0.4-{{ checksum "setup.py" }}
                paths:
@@ -103,7 +103,7 @@ jobs:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,torch,testing,sentencepiece]
+            - run: pip install .[sklearn,torch,testing]
            - save_cache:
                  key: v0.4-torch-{{ checksum "setup.py" }}
                  paths:
@@ -129,7 +129,7 @@ jobs:
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
+            - run: pip install .[sklearn,tf-cpu,testing]
            - save_cache:
                  key: v0.4-tf-{{ checksum "setup.py" }}
                  paths:
@@ -155,7 +155,7 @@ jobs:
                    - v0.4-flax-{{ checksum "setup.py" }}
                    - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
-            - run: sudo pip install .[flax,sklearn,torch,testing,sentencepiece]
+            - run: sudo pip install .[flax,sklearn,torch,testing]
            - save_cache:
                  key: v0.4-flax-{{ checksum "setup.py" }}
                  paths:
@@ -181,7 +181,7 @@ jobs:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,torch,testing,sentencepiece]
+            - run: pip install .[sklearn,torch,testing]
            - save_cache:
                  key: v0.4-torch-{{ checksum "setup.py" }}
                  paths:
@@ -207,7 +207,7 @@ jobs:
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
+            - run: pip install .[sklearn,tf-cpu,testing]
            - save_cache:
                  key: v0.4-tf-{{ checksum "setup.py" }}
                  paths:
@@ -221,7 +221,7 @@ jobs:
    run_tests_custom_tokenizers:
        working_directory: ~/transformers
        docker:
-            - image: circleci/python:3.7
+            - image: circleci/python:3.6
        environment:
            RUN_CUSTOM_TOKENIZERS: yes
        steps:
@@ -231,7 +231,7 @@ jobs:
                      - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
-            - run: pip install .[ja,testing,sentencepiece]
+            - run: pip install .[ja,testing]
            - run: python -m unidic download
            - save_cache:
                  key: v0.4-custom_tokenizers-{{ checksum "setup.py" }}
@@ -258,7 +258,7 @@ jobs:
                      - v0.4-torch_examples-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,torch,sentencepiece,testing]
+            - run: pip install .[sklearn,torch,testing]
            - run: pip install -r examples/requirements.txt
            - save_cache:
                  key: v0.4-torch_examples-{{ checksum "setup.py" }}
@@ -324,7 +324,7 @@ jobs:
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: pip install isort
-            - run: pip install .[all,quality]
+            - run: pip install .[tf,torch,flax,quality]
            - save_cache:
                  key: v0.4-code_quality-{{ checksum "setup.py" }}
                  paths:
@@ -20,7 +20,7 @@ Fixes # (issue)
 - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md#start-contributing-pull-requests),
      Pull Request section?
 - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
-      to it if that's the case.
+      to the it if that's the case.
 - [ ] Did you make sure to update the documentation with your changes? Here are the
      [documentation guidelines](https://github.com/huggingface/transformers/tree/master/docs), and
      [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/master/docs#writing-source-documentation).
@@ -8,9 +8,6 @@ on:
 jobs:
  torch_hub_integration:
    runs-on: ubuntu-latest
-    env:
-      # TODO quickfix but may need more investigation
-      ACTIONS_ALLOW_UNSECURE_COMMANDS: True
    steps:
    # no checkout necessary here.
    - name: Extract branch name
@@ -16,7 +16,7 @@ on:

 jobs:
  run_tests_torch_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2
      - name: Python version
@@ -48,7 +48,7 @@ jobs:
        run: |
          source .env/bin/activate
          pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
+          pip install .[torch,sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets

      - name: Are GPUs recognized by our DL frameworks
@@ -86,7 +86,7 @@ jobs:
                  

  run_tests_tf_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2
      - name: Python version
@@ -117,7 +117,7 @@ jobs:
        run: |
          source .env/bin/activate
          pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
+          pip install .[tf,sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets

      - name: Are GPUs recognized by our DL frameworks
@@ -154,7 +154,7 @@ jobs:
          path: reports

  run_tests_torch_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2
      - name: Python version
@@ -185,7 +185,7 @@ jobs:
        run: |
          source .env/bin/activate
          pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
+          pip install .[torch,sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets

      - name: Are GPUs recognized by our DL frameworks
@@ -213,7 +213,7 @@ jobs:
          path: reports

  run_tests_tf_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2
      - name: Python version
@@ -244,7 +244,7 @@ jobs:
        run: |
          source .env/bin/activate
          pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
+          pip install .[tf,sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets

      - name: Are GPUs recognized by our DL frameworks
@@ -9,14 +9,13 @@ on:
  push:
    branches:
      - ci_*
-      - framework-agnostic-tokenizers
  repository_dispatch:
  schedule:
    - cron: "0 0 * * *"

 jobs:
  run_all_tests_torch_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2

@@ -50,7 +49,7 @@ jobs:
        run: |
          source .env/bin/activate
          pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
+          pip install .[torch,sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets
          pip list

@@ -110,7 +109,7 @@ jobs:


  run_all_tests_tf_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
    steps:
      - uses: actions/checkout@v2

@@ -144,7 +143,7 @@ jobs:
        run: |
          source .env/bin/activate
          pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
+          pip install .[tf,sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets
          pip list

@@ -189,7 +188,7 @@ jobs:
          path: reports
          
  run_all_tests_torch_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2

@@ -223,7 +222,7 @@ jobs:
        run: |
          source .env/bin/activate
          pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
+          pip install .[torch,sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets
          pip list

@@ -251,11 +250,11 @@ jobs:
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_examples_multi_gpu examples
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_multi_gpu examples

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_torch_examples_multi_gpu_failures_short.txt
+        run: cat reports/examples_torch_multi_gpu_failures_short.txt

      - name: Run all pipeline tests on multi-GPU
        if: ${{ always() }}
@@ -280,7 +279,7 @@ jobs:
          path: reports

  run_all_tests_tf_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2

@@ -314,7 +313,7 @@ jobs:
        run: |
          source .env/bin/activate
          pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
+          pip install .[tf,sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets
          pip list

@@ -345,11 +344,11 @@ jobs:
          RUN_PIPELINE_TESTS: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipeline_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multi_gpu tests
          
      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_tf_pipeline_multi_gpu_failures_short.txt
+        run: cat reports/tests_tf_multi_gpu_pipelines_failures_short.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
@@ -133,6 +133,7 @@ dmypy.json
 tensorflow_code

 # Models
+models
 proc_data

 # examples
@@ -181,7 +181,6 @@ Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
 1. **[LXMERT](https://huggingface.co/transformers/model_doc/lxmert.html)** (from UNC Chapel Hill) released with the paper [LXMERT: Learning Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal.
 1. **[MarianMT](https://huggingface.co/transformers/model_doc/marian.html)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team.
 1. **[MBart](https://huggingface.co/transformers/model_doc/mbart.html)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-1. **[MT5](https://huggingface.co/transformers/model_doc/mt5.html)** (from Google AI) released with the paper [mT5: A massively multilingual pre-trained text-to-text transformer](https://arxiv.org/abs/2010.11934) by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
 1. **[Pegasus](https://huggingface.co/transformers/model_doc/pegasus.html)** (from Google) released with the paper [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization](https://arxiv.org/abs/1912.08777)> by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
 1. **[ProphetNet](https://huggingface.co/transformers/model_doc/prophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
 1. **[Reformer](https://huggingface.co/transformers/model_doc/reformer.html)** (from Google Research) released with the paper [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
@@ -214,17 +213,13 @@ These implementations have been tested on several datasets (see the example scri

 ## Citation

-We now have a [paper](https://www.aclweb.org/anthology/2020.emnlp-demos.6/) you can cite for the 🤗 Transformers library:
+We now have a [paper](https://arxiv.org/abs/1910.03771) you can cite for the 🤗 Transformers library:
 ```bibtex
-@inproceedings{wolf-etal-2020-transformers,
-    title = "Transformers: State-of-the-Art Natural Language Processing",
-    author = "Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Joe Davison and Sam Shleifer and Patrick von Platen and Clara Ma and Yacine Jernite and Julien Plu and Canwen Xu and Teven Le Scao and Sylvain Gugger and Mariama Drame and Quentin Lhoest and Alexander M. Rush",
-    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",
-    month = oct,
-    year = "2020",
-    address = "Online",
-    publisher = "Association for Computational Linguistics",
-    url = "https://www.aclweb.org/anthology/2020.emnlp-demos.6",
-    pages = "38--45"
+@article{Wolf2019HuggingFacesTS,
+  title={HuggingFace's Transformers: State-of-the-art Natural Language Processing},
+  author={Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Joe Davison and Sam Shleifer and Patrick von Platen and Clara Ma and Yacine Jernite and Julien Plu and Canwen Xu and Teven Le Scao and Sylvain Gugger and Mariama Drame and Quentin Lhoest and Alexander M. Rush},
+  journal={ArXiv},
+  year={2019},
+  volume={abs/1910.03771}
 }
 ```
@@ -44,7 +44,7 @@ The documentation is organized in five parts:
  and a glossary.
 - **USING 🤗 TRANSFORMERS** contains general tutorials on how to use the library.
 - **ADVANCED GUIDES** contains more advanced guides that are more specific to a given script or part of the library.
- **RESEARCH** focuses on tutorials that have less to do with how to use the library but more about general research in
+- **RESEARCH** focuses on tutorials that have less to do with how to use the library but more about general resarch in
  transformers model
 - The three last section contain the documentation of each public class and function, grouped in:

@@ -126,44 +126,41 @@ conversion utilities for the following models:
 21. :doc:`MBart <model_doc/mbart>` (from Facebook) released with the paper `Multilingual Denoising Pre-training for
    Neural Machine Translation <https://arxiv.org/abs/2001.08210>`__ by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li,
    Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-22. :doc:`MT5 <model_doc/mt5>` (from Google AI) released with the paper `mT5: A massively multilingual pre-trained
-    text-to-text transformer <https://arxiv.org/abs/2010.11934>`__ by Linting Xue, Noah Constant, Adam Roberts, Mihir
-    Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
-23. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
+22. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
    Gap-sentences for Abstractive Summarization <https://arxiv.org/abs/1912.08777>`__> by Jingqing Zhang, Yao Zhao,
    Mohammad Saleh and Peter J. Liu.
-24. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
+23. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
    Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi,
    Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-25. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
+24. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
    Transformer <https://arxiv.org/abs/2001.04451>`__ by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
-26. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
+25. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
    Pretraining Approach <https://arxiv.org/abs/1907.11692>`__ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar
    Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. ultilingual BERT into `DistilmBERT
    <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German version of
    DistilBERT.
-27. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
+26. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
    about efficient neural networks? <https://arxiv.org/abs/2006.11316>`__ by Forrest N. Iandola, Albert E. Shaw, Ravi
    Krishna, and Kurt W. Keutzer.
-28. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
+27. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
    Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by Colin Raffel and Noam Shazeer and Adam
    Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
-29. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
+28. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
    Attentive Language Models Beyond a Fixed-Length Context <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*,
    Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
-30. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
+29. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
    Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis Conneau.
-31. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
+30. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
    Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan,
    Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-32. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
+31. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
    Cross-lingual Representation Learning at Scale <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay
    Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke
    Zettlemoyer and Veselin Stoyanov.
-33. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `XLNet: Generalized Autoregressive
+32. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `XLNet: Generalized Autoregressive
    Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by Zhilin Yang*, Zihang Dai*, Yiming
    Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
-34. `Other community models <https://huggingface.co/models>`__, contributed by the `community
+33. `Other community models <https://huggingface.co/models>`__, contributed by the `community
    <https://huggingface.co/users>`__.

 .. toctree::
@@ -251,7 +248,6 @@ conversion utilities for the following models:
    model_doc/marian
    model_doc/mbart
    model_doc/mobilebert
-    model_doc/mt5
    model_doc/gpt
    model_doc/gpt2
    model_doc/pegasus
@@ -70,15 +70,15 @@ to check 🤗 Transformers is properly installed.

 This library provides pretrained models that will be downloaded and cached locally. Unless you specify a location with
 `cache_dir=...` when you use methods like `from_pretrained`, these models will automatically be downloaded in the
-folder given by the shell environment variable ``TRANSFORMERS_CACHE``. The default value for it will be the Hugging
-Face cache home followed by ``/transformers/``. This is (by order of priority):
+folder given by the shell environment variable ``TRANSFORMERS_CACHE``. The default value for it will be the PyTorch
+cache home followed by ``/transformers/`` (even if you don't have PyTorch installed). This is (by order of priority):

-  * shell environment variable ``HF_HOME`` 
-  * shell environment variable ``XDG_CACHE_HOME`` + ``/huggingface/``
-  * default: ``~/.cache/huggingface/``
+  * shell environment variable ``TORCH_HOME``
+  * shell environment variable ``XDG_CACHE_HOME`` + ``/torch/``
+  * default: ``~/.cache/torch/``

 So if you don't have any specific environment variable set, the cache directory will be at
-``~/.cache/huggingface/transformers/``.
+``~/.cache/torch/transformers/``.

 **Note:** If you have set a shell environment variable for one of the predecessors of this library
 (``PYTORCH_TRANSFORMERS_CACHE`` or ``PYTORCH_PRETRAINED_BERT_CACHE``), those will be used if there is no shell
@@ -51,10 +51,10 @@ AlbertTokenizer
 Albert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.albert.modeling_albert.AlbertForPreTrainingOutput
+.. autoclass:: transformers.modeling_albert.AlbertForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.albert.modeling_tf_albert.TFAlbertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_albert.TFAlbertForPreTrainingOutput
    :members:


@@ -44,8 +44,8 @@ Implementation Notes
 - Bart doesn't use :obj:`token_type_ids` for sequence classification. Use :class:`~transformers.BartTokenizer` or
  :meth:`~transformers.BartTokenizer.encode` to get the proper splitting.
 - The forward pass of :class:`~transformers.BartModel` will create decoder inputs (using the helper function
-  :func:`transformers.models.bart.modeling_bart._prepare_bart_decoder_inputs`) if they are not passed. This is
-  different than some other modeling APIs.
+  :func:`transformers.modeling_bart._prepare_bart_decoder_inputs`) if they are not passed. This is different than some
+  other modeling APIs.
 - Model predictions are intended to be identical to the original implementation when
  :obj:`force_bos_token_to_be_generated=True`. This only works, however, if the string you pass to
  :func:`fairseq.encode` starts with a space.
@@ -93,7 +93,7 @@ BartModel
 .. autoclass:: transformers.BartModel
    :members: forward

-.. autofunction:: transformers.models.bart.modeling_bart._prepare_bart_decoder_inputs
+.. autofunction:: transformers.modeling_bart._prepare_bart_decoder_inputs


 BartForConditionalGeneration
@@ -57,10 +57,10 @@ BertTokenizerFast
 Bert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.bert.modeling_bert.BertForPreTrainingOutput
+.. autoclass:: transformers.modeling_bert.BertForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.bert.modeling_tf_bert.TFBertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_bert.TFBertForPreTrainingOutput
    :members:


@@ -5,7 +5,7 @@ Overview
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 Dense Passage Retrieval (DPR) is a set of tools and models for state-of-the-art open-domain Q&A research. It was
-introduced in `Dense Passage Retrieval for Open-Domain Question Answering <https://arxiv.org/abs/2004.04906>`__ by
+intorduced in `Dense Passage Retrieval for Open-Domain Question Answering <https://arxiv.org/abs/2004.04906>`__ by
 Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, Wen-tau Yih.

 The abstract from the paper is the following:
@@ -71,13 +71,13 @@ DPRReaderTokenizerFast
 DPR specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.dpr.modeling_dpr.DPRContextEncoderOutput
+.. autoclass:: transformers.modeling_dpr.DPRContextEncoderOutput
    :members:

-.. autoclass:: transformers.models.dpr.modeling_dpr.DPRQuestionEncoderOutput
+.. autoclass:: transformers.modeling_dpr.DPRQuestionEncoderOutput
    :members:

-.. autoclass:: transformers.models.dpr.modeling_dpr.DPRReaderOutput
+.. autoclass:: transformers.modeling_dpr.DPRReaderOutput
    :members:


@@ -69,10 +69,10 @@ ElectraTokenizerFast
 Electra specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput
+.. autoclass:: transformers.modeling_electra.ElectraForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.electra.modeling_tf_electra.TFElectraForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_electra.TFElectraForPreTrainingOutput
    :members:


@@ -65,10 +65,10 @@ FunnelTokenizerFast
 Funnel specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.funnel.modeling_funnel.FunnelForPreTrainingOutput
+.. autoclass:: transformers.modeling_funnel.FunnelForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.funnel.modeling_tf_funnel.TFFunnelForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_funnel.TFFunnelForPreTrainingOutput
    :members:


@@ -72,10 +72,10 @@ OpenAIGPTTokenizerFast
 OpenAI specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.openai.modeling_openai.OpenAIGPTDoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_openai.OpenAIGPTDoubleHeadsModelOutput
    :members:

-.. autoclass:: transformers.models.openai.modeling_tf_openai.TFOpenAIGPTDoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_tf_openai.TFOpenAIGPTDoubleHeadsModelOutput
    :members:


@@ -60,10 +60,10 @@ GPT2TokenizerFast
 GPT2 specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.gpt2.modeling_gpt2.GPT2DoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_gpt2.GPT2DoubleHeadsModelOutput
    :members:

-.. autoclass:: transformers.models.gpt2.modeling_tf_gpt2.TFGPT2DoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_tf_gpt2.TFGPT2DoubleHeadsModelOutput
    :members:


@@ -93,47 +93,29 @@ LongformerTokenizerFast
 Longformer specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerBaseModelOutput
+.. autoclass:: transformers.modeling_longformer.LongformerBaseModelOutput
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerBaseModelOutputWithPooling
+.. autoclass:: transformers.modeling_longformer.LongformerBaseModelOutputWithPooling
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMaskedLMOutput
+.. autoclass:: transformers.modeling_longformer.LongformerMultipleChoiceModelOutput
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerQuestionAnsweringModelOutput
+.. autoclass:: transformers.modeling_longformer.LongformerQuestionAnsweringModelOutput
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerSequenceClassifierOutput
+.. autoclass:: transformers.modeling_tf_longformer.TFLongformerBaseModelOutput
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMultipleChoiceModelOutput
+.. autoclass:: transformers.modeling_tf_longformer.TFLongformerBaseModelOutputWithPooling
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerTokenClassifierOutput
+.. autoclass:: transformers.modeling_tf_longformer.TFLongformerQuestionAnsweringModelOutput
    :members: 

-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutput
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutputWithPooling
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerMaskedLMOutput
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerQuestionAnsweringModelOutput
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerSequenceClassifierOutput
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerMultipleChoiceModelOutput
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerTokenClassifierOutput
-    :members: 
+LongformerModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 LongformerModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -197,24 +179,3 @@ TFLongformerForQuestionAnswering
 .. autoclass:: transformers.TFLongformerForQuestionAnswering
    :members: call

-
-TFLongformerForSequenceClassification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFLongformerForSequenceClassification
-    :members: call
-
-
-TFLongformerForTokenClassification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFLongformerForTokenClassification
-    :members: call
-
-
-TFLongformerForMultipleChoice
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFLongformerForMultipleChoice
-    :members: call
-
@@ -67,19 +67,19 @@ LxmertTokenizerFast
 Lxmert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.lxmert.modeling_lxmert.LxmertModelOutput
+.. autoclass:: transformers.modeling_lxmert.LxmertModelOutput
    :members:

-.. autoclass:: transformers.models.lxmert.modeling_lxmert.LxmertForPreTrainingOutput
+.. autoclass:: transformers.modeling_lxmert.LxmertForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.lxmert.modeling_lxmert.LxmertForQuestionAnsweringOutput
+.. autoclass:: transformers.modeling_lxmert.LxmertForQuestionAnsweringOutput
    :members:

-.. autoclass:: transformers.models.lxmert.modeling_tf_lxmert.TFLxmertModelOutput
+.. autoclass:: transformers.modeling_tf_lxmert.TFLxmertModelOutput
    :members:

-.. autoclass:: transformers.models.lxmert.modeling_tf_lxmert.TFLxmertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_lxmert.TFLxmertForPreTrainingOutput
    :members:


@@ -78,7 +78,7 @@ require 3 character language codes:
    tokenizer = MarianTokenizer.from_pretrained(model_name)
    print(tokenizer.supported_language_codes)
    model = MarianMTModel.from_pretrained(model_name)
-    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text, return_tensors="pt"))
+    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text))
    tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
    # ["c'est une phrase en anglais que nous voulons traduire en français",
    # 'Isto deve ir para o português.',
@@ -150,7 +150,7 @@ Example of translating english to many romance languages, using old-style 2 char
    print(tokenizer.supported_language_codes)

    model = MarianMTModel.from_pretrained(model_name)
-    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text, return_tensors="pt"))
+    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text))
    tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
    # ["c'est une phrase en anglais que nous voulons traduire en français", 'Isto deve ir para o português.',  'Y esto al español']

@@ -44,7 +44,7 @@ the sequences for sequence-to-sequence fine-tuning.

    example_english_phrase = "UN Chief Says There Is No Military Solution in Syria"
    expected_translation_romanian = "Şeful ONU declară că nu există o soluţie militară în Siria"
-    batch = tokenizer.prepare_seq2seq_batch(example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian, return_tensors="pt")
+    batch = tokenizer.prepare_seq2seq_batch(example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian)
    model(input_ids=batch['input_ids'], labels=batch['labels']) # forward pass

 - Generation
@@ -58,7 +58,7 @@ the sequences for sequence-to-sequence fine-tuning.
    model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-en-ro")
    tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-en-ro")
    article = "UN Chief Says There Is No Military Solution in Syria"
-    batch = tokenizer.prepare_seq2seq_batch(src_texts=[article], src_lang="en_XX", return_tensors="pt")
+    batch = tokenizer.prepare_seq2seq_batch(src_texts=[article], src_lang="en_XX")
    translated_tokens = model.generate(**batch, decoder_start_token_id=tokenizer.lang_code_to_id["ro_RO"])
    translation = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0]
    assert translation == "Şeful ONU declară că nu există o soluţie militară în Siria"
@@ -58,10 +58,10 @@ MobileBertTokenizerFast
 MobileBert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.mobilebert.modeling_mobilebert.MobileBertForPreTrainingOutput
+.. autoclass:: transformers.modeling_mobilebert.MobileBertForPreTrainingOutput
    :members:

-.. autoclass:: transformers.models.mobilebert.modeling_tf_mobilebert.TFMobileBertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_mobilebert.TFMobileBertForPreTrainingOutput
    :members:


@@ -1,53 +0,0 @@
-MT5
-----------------------------------------------------------------------------------------------------------------------
-
-Overview
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The mT5 model was presented in `mT5: A massively multilingual pre-trained text-to-text transformer
-<https://arxiv.org/abs/2010.11934>`_ by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya
-Siddhant, Aditya Barua, Colin Raffel.
-
-The abstract from the paper is the following:
-
-*The recent "Text-to-Text Transfer Transformer" (T5) leveraged a unified text-to-text format and scale to attain
-state-of-the-art results on a wide variety of English-language NLP tasks. In this paper, we introduce mT5, a
-multilingual variant of T5 that was pre-trained on a new Common Crawl-based dataset covering 101 languages. We describe
-the design and modified training of mT5 and demonstrate its state-of-the-art performance on many multilingual
-benchmarks. All of the code and model checkpoints*
-
-The original code can be found `here <https://github.com/google-research/multilingual-t5>`__.
-
-MT5Config
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5Config
-    :members:
-
-
-MT5Model
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5Model
-    :members:
-
-
-MT5ForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5ForConditionalGeneration
-    :members:
-
-
-TFMT5Model
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFMT5Model
-    :members:
-
-
-TFMT5ForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFMT5ForConditionalGeneration
-    :members:
@@ -78,7 +78,7 @@ Usage Example
    torch_device = 'cuda' if torch.cuda.is_available() else 'cpu'
    tokenizer = PegasusTokenizer.from_pretrained(model_name)
    model = PegasusForConditionalGeneration.from_pretrained(model_name).to(torch_device)
-    batch = tokenizer.prepare_seq2seq_batch(src_text, truncation=True, padding='longest', return_tensors="pt").to(torch_device)
+    batch = tokenizer.prepare_seq2seq_batch(src_text, truncation=True, padding='longest').to(torch_device)
    translated = model.generate(**batch)
    tgt_text = tokenizer.batch_decode(translated, skip_special_tokens=True)
    assert tgt_text[0] == "California's largest electricity provider has turned off power to hundreds of thousands of customers."
@@ -47,16 +47,16 @@ ProphetNetTokenizer
 ProphetNet specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetSeq2SeqLMOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqLMOutput
    :members:

-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetSeq2SeqModelOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqModelOutput
    :members:

-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetDecoderModelOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderModelOutput
    :members:

-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetDecoderLMOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderLMOutput
    :members:

 ProphetNetModel
@@ -50,10 +50,10 @@ RagTokenizer
 Rag specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.rag.modeling_rag.RetrievAugLMMarginOutput
+.. autoclass:: transformers.modeling_rag.RetrievAugLMMarginOutput
    :members:

-.. autoclass:: transformers.models.rag.modeling_rag.RetrievAugLMOutput
+.. autoclass:: transformers.modeling_rag.RetrievAugLMOutput
    :members:

 RagRetriever
@@ -49,16 +49,16 @@ TransfoXLTokenizer
 TransfoXL specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.transfo_xl.modeling_transfo_xl.TransfoXLModelOutput
+.. autoclass:: transformers.modeling_transfo_xl.TransfoXLModelOutput
    :members:

-.. autoclass:: transformers.models.transfo_xl.modeling_transfo_xl.TransfoXLLMHeadModelOutput
+.. autoclass:: transformers.modeling_transfo_xl.TransfoXLLMHeadModelOutput
    :members:

-.. autoclass:: transformers.models.transfo_xl.modeling_tf_transfo_xl.TFTransfoXLModelOutput
+.. autoclass:: transformers.modeling_tf_transfo_xl.TFTransfoXLModelOutput
    :members:

-.. autoclass:: transformers.models.transfo_xl.modeling_tf_transfo_xl.TFTransfoXLLMHeadModelOutput
+.. autoclass:: transformers.modeling_tf_transfo_xl.TFTransfoXLLMHeadModelOutput
    :members:


@@ -50,7 +50,7 @@ XLMTokenizer
 XLM specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.xlm.modeling_xlm.XLMForQuestionAnsweringOutput
+.. autoclass:: transformers.modeling_xlm.XLMForQuestionAnsweringOutput
    :members:


@@ -53,43 +53,43 @@ XLNetTokenizer
 XLNet specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetModelOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetModelOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetLMHeadModelOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetLMHeadModelOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForSequenceClassificationOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForSequenceClassificationOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForMultipleChoiceOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForMultipleChoiceOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForTokenClassificationOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForTokenClassificationOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForQuestionAnsweringSimpleOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForQuestionAnsweringSimpleOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForQuestionAnsweringOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForQuestionAnsweringOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetModelOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetModelOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetLMHeadModelOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetLMHeadModelOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForSequenceClassificationOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForSequenceClassificationOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForMultipleChoiceOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForMultipleChoiceOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForTokenClassificationOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForTokenClassificationOutput
    :members:

-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForQuestionAnsweringSimpleOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForQuestionAnsweringSimpleOutput
    :members:


@@ -530,7 +530,7 @@ Sequence-to-sequence model with the same encoder-decoder model architecture as B
 two self-supervised objective functions: Masked Language Modeling (MLM) and a novel summarization specific pre-training
 objective, called Gap Sentence Generation (GSG).

-  * MLM: encoder input tokens are randomly replaced by a mask tokens and have to be predicted by the encoder (like in
+  * MLM: encoder input tokens are randomely replaced by a mask tokens and have to be predicted by the encoder (like in
    BERT)
  * GSG: whole encoder input sentences are replaced by a second mask token and fed to the decoder, but which has a
    causal mask to hide the future words like a regular auto-regressive transformer decoder.
@@ -560,7 +560,6 @@ A framework for translation models, using the same models as BART

 The library provides a version of this model for conditional generation.

-
 T5
 -----------------------------------------------------------------------------------------------------------------------

@@ -593,28 +592,6 @@ For instance, if we have the sentence “My dog is very cute .”, and we decide

 The library provides a version of this model for conditional generation.

-
-MT5
-----------------------------------------------------------------------------------------------------------------------
-
-.. raw:: html
-
-   <a href="https://huggingface.co/models?filter=mt5">
-       <img alt="Models" src="https://img.shields.io/badge/All_model_pages-mt5-blueviolet">
-   </a>
-   <a href="model_doc/mt5.html">
-       <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-mt5-blueviolet">
-   </a>
-
-`mT5: A massively multilingual pre-trained text-to-text transformer <https://arxiv.org/abs/2010.11934>`_, Linting Xue
-et al.
-
-The model architecture is same as T5. mT5's pre-training objective includes T5's self-supervised training, but not T5's
-supervised training. mT5 is trained on 101 languages.
-
-The library provides a version of this model for conditional generation.
-
-
 MBart
 -----------------------------------------------------------------------------------------------------------------------

@@ -109,7 +109,7 @@ XLM-RoBERTa
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 XLM-RoBERTa was trained on 2.5TB of newly created clean CommonCrawl data in 100 languages. It provides strong gains
-over previously released multi-lingual models like mBERT or XLM on downstream tasks like classification, sequence
+over previously released multi-lingual models like mBERT or XLM on downstream taks like classification, sequence
 labeling and question answering.

 Two XLM-RoBERTa checkpoints can be used for multi-lingual tasks:
@@ -62,7 +62,7 @@ sliding the context window so that the model has more context when making each p
 This is a closer approximation to the true decomposition of the sequence probability and will typically yield a more
 favorable score. The downside is that it requires a separate forward pass for each token in the corpus. A good
 practical compromise is to employ a strided sliding window, moving the context by larger strides rather than sliding by
-1 token a time. This allows computation to proceed much faster while still giving the model a large context to make
+1 token a time. This allows computation to procede much faster while still giving the model a large context to make
 predictions at each step.

 Example: Calculating perplexity with GPT-2 in 🤗 Transformers
@@ -3,11 +3,11 @@ Pretrained models

 Here is the full list of the currently provided pretrained models together with a short presentation of each model.

-For a list that includes all community-uploaded models, refer to `https://huggingface.co/models
+For a list that includes community-uploaded models, refer to `https://huggingface.co/models
 <https://huggingface.co/models>`__.

 +--------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
-| Architecture       | Model id                                                   | Details of the model                                                                                                                  |
+| Architecture       | Shortcut name                                              | Details of the model                                                                                                                  |
 +====================+============================================================+=======================================================================================================================================+
 | BERT               | ``bert-base-uncased``                                      | | 12-layer, 768-hidden, 12-heads, 110M parameters.                                                                                    |
 |                    |                                                            | | Trained on lower-cased English text.                                                                                                |
@@ -231,9 +231,7 @@ Here is an example of question answering using a model and a tokenizer. The proc
    ...     input_ids = inputs["input_ids"].tolist()[0]
    ...
    ...     text_tokens = tokenizer.convert_ids_to_tokens(input_ids)
-    ...     outputs = model(**inputs)
-    ...     answer_start_scores = outputs.start_logits
-    ...     answer_end_scores = outputs.end_logits
+    ...     answer_start_scores, answer_end_scores = model(**inputs)
    ...
    ...     answer_start = torch.argmax(
    ...         answer_start_scores
@@ -275,9 +273,7 @@ Here is an example of question answering using a model and a tokenizer. The proc
    ...     input_ids = inputs["input_ids"].numpy()[0]
    ...
    ...     text_tokens = tokenizer.convert_ids_to_tokens(input_ids)
-    ...     outputs = model(inputs)
-    ...     answer_start_scores = outputs.start_logits
-    ...     answer_end_scores = outputs.end_logits
+    ...     answer_start_scores, answer_end_scores = model(inputs)
    ...
    ...     answer_start = tf.argmax(
    ...         answer_start_scores, axis=1
@@ -1055,7 +1055,7 @@ If you need to validate the output of a logger, you can use :obj:`CaptureLogger`

    msg = "Testing 1, 2, 3"
    logging.set_verbosity_info()
-    logger = logging.get_logger("transformers.models.bart.tokenization_bart")
+    logger = logging.get_logger("transformers.tokenization_bart")
    with CaptureLogger(logger) as cl:
        logger.info(msg)
    assert cl.out, msg+"\n"
@@ -1,243 +1,223 @@
-Summary of the tokenizers
+Tokenizer summary
 -----------------------------------------------------------------------------------------------------------------------

-On this page, we will have a closer look at tokenization. As we saw in :doc:`the preprocessing tutorial
-<preprocessing>`, tokenizing a text is splitting it into words or subwords, which then are converted to ids through a
-look-up table. Converting words or subwords to ids is straightforward, so in this summary, we will focus on splitting a
-text into words or subwords (i.e. tokenizing a text). More specifically, we will look at the three main types of
-tokenizers used in 🤗 Transformers: :ref:`Byte-Pair Encoding (BPE) <byte-pair-encoding>`, :ref:`WordPiece <wordpiece>`,
-and :ref:`SentencePiece <sentencepiece>`, and show exemplary which tokenizer type is used by which model.
+In this page, we will have a closer look at tokenization. As we saw in :doc:`the preprocessing tutorial
+<preprocessing>`, tokenizing a text is splitting it into words or subwords, which then are converted to ids. The second
+part is pretty straightforward, here we will focus on the first part. More specifically, we will look at the three main
+different kinds of tokenizers used in 🤗 Transformers: :ref:`Byte-Pair Encoding (BPE) <byte-pair-encoding>`,
+:ref:`WordPiece <wordpiece>` and :ref:`SentencePiece <sentencepiece>`, and provide examples of models using each of
+those.

-Note that on each model page, you can look at the documentation of the associated tokenizer to know which tokenizer
-type was used by the pretrained model. For instance, if we look at :class:`~transformers.BertTokenizer`, we can see
-that the model uses :ref:`WordPiece <wordpiece>`.
+Note that on each model page, you can look at the documentation of the associated tokenizer to know which of those
+algorithms the pretrained model used. For instance, if we look at :class:`~transformers.BertTokenizer`, we can see it's
+using :ref:`WordPiece <wordpiece>`.

-Introduction
+Introduction to tokenization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-Splitting a text into smaller chunks is a task that is harder than it looks, and there are multiple ways of doing so.
-For instance, let's look at the sentence ``"Don't you love 🤗 Transformers? We sure do."`` A simple way of tokenizing
-this text is to split it by spaces, which would give:
+Splitting a text in smaller chunks is a task that's harder than it looks, and there are multiple ways of doing it. For
+instance, let's look at the sentence "Don't you love 🤗 Transformers? We sure do." A first simple way of tokenizing this
+text is just to split it by spaces, which would give:

 .. code-block::

    ["Don't", "you", "love", "🤗", "Transformers?", "We", "sure", "do."]

-This is a sensible first step, but if we look at the tokens ``"Transformers?"`` and ``"do."``, we notice that the
-punctuation is attached to the words ``"Transformer"`` and ``"do"``, which is suboptimal. We should take the
-punctuation into account so that a model does not have to learn a different representation of a word and every possible
-punctuation symbol that could follow it, which would explode the number of representations the model has to learn.
-Taking punctuation into account, tokenizing our exemplary text would give:
+This is a nice first step, but if we look at the tokens "Transformers?" or "do.", we can see we can do better. Those
+will be different than the tokens "Transformers" and "do" for our model, so we should probably take the punctuation
+into account. This would give:

 .. code-block::

    ["Don", "'", "t", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]

-Better. However, it is disadvantageous, how the tokenization dealt with the word ``"Don't"``. ``"Don't"`` stands for
-``"do not"``, so it would be better tokenized as ``["Do", "n't"]``. This is where things start getting complicated, and
-part of the reason each model has its own tokenizer type. Depending on the rules we apply for tokenizing a text, a
-different tokenized output is generated for the same text. A pretrained model only performs properly if you feed it an
-input that was tokenized with the same rules that were used to tokenize its training data.
+which is better already. One thing that is annoying though is how it dealt with "Don't". "Don't" stands for do not, so
+it should probably be better tokenized as ``["Do", "n't"]``. This is where things start getting more complicated, and
+part of the reason each kind of model has its own tokenizer class. Depending on the rules we apply to split our texts
+into tokens, we'll get different tokenized versions of the same text. And of course, a given pretrained model won't
+perform properly if you don't use the exact same rules as the persons who pretrained it.

 `spaCy <https://spacy.io/>`__ and `Moses <http://www.statmt.org/moses/?n=Development.GetStarted>`__ are two popular
-rule-based tokenizers. Applying them on our example, *spaCy* and *Moses* would output something like:
+rule-based tokenizers. On the text above, they'd output something like:

 .. code-block::

    ["Do", "n't", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]

-As can be seen space and punctuation tokenization, as well as rule-based tokenization, is used here. Space and
-punctuation tokenization and rule-based tokenization are both examples of word tokenization, which is loosely defined
-as splitting sentences into words. While it's the most intuitive way to split texts into smaller chunks, this
-tokenization method can lead to problems for massive text corpora. In this case, space and punctuation tokenization
-usually generates a very big vocabulary (the set of all unique words and tokens used). *E.g.*, :doc:`Transformer XL
-<model_doc/transformerxl>` uses space and punctuation tokenization, resulting in a vocabulary size of 267,735!
+Space/punctuation-tokenization and rule-based tokenization are both examples of word tokenization, which is splitting a
+sentence into words. While it's the most intuitive way to separate texts in smaller chunks, it can have a problem when
+you have a huge corpus: it usually yields a very big vocabulary (the set of all unique tokens used). :doc:`Transformer
+XL <model_doc/transformerxl>` for instance uses space/punctuation-tokenization, and has a vocabulary size of 267,735!

-Such a big vocabulary size forces the model to have an enormous embedding matrix as the input and output layer, which
-causes both an increased memory and time complexity. In general, transformers models rarely have a vocabulary size
-greater than 50,000, especially if they are pretrained only on a single language.
+A huge vocabulary size means a huge embedding matrix at the start of the model, which will cause memory problems.
+TransformerXL deals with it by using a special kind of embeddings called adaptive embeddings, but in general,
+transformers models rarely have a vocabulary size greater than 50,000, especially if they are trained on a single
+language.

-So if simple space and punctuation tokenization is unsatisfactory, why not simply tokenize on characters? While
-character tokenization is very simple and would greatly reduce memory and time complexity it makes it much harder for
-the model to learn meaningful input representations. *E.g.* learning a meaningful context-independent representation
-for the letter ``"t"`` is much harder as learning a context-independent representation for the word ``"today"``.
-Therefore, character tokenization is often accompanied by a loss of performance. So to get the best of both worlds,
-transformers models use a hybrid between word-level and character-level tokenization called **subword** tokenization.
+So if tokenizing on words is unsatisfactory, we could go on the opposite direction and simply tokenize on characters.
+While it's very simple and would save a lot of memory, this doesn't allow the model to learn representations of texts
+as meaningful as when using a word tokenization, leading to a loss of performance. So to get the best of both worlds,
+all transformers models use a hybrid between word-level and character-level tokenization called subword tokenization.

 Subword tokenization
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-Subword tokenization algorithms rely on the principle that frequently used words should not be split into smaller
-subwords, but rare words should be decomposed into meaningful subwords. For instance ``"annoyingly"`` might be
-considered a rare word and could be decomposed into ``"annoying"`` and ``"ly"``. Both ``"annoying"`` and ``"ly"`` as
-stand-alone subwords would appear more frequently while at the same time the meaning of ``"annoyingly"`` is kept by the
-composite meaning of ``"annoying"`` and ``"ly"``. This is especially useful in agglutinative languages such as Turkish,
-where you can form (almost) arbitrarily long complex words by stringing together subwords.
+Subword tokenization algorithms rely on the principle that most common words should be left as is, but rare words
+should be decomposed in meaningful subword units. For instance "annoyingly" might be considered a rare word and
+decomposed as "annoying" and "ly". This is especially useful in agglutinative languages such as Turkish, where you can
+form (almost) arbitrarily long complex words by stringing together some subwords.

-Subword tokenization allows the model to have a reasonable vocabulary size while being able to learn meaningful
-context-independent representations. In addition, subword tokenization enables the model to process words it has never
-seen before, by decomposing them into known subwords. For instance, the :class:`~transformers.BertTokenizer` tokenizes
-``"I have a new GPU!"`` as follows:
+This allows the model to keep a reasonable vocabulary while still learning useful representations for common words or
+subwords. This also enables the model to process words it has never seen before, by decomposing them into subwords it
+knows. For instance, the base :class:`~transformers.BertTokenizer` will tokenize "I have a new GPU!" like this:

 .. code-block::

    >>> from transformers import BertTokenizer
-    >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+    >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    >>> tokenizer.tokenize("I have a new GPU!")
-    ["i", "have", "a", "new", "gp", "##u", "!"]
+    ['i', 'have', 'a', 'new', 'gp', '##u', '!']

-Because we are considering the uncased model, the sentence was lowercased first. We can see that the words ``["i",
-"have", "a", "new"]`` are present in the tokenizer's vocabulary, but the word ``"gpu"`` is not. Consequently, the
-tokenizer splits ``"gpu"`` into known subwords: ``["gp" and "##u"]``. ``"##"`` means that the rest of the token should
-be attached to the previous one, without space (for decoding or reversal of the tokenization).
+Since we are considering the uncased model, the sentence was lowercased first. Then all the words were present in the
+vocabulary of the tokenizer, except for "gpu", so the tokenizer splits it in subwords it knows: "gp" and "##u". The
+"##" means that the rest of the token should be attached to the previous one, without space (for when we need to decode
+predictions and reverse the tokenization).

-As another example, :class:`~transformers.XLNetTokenizer` tokenizes our previously exemplary text as follows:
+Another example is when we use the base :class:`~transformers.XLNetTokenizer` to tokenize our previous text:

 .. code-block::

    >>> from transformers import XLNetTokenizer
-    >>> tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased")
+    >>> tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
    >>> tokenizer.tokenize("Don't you love 🤗 Transformers? We sure do.")
-    ["▁Don", "'", "t", "▁you", "▁love", "▁", "🤗", "▁", "Transform", "ers", "?", "▁We", "▁sure", "▁do", "."]
+    ['▁Don', "'", 't', '▁you', '▁love', '▁', '🤗', '▁', 'Transform', 'ers', '?', '▁We', '▁sure', '▁do', '.']

-We'll get back to the meaning of those ``"▁"`` when we look at :ref:`SentencePiece <sentencepiece>`. As one can see,
-the rare word ``"Transformers"`` has been split into the more frequent subwords ``"Transform"`` and ``"ers"``.
+We'll get back to the meaning of those '▁' when we look at :ref:`SentencePiece <sentencepiece>` but you can see
+Transformers has been split into "Transform" and "ers".

-Let's now look at how the different subword tokenization algorithms work. Note that all of those tokenization
-algorithms rely on some form of training which is usually done on the corpus the corresponding model will be trained
-on.
+Let's now look at how the different subword tokenization algorithms work. Note that they all rely on some form of
+training which is usually done on the corpus the corresponding model will be trained on.

 .. _byte-pair-encoding:

-Byte-Pair Encoding (BPE)
+Byte-Pair Encoding
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-Byte-Pair Encoding (BPE) was introduced in `Neural Machine Translation of Rare Words with Subword Units (Sennrich et
-al., 2015) <https://arxiv.org/abs/1508.07909>`__. BPE relies on a pre-tokenizer that splits the training data into
-words. Pretokenization can be as simple as space tokenization, e.g. :doc:`GPT-2 <model_doc/gpt2>`, :doc:`Roberta
-<model_doc/roberta>`. More advanced pre-tokenization include rule-based tokenization, e.g. :doc:`XLM <model_doc/xlm>`,
-:doc:`FlauBERT <model_doc/flaubert>` which uses Moses for most languages, or :doc:`GPT <model_doc/gpt>` which uses
-Spacy and ftfy, to count the frequency of each word in the training corpus.
+Byte-Pair Encoding was introduced in `this paper <https://arxiv.org/abs/1508.07909>`__. It relies on a pretokenizer
+splitting the training data into words, which can be a simple space tokenization (:doc:`GPT-2 <model_doc/gpt2>` and
+:doc:`Roberta <model_doc/roberta>` uses this for instance) or a rule-based tokenizer (:doc:`XLM <model_doc/xlm>` use
+Moses for most languages, as does :doc:`FlauBERT <model_doc/flaubert>`),

-After pre-tokenization, a set of unique words has been created and the frequency of each word it occurred in the
-training data has been determined. Next, BPE creates a base vocabulary consisting of all symbols that occur in the set
-of unique words and learns merge rules to form a new symbol from two symbols of the base vocabulary. It does so until
-the vocabulary has attained the desired vocabulary size. Note that the desired vocabulary size is a hyperparameter to
-define before training the tokenizer.
+:doc:`GPT <model_doc/gpt>` uses Spacy and ftfy, and counts the frequency of each word in the training corpus.

-As an example, let's assume that after pre-tokenization, the following set of words including their frequency has been
-determined:
+It then begins from the list of all characters and will learn merge rules to form a new token from two symbols in the
+vocabulary until it has learned a vocabulary of the desired size (this is a hyperparameter to pick).
+
+Let's say that after the pre-tokenization we have the following words (the number indicating the frequency of each
+word):

 .. code-block::

-    ("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)
+    ('hug', 10), ('pug', 5), ('pun', 12), ('bun', 4), ('hugs', 5)

-Consequently, the base vocabulary is ``["b", "g", "h", "n", "p", "s", "u"]``. Splitting all words into symbols of the
-base vocabulary, we obtain:
+Then the base vocabulary is ['b', 'g', 'h', 'n', 'p', 's', 'u'] and all our words are first split by character:

 .. code-block::

-    ("h" "u" "g", 10), ("p" "u" "g", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "u" "g" "s", 5)
+    ('h' 'u' 'g', 10), ('p' 'u' 'g', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'u' 'g' 's', 5)

-BPE then counts the frequency of each possible symbol pair and picks the symbol pair that occurs most frequently. In
-the example above ``"h"`` followed by ``"u"`` is present `10 + 5 = 15` times (10 times in the 10 occurrences of
-``"hug"``, 5 times in the 5 occurrences of "hugs"). However, the most frequent symbol pair is ``"u"`` followed by "g",
-occurring `10 + 5 + 5 = 20` times in total. Thus, the first merge rule the tokenizer learns is to group all ``"u"``
-symbols followed by a ``"g"`` symbol together. Next, "ug" is added to the vocabulary. The set of words then becomes
+We then take each pair of symbols and look at the most frequent. For instance 'hu' is present `10 + 5 = 15` times (10
+times in the 10 occurrences of 'hug', 5 times in the 5 occurrences of 'hugs'). The most frequent here is 'ug', present
+`10 + 5 + 5 = 20` times in total. So the first merge rule the tokenizer learns is to group all 'u' and 'g' together
+then it adds 'ug' to the vocabulary. Our corpus then becomes

 .. code-block::

-    ("h" "ug", 10), ("p" "ug", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "ug" "s", 5)
+    ('h' 'ug', 10), ('p' 'ug', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'ug' 's', 5)

-BPE then identifies the next most common symbol pair. It's ``"u"`` followed by ``"n"``, which occurs 16 times. ``"u"``,
-``"n"`` is merged to ``"un"`` and added to the vocabulary. The next most frequent symbol pair is ``"h"`` followed by
-``"ug"``, occurring 15 times. Again the pair is merged and ``"hug"`` can be added to the vocabulary.
+and we continue by looking at the next most common pair of symbols. It's 'un', present 16 times, so we merge those two
+and add 'un' to the vocabulary. Then it's 'hug' (as 'h' + 'ug'), present 15 times, so we merge those two and add 'hug'
+to the vocabulary.

-At this stage, the vocabulary is ``["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"]`` and our set of unique words
-is represented as
+At this stage, the vocabulary is ``['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']`` and our corpus is
+represented as

 .. code-block::

-    ("hug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("hug" "s", 5)
+    ('hug', 10), ('p' 'ug', 5), ('p' 'un', 12), ('b' 'un', 4), ('hug' 's', 5)

-Assuming, that the Byte-Pair Encoding training would stop at this point, the learned merge rules would then be applied
-to new words (as long as those new words do not include symbols that were not in the base vocabulary). For instance,
-the word ``"bug"`` would be tokenized to ``["b", "ug"]`` but ``"mug"`` would be tokenized as ``["<unk>", "ug"]`` since
-the symbol ``"m"`` is not in the base vocabulary. In general, single letters such as ``"m"`` are not replaced by the
-``"<unk>"`` symbol because the training data usually includes at least one occurrence of each letter, but it is likely
-to happen for very special characters like emojis.
+If we stop there, the tokenizer can apply the rules it learned to new words (as long as they don't contain characters
+that were not in the base vocabulary). For instance 'bug' would be tokenized as ``['b', 'ug']`` but mug would be
+tokenized as ``['<unk>', 'ug']`` since the 'm' is not in the base vocabulary. This doesn't happen to letters in general
+(since the base corpus uses all of them), but to special characters like emojis.

-As mentioned earlier, the vocabulary size, *i.e.* the base vocabulary size + the number of merges, is a hyperparameter
+As we said before, the vocabulary size (which is the base vocabulary size + the number of merges) is a hyperparameter
 to choose. For instance :doc:`GPT <model_doc/gpt>` has a vocabulary size of 40,478 since they have 478 base characters
-and chose to stop training after 40,000 merges.
+and chose to stop the training of the tokenizer at 40,000 merges.

 Byte-level BPE
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-A base vocabulary that includes all possible base characters can be quite large if *e.g.* all unicode characters are
-considered as base characters. To have a better base vocabulary, `GPT-2
-<https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf>`__ uses bytes
-as the base vocabulary, which is a clever trick to force the base vocabulary to be of size 256 while ensuring that
-every base character is included in the vocabulary. With some additional rules to deal with punctuation, the GPT2's
-tokenizer can tokenize every text without the need for the <unk> symbol. :doc:`GPT-2 <model_doc/gpt>` has a vocabulary
-size of 50,257, which corresponds to the 256 bytes base tokens, a special end-of-text token and the symbols learned
-with 50,000 merges.
+To deal with the fact the base vocabulary needs to get all base characters, which can be quite big if one allows for
+all unicode characters, the `GPT-2 paper
+<https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf>`__ introduces a
+clever trick, which is to use bytes as the base vocabulary (which gives a size of 256). With some additional rules to
+deal with punctuation, this manages to be able to tokenize every text without needing an unknown token. For instance,
+the :doc:`GPT-2 model <model_doc/gpt>` has a vocabulary size of 50,257, which corresponds to the 256 bytes base tokens,
+a special end-of-text token and the symbols learned with 50,000 merges.

 .. _wordpiece:

 WordPiece
 =======================================================================================================================

-WordPiece is the subword tokenization algorithm used for :doc:`BERT <model_doc/bert>`, :doc:`DistilBERT
-<model_doc/distilbert>`, and :doc:`Electra <model_doc/electra>`. The algorithm was outlined in `Japanese and Korean
-Voice Seach (Schuster et al., 2012)
-<https://static.googleusercontent.com/media/research.google.com/ja//pubs/archive/37842.pdf>`__ and is very similar to
-BPE. WordPiece first initializes the vocabulary to include every character present in the training data and
-progressively learn a given number of merge rules. In contrast to BPE, WordPiece does not choose the most frequent
-symbol pair, but the one that maximizes the likelihood of the training data once added to the vocabulary.
+WordPiece is the subword tokenization algorithm used for :doc:`BERT <model_doc/bert>` (as well as :doc:`DistilBERT
+<model_doc/distilbert>` and :doc:`Electra <model_doc/electra>`) and was outlined in `this paper
+<https://static.googleusercontent.com/media/research.google.com/ja//pubs/archive/37842.pdf>`__. It relies on the same
+base as BPE, which is to initialize the vocabulary to every character present in the corpus and progressively learn a
+given number of merge rules, the difference is that it doesn't choose the pair that is the most frequent but the one
+that will maximize the likelihood on the corpus once merged.

-So what does this mean exactly? Referring to the previous example, maximizing the likelihood of the training data is
-equivalent to finding the symbol pair, whose probability divided by the probabilities of its first symbol followed by
-its second symbol is the greatest among all symbol pairs. *E.g.* ``"u"``, followed by ``"g"`` would have only been
-merged if the probability of ``"ug"`` divided by ``"u"``, ``"g"`` would have been greater than for any other symbol
-pair. Intuitively, WordPiece is slightly different to BPE in that it evaluates what it `loses` by merging two symbols
-to make ensure it's `worth it`.
+What does this mean? Well, in the previous example, it means we would only merge 'u' and 'g' if the probability of
+having 'ug' divided by the probability of having 'u' then 'g' is greater than for any other pair of symbols. It's
+subtly different from what BPE does in the sense that it evaluates what it "loses" by merging two symbols and makes
+sure it's `worth it`.

 .. _unigram:

 Unigram
 =======================================================================================================================

-Unigram is a subword tokenization algorithm introduced in `Subword Regularization: Improving Neural Network Translation
-Models with Multiple Subword Candidates (Kudo, 2018) <https://arxiv.org/pdf/1804.10959.pdf>`__. In contrast to BPE or
-WordPiece, Unigram initializes its base vocabulary to a large number of symbols and progressively trims down each
-symbol to obtain a smaller vocabulary. The base vocabulary could for instance correspond to all pre-tokenized words and
-the most common substrings. Unigram is not used directly for any of the models in the transformers, but it's used in
-conjunction with :ref:`SentencePiece <sentencepiece>`.
+Unigram is a subword tokenization algorithm introduced in `this paper <https://arxiv.org/pdf/1804.10959.pdf>`__.
+Instead of starting with a group of base symbols and learning merges with some rule, like BPE or WordPiece, it starts
+from a large vocabulary (for instance, all pretokenized words and the most common substrings) that it will trim down
+progressively. It's not used directly for any of the pretrained models in the library, but it's used in conjunction
+with :ref:`SentencePiece <sentencepiece>`.

-At each training step, the Unigram algorithm defines a loss (often defined as the log-likelihood) over the training
-data given the current vocabulary and a unigram language model. Then, for each symbol in the vocabulary, the algorithm
-computes how much the overall loss would increase if the symbol was to be removed from the vocabulary. Unigram then
-removes p (with p usually being 10% or 20%) percent of the symbols whose loss increase is the lowest, *i.e.* those
-symbols that least affect the overall loss over the training data. This process is repeated until the vocabulary has
-reached the desired size. The Unigram algorithm always keeps the base characters so that any word can be tokenized.
+More specifically, at a given step, unigram computes a loss from the corpus we have and the current vocabulary, then,
+for each subword, evaluate how much the loss would increase if the subword was removed from the vocabulary. It then
+sorts the subwords by this quantity (that represents how much worse the loss becomes if the token is removed) and
+removes all the worst p tokens (for instance p could be 10% or 20%). It then repeats the process until the vocabulary
+has reached the desired size, always keeping the base characters (to be able to tokenize any word written with them,
+like BPE or WordPiece).

-Because Unigram is not based on merge rules (in contrast to BPE and WordPiece), the algorithm has several ways of
-tokenizing new text after training. As an example, if a trained Unigram tokenizer exhibits the vocabulary:
+Contrary to BPE and WordPiece that work out rules in a certain order that you can then apply in the same order when
+tokenizing new text, Unigram will have several ways of tokenizing a new text. For instance, if it ends up with the
+vocabulary

 .. code-block::

-    ["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"],
+    ['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']

-``"hugs"`` could be tokenized both as ``["hug", "s"]``, ``["h", "ug", "s"]`` or ``["h", "u", "g", "s"]``. So which one
-to choose? Unigram saves the probability of each token in the training corpus on top of saving the vocabulary so that
-the probability of each possible tokenization can be computed after training. The algorithm simply picks the most
-likely tokenization in practice, but also offers the possibility to sample a possible tokenization according to their
-probabilities.
+we had before, it could tokenize "hugs" as ``['hug', 's']``, ``['h', 'ug', 's']`` or ``['h', 'u', 'g', 's']``. So which
+one choose? On top of saving the vocabulary, the trained tokenizer will save the probability of each token in the
+training corpus. You can then give a probability to each tokenization (which is the product of the probabilities of the
+tokens forming it) and pick the most likely one (or if you want to apply some data augmentation, you could sample one
+of the tokenization according to their probabilities).

-Those probabilities are defined by the loss the tokenizer is trained on. Assuming that the training data consists of
-the words :math:`x_{1}, \dots, x_{N}` and that the set of all possible tokenizations for a word :math:`x_{i}` is
-defined as :math:`S(x_{i})`, then the overall loss is defined as
+Those probabilities define the loss that trains the tokenizer: if our corpus consists of the words :math:`x_{1}, \dots,
+x_{N}` and if for the word :math:`x_{i}` we note :math:`S(x_{i})` the set of all possible tokenizations of
+:math:`x_{i}` (with the current vocabulary), then the loss is defined as

 .. math::
    \mathcal{L} = -\sum_{i=1}^{N} \log \left ( \sum_{x \in S(x_{i})} p(x) \right )
@@ -247,18 +227,15 @@ defined as :math:`S(x_{i})`, then the overall loss is defined as
 SentencePiece
 =======================================================================================================================

-All tokenization algorithms described so far have the same problem: It is assumed that the input text uses spaces to
-separate words. However, not all languages use spaces to separate words. One possible solution is to use language
-specific pre-tokenizers, *e.g.* :doc:`XLM <model_doc/xlm>` uses a specific Chinese, Japanese, and Thai pre-tokenizer).
-To solve this problem more generally, `SentencePiece: A simple and language independent subword tokenizer and
-detokenizer for Neural Text Processing (Kudo et al., 2018) <https://arxiv.org/pdf/1808.06226.pdf>`__ treats the input
-as a raw input stream, thus including the space in the set of characters to use. It then uses the BPE or unigram
-algorithm to construct the appropriate vocabulary.
+All the methods we have been looking at so far required some form of pretokenization, which has a central problem: not
+all languages use spaces to separate words. This is a problem :doc:`XLM <model_doc/xlm>` solves by using specific
+pretokenizers for each of those languages (in this case, Chinese, Japanese and Thai). To solve this problem,
+SentencePiece (introduced in `this paper <https://arxiv.org/pdf/1808.06226.pdf>`__) treats the input as a raw stream,
+includes the space in the set of characters to use, then uses BPE or unigram to construct the appropriate vocabulary.

-The :class:`~transformers.XLNetTokenizer` uses SentencePiece for example, which is also why in the example earlier the
-``"▁"`` character was included in the vocabulary. Decoding with SentencePiece is very easy since all tokens can just be
-concatenated and ``"▁"`` is replaced by a space.
+That's why in the example we saw before using :class:`~transformers.XLNetTokenizer` (which uses SentencePiece), we had
+the '▁' character, that represents space. Decoding a tokenized text is then super easy: we just have to concatenate all
+of them together and replace '▁' with space.

-All transformers models in the library that use SentencePiece use it in combination with unigram. Examples of models
-using SentencePiece are :doc:`ALBERT <model_doc/albert>`, :doc:`XLNet <model_doc/xlnet>`, :doc:`Marian
-<model_doc/marian>`, and :doc:`T5 <model_doc/t5>`.
+All transformers models in the library that use SentencePiece use it with unigram. Examples of models using it are
+:doc:`ALBERT <model_doc/albert>`, :doc:`XLNet <model_doc/xlnet>` or the :doc:`Marian framework <model_doc/marian>`.
@@ -57,8 +57,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -25,7 +25,7 @@ class PlotArguments:
    )
    plot_along_batch: bool = field(
        default=False,
-        metadata={"help": "Whether to plot along batch size or sequence length. Defaults to sequence length."},
+        metadata={"help": "Whether to plot along batch size or sequence lengh. Defaults to sequence length."},
    )
    is_time: bool = field(
        default=False,
@@ -21,7 +21,7 @@ import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss

 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.albert.modeling_albert import (
+from transformers.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
@@ -23,7 +23,7 @@ from torch import nn
 from torch.nn import CrossEntropyLoss, MSELoss

 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.bert.modeling_bert import (
+from transformers.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
@@ -476,7 +476,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -298,7 +298,7 @@ def main():
        "--cache_dir",
        default=None,
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--data_subset", type=int, default=-1, help="If > 0: limit the data to a subset of data_subset instances."
@@ -81,8 +81,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -350,7 +350,7 @@ def main():
        "--cache_dir",
        default=None,
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -1,6 +1,7 @@
 import torch

-from transformers import CamembertForMaskedLM, CamembertTokenizer
+from transformers.modeling_camembert import CamembertForMaskedLM
+from transformers.tokenization_camembert import CamembertTokenizer


 def fill_mask(masked_input, model, tokenizer, topk=5):
@@ -3,7 +3,7 @@ import json
 from typing import List

 from ltp import LTP
-from transformers import BertTokenizer
+from transformers.tokenization_bert import BertTokenizer


 def _is_chinese_char(cp):
@@ -32,14 +32,8 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 import transformers
-from transformers import (
-    WEIGHTS_NAME,
-    AdamW,
-    AutoConfig,
-    AutoModelForMultipleChoice,
-    AutoTokenizer,
-    get_linear_schedule_with_warmup,
-)
+from transformers import WEIGHTS_NAME, AdamW, AutoConfig, AutoTokenizer, get_linear_schedule_with_warmup
+from transformers.modeling_auto import AutoModelForMultipleChoice
 from transformers.trainer_utils import is_main_process


@@ -452,7 +452,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -3,7 +3,7 @@ from torch import nn
 from torch.nn import CrossEntropyLoss, MSELoss

 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.bert.modeling_bert import (
+from transformers.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
@@ -3,13 +3,9 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss

-from transformers import RobertaConfig
+from transformers.configuration_roberta import RobertaConfig
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.roberta.modeling_roberta import (
-    ROBERTA_INPUTS_DOCSTRING,
-    ROBERTA_START_DOCSTRING,
-    RobertaEmbeddings,
-)
+from transformers.modeling_roberta import ROBERTA_INPUTS_DOCSTRING, ROBERTA_START_DOCSTRING, RobertaEmbeddings

 from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy

@@ -17,7 +17,7 @@ This folder contains the original code used to train Distil* as well as examples

 ## What is Distil*

-Distil* is a class of compressed models that started with DistilBERT. DistilBERT stands for Distilled-BERT. DistilBERT is a small, fast, cheap and light Transformer model based on Bert architecture. It has 40% less parameters than `bert-base-uncased`, runs 60% faster while preserving 97% of BERT's performances as measured on the GLUE language understanding benchmark. DistilBERT is trained using knowledge distillation, a technique to compress a large model called the teacher into a smaller model called the student. By distillating Bert, we obtain a smaller Transformer model that bears a lot of similarities with the original BERT model while being lighter, smaller and faster to run. DistilBERT is thus an interesting option to put large-scaled trained Transformer model into production.
+Distil* is a class of compressed models that started with DistilBERT. DistilBERT stands for Distillated-BERT. DistilBERT is a small, fast, cheap and light Transformer model based on Bert architecture. It has 40% less parameters than `bert-base-uncased`, runs 60% faster while preserving 97% of BERT's performances as measured on the GLUE language understanding benchmark. DistilBERT is trained using knowledge distillation, a technique to compress a large model called the teacher into a smaller model called the student. By distillating Bert, we obtain a smaller Transformer model that bears a lot of similarities with the original BERT model while being lighter, smaller and faster to run. DistilBERT is thus an interesting option to put large-scaled trained Transformer model into production.

 We have applied the same method to other Transformer architectures and released the weights:
 - GPT2: on the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test set of 16.3 compared to 21.1 for **DistilGPT2** (after fine-tuning on the train set).
@@ -57,7 +57,7 @@ Here are the results on the *test* sets for 6 of the languages available in XNLI

 This part of the library has only be tested with Python3.6+. There are few specific dependencies to install before launching a distillation, you can install them with the command `pip install -r requirements.txt`.

-**Important note:** The training scripts have been updated to support PyTorch v1.2.0 (there are breaking changes compared to v1.1.0).
+**Important note:** The training scripts have been updated to support PyTorch v1.2.0 (there are breakings changes compared to v1.1.0).


 ## How to use DistilBERT
@@ -111,7 +111,7 @@ python scripts/binarized_data.py \
    --dump_file data/binarized_text
 ```

-Our implementation of masked language modeling loss follows [XLM](https://github.com/facebookresearch/XLM)'s one and smooths the probability of masking with a factor that put more emphasis on rare words. Thus we count the occurrences of each tokens in the data:
+Our implementation of masked language modeling loss follows [XLM](https://github.com/facebookresearch/XLM)'s one and smoothes the probability of masking with a factor that put more emphasis on rare words. Thus we count the occurrences of each tokens in the data:

 ```bash
 python scripts/token_counts.py \
@@ -173,7 +173,7 @@ python -m torch.distributed.launch \
        --token_counts data/token_counts.bert-base-uncased.pickle
 ```

-**Tips:** Starting distilled training with good initialization of the model weights is crucial to reach decent performance. In our experiments, we initialized our model from a few layers of the teacher (Bert) itself! Please refer to `scripts/extract.py` and `scripts/extract_distilbert.py` to create a valid initialization checkpoint and use `--student_pretrained_weights` argument to use this initialization for the distilled training!
+**Tips:** Starting distillated training with good initialization of the model weights is crucial to reach decent performance. In our experiments, we initialized our model from a few layers of the teacher (Bert) itself! Please refer to `scripts/extract.py` and `scripts/extract_distilbert.py` to create a valid initialization checkpoint and use `--student_pretrained_weights` argument to use this initialization for the distilled training!

 Happy distillation!

@@ -188,7 +188,7 @@ class Distiller:

    def prepare_batch_mlm(self, batch):
        """
-        Prepare the batch: from the token_ids and the lengths, compute the attention mask and the masked label for MLM.
+        Prepare the batch: from the token_ids and the lenghts, compute the attention mask and the masked label for MLM.

        Input:
        ------
@@ -200,7 +200,7 @@ class Distiller:
        -------
            token_ids: `torch.tensor(bs, seq_length)` - The token ids after the modifications for MLM.
            attn_mask: `torch.tensor(bs, seq_length)` - The attention mask for the self-attention.
-            mlm_labels: `torch.tensor(bs, seq_length)` - The masked language modeling labels. There is a -100 where there is nothing to predict.
+            mlm_labels: `torch.tensor(bs, seq_length)` - The masked languge modeling labels. There is a -100 where there is nothing to predict.
        """
        token_ids, lengths = batch
        token_ids, lengths = self.round_batch(x=token_ids, lengths=lengths)
@@ -253,7 +253,7 @@ class Distiller:

    def prepare_batch_clm(self, batch):
        """
-        Prepare the batch: from the token_ids and the lengths, compute the attention mask and the labels for CLM.
+        Prepare the batch: from the token_ids and the lenghts, compute the attention mask and the labels for CLM.

        Input:
        ------
@@ -578,7 +578,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )

    parser.add_argument(
@@ -86,7 +86,7 @@ if __name__ == "__main__":
            compressed_sd[f"vocab_layer_norm.{w}"] = state_dict[f"cls.predictions.transform.LayerNorm.{w}"]

    print(f"N layers selected for distillation: {std_idx}")
-    print(f"Number of params transferred for distillation: {len(compressed_sd.keys())}")
+    print(f"Number of params transfered for distillation: {len(compressed_sd.keys())}")

-    print(f"Save transferred checkpoint to {args.dump_checkpoint}.")
+    print(f"Save transfered checkpoint to {args.dump_checkpoint}.")
    torch.save(compressed_sd, args.dump_checkpoint)
@@ -90,7 +90,7 @@ selected tokens (which may be part of words), they mask randomly selected words
 to that word). This technique has been refined for Chinese in [this paper](https://arxiv.org/abs/1906.08101).

 To fine-tune a model using whole word masking, use the following script:
-```bash
+
 python run_mlm_wwm.py \
    --model_name_or_path roberta-base \
    --dataset_name wikitext \
@@ -164,7 +164,7 @@ context length for permutation language modeling.
 The `--max_span_length` flag may also be used to limit the length of a span of masked tokens used 
 for permutation language modeling.

-Here is how to fine-tune XLNet on wikitext-2:
+Here is how to fine-tun XLNet on wikitext-2:

 ```bash
 python run_plm.py \
@@ -76,8 +76,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -313,12 +312,9 @@ def main():

    # Training
    if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
        )
-        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

    # Evaluation
@@ -74,8 +74,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -354,12 +353,9 @@ def main():

    # Training
    if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
        )
-        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

    # Evaluation
@@ -76,8 +76,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -302,12 +301,9 @@ def main():

    # Training
    if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
        )
-        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

    # Evaluation
@@ -64,8 +64,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -344,12 +343,9 @@ def main():

    # Training
    if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
        )
-        trainer.train(model_path=model_path)
        trainer.save_model()  # Saves the tokenizer too for easy upload

    # Evaluation
@@ -236,7 +236,7 @@ class BaseTransformer(pl.LightningModule):
            "--cache_dir",
            default="",
            type=str,
-            help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+            help="Where do you want to store the pre-trained models downloaded from s3",
        )
        parser.add_argument(
            "--encoder_layerdrop",
@@ -384,8 +384,6 @@ def generic_train(
        train_params["distributed_backend"] = "ddp"

    train_params["accumulate_grad_batches"] = args.accumulate_grad_batches
-    train_params["accelerator"] = extra_train_kwargs.get("accelerator", None)
-    train_params["profiler"] = extra_train_kwargs.get("profiler", None)

    trainer = pl.Trainer.from_argparse_args(
        args,
@@ -107,12 +107,7 @@ def make_support(question, source="wiki40b", method="dense", n_results=10):
    return question_doc, support_list


-@st.cache(
-    hash_funcs={
-        torch.Tensor: (lambda _: None),
-        transformers.models.bart.tokenization_bart.BartTokenizer: (lambda _: None),
-    }
-)
+@st.cache(hash_funcs={torch.Tensor: (lambda _: None), transformers.tokenization_bart.BartTokenizer: (lambda _: None)})
 def answer_question(
    question_doc, s2s_model, s2s_tokenizer, min_len=64, max_len=256, sampling=False, n_beams=2, top_p=0.95, temp=0.8
 ):
@@ -21,7 +21,7 @@ You can also have a look at this fun *Explain Like I'm Five* introductory [slide

 One promise of extreme pruning is to obtain extremely small models that can be easily sent (and stored) on edge devices. By setting weights to 0., we reduce the amount of information we need to store, and thus decreasing the memory size. We are able to obtain extremely sparse fine-pruned models with movement pruning: ~95% of the dense performance with ~5% of total remaining weights in the BERT encoder.

-In [this notebook](https://github.com/huggingface/transformers/blob/master/examples/movement-pruning/Saving_PruneBERT.ipynb), we showcase how we can leverage standard tools that exist out-of-the-box to efficiently store an extremely sparse question answering model (only 6% of total remaining weights in the encoder). We are able to reduce the memory size of the encoder **from the 340MB (the original dense BERT) to 11MB**, without any additional training of the model (every operation is performed *post fine-pruning*). It is sufficiently small to store it on a [91' floppy disk](https://en.wikipedia.org/wiki/Floptical) 📎!
+In [this notebook](https://github.com/huggingface/transformers/blob/master/examples/movement-pruning/Saving_PruneBERT.ipynb), we showcase how we can leverage standard tools that exist out-of-the-box to efficiently store an extremely sparse question answering model (only 6% of total remaining weights in the encoder). We are able to reduce the memory size of the encoder **from the 340MB (the orignal dense BERT) to 11MB**, without any additional training of the model (every operation is performed *post fine-pruning*). It is sufficiently small to store it on a [91' floppy disk](https://en.wikipedia.org/wiki/Floptical) 📎!

 While movement pruning does not directly optimize for memory footprint (but rather the number of non-null weights), we hypothetize that further memory compression ratios can be achieved with specific quantization aware trainings (see for instance [Q8BERT](https://arxiv.org/abs/1910.06188), [And the Bit Goes Down](https://arxiv.org/abs/1907.05686) or [Quant-Noise](https://arxiv.org/abs/2004.07320)).

@@ -16,7 +16,7 @@
 """Masked Version of BERT. It replaces the `torch.nn.Linear` layers with
 :class:`~emmental.MaskedLinear` and add an additional parameters in the forward pass to
 compute the adaptive mask.
-Built on top of `transformers.models.bert.modeling_bert`"""
+Built on top of `transformers.modeling_bert`"""


 import logging
@@ -29,8 +29,8 @@ from torch.nn import CrossEntropyLoss, MSELoss
 from emmental import MaskedBertConfig
 from emmental.modules import MaskedLinear
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
+from transformers.modeling_bert import ACT2FN, BertLayerNorm, load_tf_weights_in_bert
 from transformers.modeling_utils import PreTrainedModel, prune_linear_layer
-from transformers.models.bert.modeling_bert import ACT2FN, BertLayerNorm, load_tf_weights_in_bert


 logger = logging.getLogger(__name__)
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-Binarizers take a (real value) matrix as input and produce a binary (values in {0,1}) mask of the same shape.
+Binarizers take a (real value) matrice as input and produce a binary (values in {0,1}) mask of the same shape.
 """

 import torch
@@ -620,7 +620,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -725,7 +725,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )

    parser.add_argument(
@@ -61,8 +61,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -65,8 +65,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -532,7 +532,7 @@ def main():
        "--cache_dir",
        default="",
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )

    parser.add_argument(
@@ -51,8 +51,7 @@ class ModelArguments:
    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
    # or just modify its tokenizer_config.json.
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -63,8 +63,7 @@ class ModelArguments:
    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
    # or just modify its tokenizer_config.json.
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -7,9 +7,9 @@ to the retriever to extract relevant context documents. The documents are then p
 Such contextualized inputs are passed to the generator.

 Read more about RAG  at https://arxiv.org/abs/2005.11401.
-
 # Finetuning

+
 Our finetuning logic is based on scripts from [`examples/seq2seq`](https://github.com/huggingface/transformers/tree/master/examples/seq2seq). We accept training data in the same format as specified there - we expect a directory consisting of 6 text files:
 ```bash
 train.source
@@ -20,10 +20,10 @@ test.source
 test.target
 ```

-A sample finetuning command (run ` ./examples/rag/finetune_rag.py --help` to list all available options):
+A sample finetuning command (run ` ./examples/rag/finetune.py --help` to list all available options):

 ```bash
-python examples/rag/finetune_rag.py \
+python examples/rag/finetune.py \
    --data_dir $DATA_DIR \
    --output_dir $OUTPUT_DIR \
    --model_name_or_path $MODEL_NAME_OR_PATH \
@@ -45,7 +45,7 @@ python examples/rag/consolidate_rag_checkpoint.py \
    --question_encoder_name_or_path facebook/dpr-question_encoder-single-nq-base \
    --dest path/to/checkpoint
 ```
-You will then be able to pass `path/to/checkpoint` as `model_name_or_path` to the `finetune_rag.py` script.
+You will then be able to pass `path/to/checkpoint` as `model_name_or_path` to the `finetune.py` script.


 # Evaluation
@@ -130,29 +130,3 @@ python examples/rag/eval_rag.py \
    --print_predictions \
    --recalculate \ # adding this parameter will force recalculating predictions even if predictions_path already exists
 ```
-
-# Use your own knowledge source
-
-By default, RAG uses the English Wikipedia as a knowledge source, known as the 'wiki_dpr' dataset.
-With `use_custom_knowledge_dataset.py` you can build your own knowledge source, *e.g.* for RAG.
-
-For instance, if documents are serialized as tab-separated csv files with the columns "title" and "text", one can use `use_own_knowledge_dataset.py` as follows:
-```bash
-python examples/rag/use_own_knowledge_dataset.py \
-    --csv_path path/to/my_csv \
-    --output_dir path/to/my_knowledge_dataset \
-```
-
-The created outputs in `path/to/my_knowledge_dataset` can then be used to finetune RAG as follows:
-```bash
-python examples/rag/finetune_rag.py \
-    --data_dir $DATA_DIR \
-    --output_dir $OUTPUT_DIR \
-    --model_name_or_path $MODEL_NAME_OR_PATH \
-    --model_type rag_sequence \
-    --fp16 \
-    --gpus 8
-    --index_name custom
-    --passages_path path/to/data/my_knowledge_dataset
-    --index_path path/to/my_knowledge_dataset_hnsw_index.faiss
-```
@@ -8,7 +8,7 @@ import torch
 from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
 from pytorch_lightning.utilities import rank_zero_only

-from utils_rag import save_json
+from utils import save_json


 def count_trainable_parameters(model):
@@ -38,7 +38,7 @@ def get_checkpoint_callback(output_dir, metric):
        monitor=f"val_{metric}",
        mode="max",
        save_top_k=3,
-        period=1,  # maybe save a checkpoint every time val is run, not just end of epoch.
+        period=0,  # maybe save a checkpoint every time val is run, not just end of epoch.
    )
    return checkpoint_callback

@@ -27,7 +27,7 @@ class RagPyTorchDistributedRetriever(RagRetriever):
            It is used to decode the question and then use the generator_tokenizer.
        generator_tokenizer (:class:`~transformers.PretrainedTokenizer`):
            The tokenizer used for the generator part of the RagModel.
-        index (:class:`~transformers.models.rag.retrieval_rag.Index`, optional, defaults to the one defined by the configuration):
+        index (:class:`~transformers.retrieval_rag.Index`, optional, defaults to the one defined by the configuration):
            If specified, use this index instead of the one built using the configuration
    """

@@ -40,6 +40,7 @@ class RagPyTorchDistributedRetriever(RagRetriever):
            generator_tokenizer=generator_tokenizer,
            index=index,
        )
+
        self.process_group = None

    def init_retrieval(self, distributed_port: int):
@@ -1,10 +1,12 @@
 """Finetuning script for RAG models. Adapted from examples.seq2seq.finetune.py"""

 import argparse
+import glob
 import logging
 import os
 import sys
 import time
+import warnings
 from collections import defaultdict
 from pathlib import Path
 from typing import Any, Dict, List, Tuple
@@ -13,31 +15,29 @@ import numpy as np
 import pytorch_lightning as pl
 import torch
 import torch.distributed as dist
-from pytorch_lightning.accelerators.ddp_accelerator import DDPAccelerator
-from pytorch_lightning.cluster_environments import TorchElasticEnvironment
 from torch.utils.data import DataLoader

 from transformers import (
    AutoConfig,
    AutoTokenizer,
    BartForConditionalGeneration,
-    BatchEncoding,
    RagConfig,
    RagSequenceForGeneration,
    RagTokenForGeneration,
    RagTokenizer,
    T5ForConditionalGeneration,
+    get_linear_schedule_with_warmup,
 )
 from transformers import logging as transformers_logging


-from callbacks_rag import (  # noqa: E402 # isort:skipq
+from callbacks import (  # noqa: E402 # isort:skipq
    get_checkpoint_callback,
    get_early_stopping_callback,
    Seq2SeqLoggingCallback,
 )
 from distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
-from utils_rag import (  # noqa: E402 # isort:skip
+from utils import (  # noqa: E402 # isort:skip
    calculate_exact_match,
    flatten_list,
    get_git_info,
@@ -67,30 +67,6 @@ class AttrDict(dict):
        self.__dict__ = self


-# In PTL >v1.0, `init_ddp_connection` method in the `LightningModule`
-# is no longer used, and is moved into DDPAccelerator instead.
-# We override DDPAccelerator to add our custom logic for initializing the
-# retriever.
-# https://github.com/PyTorchLightning/pytorch-lightning/blob/master/tests/backends/test_accelerator_connector.py
-
-
-class CustomAccel(DDPAccelerator):
-    def __init__(self, trainer=None, **kwargs):
-        # Trainer is set later.
-        super().__init__(trainer, **kwargs)
-
-    def init_ddp_connection(self, global_rank: int, world_size: int, is_slurm_managing_tasks: bool = True):
-        logger.info("Custom init_ddp_connection.")
-        module = self.trainer.model
-        if self.cluster_environment is None:
-            self.cluster_environment = TorchElasticEnvironment()
-        self.distributed_port = module.hparams.distributed_port
-        os.environ["MASTER_PORT"] = str(self.distributed_port)
-        super().init_ddp_connection(global_rank, world_size, is_slurm_managing_tasks)
-        if module.is_rag_model:
-            module.model.rag.retriever.init_retrieval(self.distributed_port)
-
-
 class GenerativeQAModule(BaseTransformer):
    mode = "generative_qa"
    loss_names = ["loss"]
@@ -115,24 +91,23 @@ class GenerativeQAModule(BaseTransformer):
        config = config_class.from_pretrained(hparams.model_name_or_path)

        # set retriever parameters
-        config.index_name = hparams.index_name or config.index_name
-        config.passages_path = hparams.passages_path or config.passages_path
-        config.index_path = hparams.index_path or config.index_path
-        config.use_dummy_dataset = hparams.use_dummy_dataset
+        config.index_name = args.index_name or config.index_name
+        config.passages_path = args.passages_path or config.passages_path
+        config.index_path = args.index_path or config.index_path

        # set extra_model_params for generator configs and load_model
        extra_model_params = ("encoder_layerdrop", "decoder_layerdrop", "attention_dropout", "dropout")
        if self.is_rag_model:
-            if hparams.prefix is not None:
-                config.generator.prefix = hparams.prefix
+            if args.prefix is not None:
+                config.generator.prefix = args.prefix
            config.label_smoothing = hparams.label_smoothing
            hparams, config.generator = set_extra_model_params(extra_model_params, hparams, config.generator)
            retriever = RagPyTorchDistributedRetriever.from_pretrained(hparams.model_name_or_path, config=config)
            model = self.model_class.from_pretrained(hparams.model_name_or_path, config=config, retriever=retriever)
            prefix = config.question_encoder.prefix
        else:
-            if hparams.prefix is not None:
-                config.prefix = hparams.prefix
+            if args.prefix is not None:
+                config.prefix = args.prefix
            hparams, config = set_extra_model_params(extra_model_params, hparams, config)
            model = self.model_class.from_pretrained(hparams.model_name_or_path, config=config)
            prefix = config.prefix
@@ -177,9 +152,11 @@ class GenerativeQAModule(BaseTransformer):
        self.num_workers = hparams.num_workers
        self.distributed_port = self.hparams.distributed_port

-        # For single GPU training, init_ddp_connection is not called.
-        # So we need to initialize the retrievers here.
-        if hparams.gpus <= 1:
+    def init_ddp_connection(self, global_rank: int, world_size: int, is_slurm_managing_tasks: bool = True):
+        logger.info("Custom init_ddp_connection.")
+        os.environ["MASTER_PORT"] = str(self.distributed_port)
+        super().init_ddp_connection(global_rank, world_size, is_slurm_managing_tasks)
+        if self.is_rag_model:
            self.model.retriever.init_retrieval(self.distributed_port)

    def forward(self, input_ids, **kwargs):
@@ -293,7 +270,6 @@ class GenerativeQAModule(BaseTransformer):

    def _generative_step(self, batch: dict) -> dict:
        start_time = time.time()
-        batch = BatchEncoding(batch).to(device=self.model.device)
        generated_ids = self.model.generate(
            batch["input_ids"],
            attention_mask=batch["attention_mask"],
@@ -346,6 +322,17 @@ class GenerativeQAModule(BaseTransformer):

    def train_dataloader(self) -> DataLoader:
        dataloader = self.get_dataloader("train", batch_size=self.hparams.train_batch_size, shuffle=True)
+        t_total = (
+            (len(dataloader.dataset) // (self.hparams.train_batch_size * max(1, self.hparams.gpus)))
+            // self.hparams.accumulate_grad_batches
+            * float(self.hparams.max_epochs)
+        )
+        scheduler = get_linear_schedule_with_warmup(
+            self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=t_total
+        )
+        if max(scheduler.get_last_lr()) > 0:
+            warnings.warn("All learning rates are 0")
+        self.lr_scheduler = scheduler
        return dataloader

    def val_dataloader(self) -> DataLoader:
@@ -442,24 +429,10 @@ class GenerativeQAModule(BaseTransformer):
            default=None,
            help="Path to the faiss index for custom index. More info about custom indexes in the RagRetriever documentation as well as in `examples/rag/use_own_knowledge_dataset.py`",
        )
-        parser.add_argument(
-            "--use_dummy_dataset",
-            type=bool,
-            default=False,
-            help="Whether to use the dummy version of the dataset index. More info about custom indexes in the RagRetriever documentation as well as in `examples/rag/use_own_knowledge_dataset.py`",
-        )
        return parser


-def main(args=None, model=None) -> GenerativeQAModule:
-
-    parser = argparse.ArgumentParser()
-    parser = pl.Trainer.add_argparse_args(parser)
-    parser = GenerativeQAModule.add_model_specific_args(parser, os.getcwd())
-    parser = GenerativeQAModule.add_retriever_specific_args(parser)
-
-    args = args or parser.parse_args()
-
+def main(args, model=None) -> GenerativeQAModule:
    Path(args.output_dir).mkdir(exist_ok=True)
    if model is None:
        model: GenerativeQAModule = GenerativeQAModule(args)
@@ -488,7 +461,6 @@ def main(args=None, model=None) -> GenerativeQAModule:
        if args.early_stopping_patience >= 0
        else False
    )
-
    trainer: pl.Trainer = generic_train(
        model,
        args,
@@ -496,17 +468,31 @@ def main(args=None, model=None) -> GenerativeQAModule:
        checkpoint_callback=get_checkpoint_callback(args.output_dir, model.val_metric),
        early_stopping_callback=es_callback,
        logger=logger,
-        accelerator=CustomAccel() if args.gpus > 1 else None,
    )
    pickle_save(model.hparams, model.output_dir / "hparams.pkl")

    if not args.do_predict:
        return model

+    model.hparams.test_checkpoint = ""
+    checkpoints = list(sorted(glob.glob(os.path.join(args.output_dir, "*.ckpt"), recursive=True)))
+    if checkpoints:
+        model.hparams.test_checkpoint = checkpoints[-1]
+        trainer.resume_from_checkpoint = checkpoints[-1]  # best checkpoint
+    trainer.logger.log_hyperparams(model.hparams)
+
    # test() without a model tests using the best checkpoint automatically
    trainer.test()
+
    return model


 if __name__ == "__main__":
-    main()
+    parser = argparse.ArgumentParser()
+    parser = pl.Trainer.add_argparse_args(parser)
+    parser = GenerativeQAModule.add_model_specific_args(parser, os.getcwd())
+    parser = GenerativeQAModule.add_retriever_specific_args(parser)
+
+    args = parser.parse_args()
+
+    main(args)
@@ -4,10 +4,10 @@ export PYTHONPATH="../":"${PYTHONPATH}"
 # A sample finetuning run, you need to specify data_dir, output_dir and model_name_or_path
 # run ./examples/rag/finetune.sh --help to see all the possible options

-python examples/rag/finetune_rag.py \
+python examples/rag/finetune.py \
    --data_dir $DATA_DIR \
    --output_dir $OUTPUT_DIR \
-    --model_name_or_path $MODEL_NAME_OR_PATH \
+    --model_name_or_path $MODLE_NAME_OR_PATH \
    --model_type rag_sequence \
    --fp16 \
    --gpus 8 \
@@ -11,12 +11,16 @@ import numpy as np
 from datasets import Dataset

 import faiss
-from transformers import BartConfig, BartTokenizer, DPRConfig, DPRQuestionEncoderTokenizer, RagConfig
+from transformers.configuration_bart import BartConfig
+from transformers.configuration_dpr import DPRConfig
+from transformers.configuration_rag import RagConfig
 from transformers.file_utils import is_datasets_available, is_faiss_available, is_psutil_available, is_torch_available
-from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
-from transformers.models.rag.retrieval_rag import CustomHFIndex
-from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
+from transformers.retrieval_rag import CustomHFIndex
 from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me
+from transformers.tokenization_bart import BartTokenizer
+from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
+from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer
+from transformers.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES


 sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip
@@ -133,7 +137,7 @@ class RagRetrieverTest(TestCase):
            question_encoder=DPRConfig().to_dict(),
            generator=BartConfig().to_dict(),
        )
-        with patch("transformers.models.rag.retrieval_rag.load_dataset") as mock_load_dataset:
+        with patch("transformers.retrieval_rag.load_dataset") as mock_load_dataset:
            mock_load_dataset.return_value = dataset
            retriever = RagPyTorchDistributedRetriever(
                config,
@@ -1,96 +0,0 @@
-import json
-import logging
-import os
-import sys
-from pathlib import Path
-
-import finetune_rag
-from transformers.file_utils import is_apex_available
-from transformers.testing_utils import (
-    TestCasePlus,
-    execute_subprocess_async,
-    require_torch_gpu,
-    require_torch_multi_gpu,
-)
-
-
-logging.basicConfig(level=logging.DEBUG)
-logger = logging.getLogger()
-
-
-class RagFinetuneExampleTests(TestCasePlus):
-    def _create_dummy_data(self, data_dir):
-        os.makedirs(data_dir, exist_ok=True)
-        contents = {"source": "What is love ?", "target": "life"}
-        n_lines = {"train": 12, "val": 2, "test": 2}
-        for split in ["train", "test", "val"]:
-            for field in ["source", "target"]:
-                content = "\n".join([contents[field]] * n_lines[split])
-                with open(os.path.join(data_dir, f"{split}.{field}"), "w") as f:
-                    f.write(content)
-
-    def _run_finetune(self, gpus: int):
-        stream_handler = logging.StreamHandler(sys.stdout)
-        logger.addHandler(stream_handler)
-
-        tmp_dir = self.get_auto_remove_tmp_dir()
-        output_dir = os.path.join(tmp_dir, "output")
-        data_dir = os.path.join(tmp_dir, "data")
-        self._create_dummy_data(data_dir=data_dir)
-
-        testargs = f"""
-                --data_dir {data_dir} \
-                --output_dir {output_dir} \
-                --model_name_or_path facebook/rag-sequence-base \
-                --model_type rag_sequence \
-                --do_train \
-                --do_predict \
-                --n_val -1 \
-                --val_check_interval 1.0 \
-                --train_batch_size 2 \
-                --eval_batch_size 1 \
-                --max_source_length 25 \
-                --max_target_length 25 \
-                --val_max_target_length 25 \
-                --test_max_target_length 25 \
-                --label_smoothing 0.1 \
-                --dropout 0.1 \
-                --attention_dropout 0.1 \
-                --weight_decay 0.001 \
-                --adam_epsilon 1e-08 \
-                --max_grad_norm 0.1 \
-                --lr_scheduler polynomial \
-                --learning_rate 3e-04 \
-                --num_train_epochs 1 \
-                --warmup_steps 4 \
-                --gradient_accumulation_steps 1 \
-                --distributed-port 8787 \
-                --use_dummy_dataset 1 \
-            """.split()
-
-        if gpus > 0:
-            testargs.append(f"--gpus={gpus}")
-            if is_apex_available():
-                testargs.append("--fp16")
-        else:
-            testargs.append("--gpus=0")
-            testargs.append("--distributed_backend=ddp_cpu")
-            testargs.append("--num_processes=2")
-
-        cmd = [sys.executable, str(Path(finetune_rag.__file__).resolve())] + testargs
-        execute_subprocess_async(cmd, env=self.get_env())
-
-        metrics_save_path = os.path.join(output_dir, "metrics.json")
-        with open(metrics_save_path) as f:
-            result = json.load(f)
-        return result
-
-    @require_torch_gpu
-    def test_finetune_gpu(self):
-        result = self._run_finetune(gpus=1)
-        self.assertGreaterEqual(result["test"][0]["test_avg_em"], 0.2)
-
-    @require_torch_multi_gpu
-    def test_finetune_multigpu(self):
-        result = self._run_finetune(gpus=2)
-        self.assertGreaterEqual(result["test"][0]["test_avg_em"], 0.2)
@@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory
 from typing import List, Optional

 import torch
-from datasets import Features, Sequence, Value, load_dataset
+from datasets import load_dataset

 import faiss
 from transformers import (
@@ -47,7 +47,7 @@ def embed(documents: dict, ctx_encoder: DPRContextEncoder, ctx_tokenizer: DPRCon
    input_ids = ctx_tokenizer(
        documents["title"], documents["text"], truncation=True, padding="longest", return_tensors="pt"
    )["input_ids"]
-    embeddings = ctx_encoder(input_ids.to(device=device), return_dict=True).pooler_output
+    embeddings = ctx_encoder(input_ids.to(device=device)).pooler_output
    return {"embeddings": embeddings.detach().cpu().numpy()}


@@ -82,14 +82,10 @@ def main(
    # And compute the embeddings
    ctx_encoder = DPRContextEncoder.from_pretrained(rag_example_args.dpr_ctx_encoder_model_name).to(device=device)
    ctx_tokenizer = DPRContextEncoderTokenizerFast.from_pretrained(rag_example_args.dpr_ctx_encoder_model_name)
-    new_features = Features(
-        {"text": Value("string"), "title": Value("string"), "embeddings": Sequence(Value("float32"))}
-    )  # optional, save as float32 instead of float64 to save space
    dataset = dataset.map(
        partial(embed, ctx_encoder=ctx_encoder, ctx_tokenizer=ctx_tokenizer),
        batched=True,
        batch_size=processing_args.batch_size,
-        features=new_features,
    )

    # And finally save your dataset
@@ -13,7 +13,7 @@ streamlit
 elasticsearch
 nltk
 pandas
-datasets >= 1.1.3
+datasets
 fire
 pytest
 conllu
@@ -1,4 +1,5 @@
 import logging
+import os
 from pathlib import Path

 import numpy as np
@@ -97,8 +98,7 @@ def get_checkpoint_callback(output_dir, metric, save_top_k=1, lower_is_better=Fa
        )

    checkpoint_callback = ModelCheckpoint(
-        dirpath=output_dir,
-        filename=exp,
+        filepath=os.path.join(output_dir, exp),
        monitor=f"val_{metric}",
        mode="min" if "loss" in metric else "max",
        save_top_k=save_top_k,
@@ -16,7 +16,7 @@ from finetune import SummarizationModule, TranslationModule
 from finetune import main as ft_main
 from make_student import create_student_by_copying_alternating_layers, get_layers_to_supervise
 from transformers import AutoModelForSeq2SeqLM, MBartTokenizer, T5ForConditionalGeneration
-from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.modeling_bart import shift_tokens_right
 from utils import calculate_bleu, check_output_dir, freeze_params, label_smoothed_nll_loss, use_task_specific_params


@@ -154,7 +154,7 @@ class SummarizationDistiller(SummarizationModule):
            output_attentions=False,
            use_cache=False,
        )
-        lm_logits = student_outputs["logits"]
+        lm_logits = student_outputs.logits

        # Same cross entropy vs. label smoothing logic as finetune.py
        assert lm_logits.shape[-1] == self.model.config.vocab_size
@@ -171,9 +171,7 @@ class SummarizationDistiller(SummarizationModule):
        def zero_tensor():
            return torch.tensor(0.0).type_as(student_lm_loss)

-        teacher_enc_outputs = student_outputs[
-            "encoder_last_hidden_state"
-        ]  # use this unless self.different_base_models
+        teacher_enc_outputs = student_outputs.encoder_last_hidden_state  # use this unless self.different_base_models
        hid_loss_enc, hid_loss_dec = zero_tensor(), zero_tensor()
        if self.different_encoder:  # compute encoder hidden state loss
            all_teacher_encoder_outputs = self.teacher.get_encoder()(
@@ -182,12 +180,12 @@ class SummarizationDistiller(SummarizationModule):
                output_hidden_states=self.do_calc_hidden_loss,
            )
            if self.different_base_models:
-                teacher_enc_outputs = all_teacher_encoder_outputs["last_hidden_state"]
+                teacher_enc_outputs = all_teacher_encoder_outputs.last_hidden_state
            elif self.do_calc_hidden_loss:
                hid_loss_enc = self.calc_hidden_loss(
                    src_mask,
-                    student_outputs["encoder_hidden_states"],
-                    all_teacher_encoder_outputs["hidden_states"],
+                    student_outputs.encoder_hidden_states,
+                    all_teacher_encoder_outputs.hidden_states,
                    self.e_matches,
                    normalize_hidden=self.hparams.normalize_hidden,
                )
@@ -201,12 +199,12 @@ class SummarizationDistiller(SummarizationModule):
            use_cache=False,  # since we are not passing labels, never let this default to True
        )
        dec_mask = decoder_input_ids.ne(pad_token_id)
-        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, teacher_outputs["logits"])
+        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, teacher_outputs.logits)
        if self.do_calc_hidden_loss:  # Intermediate supervision of decoder hidden states
            hid_loss_dec = self.calc_hidden_loss(
                dec_mask,
-                student_outputs["decoder_hidden_states"],
-                teacher_outputs["decoder_hidden_states"],
+                student_outputs.decoder_hidden_states,
+                teacher_outputs.decoder_hidden_states,
                self.d_matches,
                normalize_hidden=self.hparams.normalize_hidden,
            )
@@ -17,7 +17,7 @@ from torch.utils.data import DataLoader

 from callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
 from transformers import MBartTokenizer, T5ForConditionalGeneration
-from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.modeling_bart import shift_tokens_right
 from utils import (
    ROUGE_KEYS,
    LegacySeq2SeqDataset,
@@ -148,7 +148,7 @@ class SummarizationModule(BaseTransformer):
            self.save_readable_batch(batch)

        outputs = self(src_ids, attention_mask=src_mask, decoder_input_ids=decoder_input_ids, use_cache=False)
-        lm_logits = outputs["logits"]
+        lm_logits = outputs[0]
        if self.hparams.label_smoothing == 0:
            # Same behavior as modeling_bart.py, besides ignoring pad_token_id
            ce_loss_fct = torch.nn.CrossEntropyLoss(ignore_index=pad_token_id)
@@ -43,8 +43,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    freeze_encoder: bool = field(default=False, metadata={"help": "Whether tp freeze the encoder."})
    freeze_embeds: bool = field(default=False, metadata={"help": "Whether  to freeze the embeddings."})
@@ -1,12 +1,12 @@
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, Optional, Tuple, Union

 import torch
 from torch import nn
 from torch.utils.data import DistributedSampler, RandomSampler

 from transformers import PreTrainedModel, Trainer, logging
+from transformers.configuration_fsmt import FSMTConfig
 from transformers.file_utils import is_torch_tpu_available
-from transformers.models.fsmt.configuration_fsmt import FSMTConfig
 from transformers.optimization import (
    Adafactor,
    AdamW,
@@ -153,11 +153,7 @@ class Seq2SeqTrainer(Trainer):
        return loss

    def prediction_step(
-        self,
-        model: nn.Module,
-        inputs: Dict[str, Union[torch.Tensor, Any]],
-        prediction_loss_only: bool,
-        ignore_keys: Optional[List[str]] = None,
+        self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], prediction_loss_only: bool
    ) -> Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
        """
        Perform an evaluation step on :obj:`model` using obj:`inputs`.
@@ -189,7 +185,7 @@ class Seq2SeqTrainer(Trainer):
        }

        if self.args.predict_with_generate and not self.args.prediction_loss_only:
-            generated_tokens = self.model.generate(
+            generated_tokens = model.generate(
                inputs["input_ids"],
                attention_mask=inputs["attention_mask"],
                **gen_kwargs,
@@ -13,7 +13,7 @@ from distillation import SummarizationDistiller, distill_main
 from finetune import SummarizationModule, main
 from transformers import MarianMTModel
 from transformers.file_utils import cached_path
-from transformers.testing_utils import TestCasePlus, require_torch_gpu, slow
+from transformers.testing_utils import TestCasePlus, require_torch_gpu, require_torch_non_multi_gpu_but_fix_me, slow
 from utils import load_json


@@ -32,6 +32,7 @@ class TestMbartCc25Enro(TestCasePlus):

    @slow
    @require_torch_gpu
+    @require_torch_non_multi_gpu_but_fix_me
    def test_model_download(self):
        """This warms up the cache so that we can time the next test without including download time, which varies between machines."""
        MarianMTModel.from_pretrained(MARIAN_MODEL)
@@ -39,6 +40,7 @@ class TestMbartCc25Enro(TestCasePlus):
    # @timeout_decorator.timeout(1200)
    @slow
    @require_torch_gpu
+    @require_torch_non_multi_gpu_but_fix_me
    def test_train_mbart_cc25_enro_script(self):
        env_vars_to_replace = {
            "$MAX_LEN": 64,
@@ -127,6 +129,7 @@ class TestDistilMarianNoTeacher(TestCasePlus):
    @timeout_decorator.timeout(600)
    @slow
    @require_torch_gpu
+    @require_torch_non_multi_gpu_but_fix_me
    def test_opus_mt_distill_script(self):
        data_dir = f"{self.test_file_dir_str}/test_data/wmt_en_ro"
        env_vars_to_replace = {
@@ -10,7 +10,7 @@ from parameterized import parameterized
 from save_len_file import save_len_file
 from test_seq2seq_examples import ARTICLES, BART_TINY, MARIAN_TINY, MBART_TINY, SUMMARIES, T5_TINY, make_test_data_dir
 from transformers import AutoTokenizer
-from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.modeling_bart import shift_tokens_right
 from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me, slow
 from utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeq2SeqDataset, Seq2SeqDataset

@@ -138,7 +138,7 @@ class TestFinetuneTrainer(TestCasePlus):
            per_device_train_batch_size=batch_size,
            per_device_eval_batch_size=batch_size,
            predict_with_generate=True,
-            evaluation_strategy="steps",
+            evaluate_during_training=True,
            do_train=True,
            do_eval=True,
            warmup_steps=0,
@@ -179,7 +179,7 @@ class TestFinetuneTrainer(TestCasePlus):
            --per_device_eval_batch_size 4
            --learning_rate 3e-3
            --warmup_steps 8
-            --evaluation_strategy steps
+            --evaluate_during_training
            --predict_with_generate
            --logging_steps 0
            --save_steps {str(eval_steps)}
@@ -19,7 +19,13 @@ import unittest

 from parameterized import parameterized
 from transformers import FSMTForConditionalGeneration, FSMTTokenizer
-from transformers.testing_utils import get_tests_dir, require_torch, slow, torch_device
+from transformers.testing_utils import (
+    get_tests_dir,
+    require_torch,
+    require_torch_non_multi_gpu_but_fix_me,
+    slow,
+    torch_device,
+)
 from utils import calculate_bleu


@@ -48,6 +54,7 @@ class ModelEvalTester(unittest.TestCase):
        ]
    )
    @slow
+    @require_torch_non_multi_gpu_but_fix_me
    def test_bleu_scores(self, pair, min_bleu_score):
        # note: this test is not testing the best performance since it only evals a small batch
        # but it should be enough to detect a regression in the output quality
@@ -19,7 +19,14 @@ from run_eval import generate_summaries_or_translations, run_generate
 from run_eval_search import run_search
 from transformers import AutoConfig, AutoModelForSeq2SeqLM
 from transformers.hf_api import HfApi
-from transformers.testing_utils import CaptureStderr, CaptureStdout, TestCasePlus, require_torch_gpu, slow
+from transformers.testing_utils import (
+    CaptureStderr,
+    CaptureStdout,
+    TestCasePlus,
+    require_torch_gpu,
+    require_torch_non_multi_gpu_but_fix_me,
+    slow,
+)
 from utils import ROUGE_KEYS, label_smoothed_nll_loss, lmap, load_json


@@ -128,6 +135,7 @@ class TestSummarizationDistiller(TestCasePlus):

    @slow
    @require_torch_gpu
+    @require_torch_non_multi_gpu_but_fix_me
    def test_hub_configs(self):
        """I put require_torch_gpu cause I only want this to run with self-scheduled."""

@@ -145,10 +153,12 @@ class TestSummarizationDistiller(TestCasePlus):
                failures.append(m)
        assert not failures, f"The following models could not be loaded through AutoConfig: {failures}"

+    @require_torch_non_multi_gpu_but_fix_me
    def test_distill_no_teacher(self):
        updates = dict(student_encoder_layers=2, student_decoder_layers=1, no_teacher=True)
        self._test_distiller_cli(updates)

+    @require_torch_non_multi_gpu_but_fix_me
    def test_distill_checkpointing_with_teacher(self):
        updates = dict(
            student_encoder_layers=2,
@@ -173,6 +183,7 @@ class TestSummarizationDistiller(TestCasePlus):
        convert_pl_to_hf(ckpts[0], transformer_ckpts[0].parent, out_path_new)
        assert os.path.exists(os.path.join(out_path_new, "pytorch_model.bin"))

+    @require_torch_non_multi_gpu_but_fix_me
    def test_loss_fn(self):
        model = AutoModelForSeq2SeqLM.from_pretrained(BART_TINY)
        input_ids, mask = model.dummy_inputs["input_ids"], model.dummy_inputs["attention_mask"]
@@ -193,6 +204,7 @@ class TestSummarizationDistiller(TestCasePlus):
            # TODO: understand why this breaks
            self.assertEqual(nll_loss, model_computed_loss)

+    @require_torch_non_multi_gpu_but_fix_me
    def test_distill_mbart(self):
        updates = dict(
            student_encoder_layers=2,
@@ -217,6 +229,7 @@ class TestSummarizationDistiller(TestCasePlus):
        assert len(all_files) > 2
        self.assertEqual(len(transformer_ckpts), 2)

+    @require_torch_non_multi_gpu_but_fix_me
    def test_distill_t5(self):
        updates = dict(
            student_encoder_layers=1,
@@ -228,6 +241,7 @@ class TestSummarizationDistiller(TestCasePlus):
        )
        self._test_distiller_cli(updates)

+    @require_torch_non_multi_gpu_but_fix_me
    def test_distill_different_base_models(self):
        updates = dict(
            teacher=T5_TINY,
@@ -307,18 +321,21 @@ class TestTheRest(TestCasePlus):

    # test one model to quickly (no-@slow) catch simple problems and do an
    # extensive testing of functionality with multiple models as @slow separately
+    @require_torch_non_multi_gpu_but_fix_me
    def test_run_eval(self):
        self.run_eval_tester(T5_TINY)

    # any extra models should go into the list here - can be slow
    @parameterized.expand([BART_TINY, MBART_TINY])
    @slow
+    @require_torch_non_multi_gpu_but_fix_me
    def test_run_eval_slow(self, model):
        self.run_eval_tester(model)

    # testing with 2 models to validate: 1. translation (t5) 2. summarization (mbart)
    @parameterized.expand([T5_TINY, MBART_TINY])
    @slow
+    @require_torch_non_multi_gpu_but_fix_me
    def test_run_eval_search(self, model):
        input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
        output_file_name = input_file_name.parent / "utest_output.txt"
@@ -369,6 +386,7 @@ class TestTheRest(TestCasePlus):
    @parameterized.expand(
        [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY],
    )
+    @require_torch_non_multi_gpu_but_fix_me
    def test_finetune(self, model):
        args_d: dict = CHEAP_ARGS.copy()
        task = "translation" if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else "summarization"
@@ -420,6 +438,7 @@ class TestTheRest(TestCasePlus):
        assert isinstance(example_batch, dict)
        assert len(example_batch) >= 4

+    @require_torch_non_multi_gpu_but_fix_me
    def test_finetune_extra_model_args(self):
        args_d: dict = CHEAP_ARGS.copy()

@@ -470,6 +489,7 @@ class TestTheRest(TestCasePlus):
            model = main(args)
        assert str(excinfo.value) == f"model config doesn't have a `{unsupported_param}` attribute"

+    @require_torch_non_multi_gpu_but_fix_me
    def test_finetune_lr_schedulers(self):
        args_d: dict = CHEAP_ARGS.copy()

@@ -2,8 +2,8 @@ import os
 import tempfile
 import unittest

+from transformers.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
 from transformers.file_utils import cached_property
-from transformers.models.marian.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
 from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow


@@ -21,7 +21,7 @@ from torch.utils.data import Dataset, Sampler
 from sentence_splitter import add_newline_to_end_of_each_sentence
 from transformers import BartTokenizer, EvalPrediction, PreTrainedTokenizer, T5Tokenizer
 from transformers.file_utils import cached_property
-from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.modeling_bart import shift_tokens_right


 try:
@@ -124,8 +124,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    use_fast_tokenizer: bool = field(
        default=True,
@@ -117,8 +117,7 @@ class ModelArguments:
    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
    # or just modify its tokenizer_config.json.
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -182,8 +182,7 @@ class ModelArguments:
    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
    # or just modify its tokenizer_config.json.
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -406,7 +406,7 @@ def main():
        "--cache_dir",
        default=None,
        type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--max_seq_length",
@@ -34,8 +34,9 @@ import torch.nn.functional as F
 from tqdm import trange

 from pplm_classification_head import ClassificationHead
-from transformers import GPT2LMHeadModel, GPT2Tokenizer
+from transformers import GPT2Tokenizer
 from transformers.file_utils import cached_path
+from transformers.modeling_gpt2 import GPT2LMHeadModel


 PPLM_BOW = 1
@@ -15,8 +15,7 @@
 """
 Fine-tuning the library models for token classification.
 """
-# You can also adapt this script on your own token classification task and datasets. Pointers for this are left as
-# comments.
+# You can also adapt this script on your own token classification task and datasets. Pointers for this are left as comments.

 import logging
 import os
@@ -25,7 +24,7 @@ from dataclasses import dataclass, field
 from typing import Optional

 import numpy as np
-from datasets import ClassLabel, load_dataset
+from datasets import load_dataset
 from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score

 import transformers
@@ -61,8 +60,7 @@ class ModelArguments:
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -199,17 +197,12 @@ def main():

    if training_args.do_train:
        column_names = datasets["train"].column_names
-        features = datasets["train"].features
    else:
        column_names = datasets["validation"].column_names
-        features = datasets["validation"].features
-    text_column_name = "tokens" if "tokens" in column_names else column_names[0]
-    label_column_name = (
-        f"{data_args.task_name}_tags" if f"{data_args.task_name}_tags" in column_names else column_names[1]
-    )
+    text_column_name = "words" if "words" in column_names else column_names[0]
+    label_column_name = data_args.task_name if data_args.task_name in column_names else column_names[1]

-    # In the event the labels are not a `Sequence[ClassLabel]`, we will need to go through the dataset to get the
-    # unique labels.
+    # Labeling (this part will be easier when https://github.com/huggingface/datasets/issues/797 is solved)
    def get_label_list(labels):
        unique_labels = set()
        for label in labels:
@@ -218,13 +211,8 @@ def main():
        label_list.sort()
        return label_list

-    if isinstance(features[label_column_name].feature, ClassLabel):
-        label_list = features[label_column_name].feature.names
-        # No need to convert the labels since they are already ints.
-        label_to_id = {i: i for i in range(len(label_list))}
-    else:
-        label_list = get_label_list(datasets["train"][label_column_name])
-        label_to_id = {l: i for i, l in enumerate(label_list)}
+    label_list = get_label_list(datasets["train"][label_column_name])
+    label_to_id = {l: i for i, l in enumerate(label_list)}
    num_labels = len(label_list)

    # Load pretrained model and tokenizer
@@ -65,8 +65,7 @@ class ModelArguments:
    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
    # or just modify its tokenizer_config.json.
    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )


@@ -254,7 +253,7 @@ def main():
        trainer.save_model()
        # For convenience, we also re-save the tokenizer to the same directory,
        # so that you can share your model easily on huggingface.co/models =)
-        if trainer.is_world_process_zero():
+        if trainer.is_world_master():
            tokenizer.save_pretrained(training_args.output_dir)

    # Evaluation
@@ -265,7 +264,7 @@ def main():
        result = trainer.evaluate()

        output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")
-        if trainer.is_world_process_zero():
+        if trainer.is_world_master():
            with open(output_eval_file, "w") as writer:
                logger.info("***** Eval results *****")
                for key, value in result.items():
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
LysandreJik	e411222a47	Sacremoses in TransfoXL	2020-11-16 16:34:58 -05:00
LysandreJik	791b6928eb	Sacremoses in FSMT	2020-11-16 16:33:31 -05:00
LysandreJik	7211ba1231	Sacremoses	2020-11-16 16:30:51 -05:00