Release: v3.4.0

* labels and decoder_input_ids to Glossary

* Formatting fixes

* Update docs/source/glossary.rst

Co-authored-by: Sam Shleifer <sshleifer@gmail.com>

* sam's comments

Co-authored-by: Sam Shleifer <sshleifer@gmail.com>

.circleci/config.yml

@@ -84,7 +84,7 @@ jobs:
                 key: v0.3-{{ checksum "setup.py" }}
                 paths:
                     - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ --cov  | tee output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ --cov --durations=0 | tee output.txt
             - run: codecov
             - store_artifacts:
                   path: ~/transformers/output.txt
@@ -139,6 +139,31 @@ jobs:
             - store_artifacts:
                path: ~/transformers/output.txt
                destination: test_output.txt
+    run_tests_flax:
+        working_directory: ~/transformers
+        docker:
+            - image: circleci/python:3.7
+        environment:
+            OMP_NUM_THREADS: 1
+        resource_class: xlarge
+        parallelism: 1
+        steps:
+            - checkout
+            - restore_cache:
+                keys:
+                    - v0.3-flax-{{ checksum "setup.py" }}
+                    - v0.3-{{ checksum "setup.py" }}
+            - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
+            - run: sudo pip install .[flax,sklearn,torch,testing]
+            - save_cache:
+                  key: v0.3-flax-{{ checksum "setup.py" }}
+                  paths:
+                      - '~/.cache/pip'
+            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
+            - store_artifacts:
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
     run_tests_custom_tokenizers:
         working_directory: ~/transformers
         docker:
@@ -198,7 +223,7 @@ jobs:
                       - v0.3-build_doc-{{ checksum "setup.py" }}
                       - v0.3-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[tf,torch,docs]
+            - run: pip install .[tf,torch,sentencepiece,docs]
             - save_cache:
                   key: v0.3-build_doc-{{ checksum "setup.py" }}
                   paths:
@@ -219,7 +244,7 @@ jobs:
                   keys:
                       - v0.3-deploy_doc-{{ checksum "setup.py" }}
                       - v0.3-{{ checksum "setup.py" }}
-            - run: pip install .[tf,torch,docs]
+            - run: pip install .[tf,torch,sentencepiece,docs]
             - save_cache:
                   key: v0.3-deploy_doc-{{ checksum "setup.py" }}
                   paths:
@@ -239,7 +264,7 @@ jobs:
                       - v0.3-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
             - run: pip install isort
-            - run: pip install .[tf,torch,quality]
+            - run: pip install .[tf,torch,flax,quality]
             - save_cache:
                   key: v0.3-code_quality-{{ checksum "setup.py" }}
                   paths:
@@ -305,6 +330,7 @@ workflows:
             - run_tests_torch_and_tf
             - run_tests_torch
             - run_tests_tf
+            - run_tests_flax
             - build_doc
             - deploy_doc: *workflow_filters
     tpu_testing_jobs:

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,15 +16,15 @@ Fixes # (issue)
 
 
 ## Before submitting
-- [ ] This PR fixes a typo or improves the docs (you can dimiss the other checks if that's the case).
-- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md#start-contributing-pull-requests), 
+- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
+- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md#start-contributing-pull-requests),
       Pull Request section?
 - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
       to the it if that's the case.
 - [ ] Did you make sure to update the documentation with your changes? Here are the
       [documentation guidelines](https://github.com/huggingface/transformers/tree/master/docs), and
       [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/master/docs#writing-source-documentation).
-- [ ] Did you write any new necessary tests? 
+- [ ] Did you write any new necessary tests?
 
 
 ## Who can review?
@@ -37,25 +37,27 @@ members/contributors which may be interested in your PR.
  If you know how to use git blame, that is the easiest way, otherwise, here is a rough guide of **who to tag**.
  Please tag fewer than 3 people.
 
- albert, bert, GPT2, XLM: @LysandreJik 
+ albert, bert, XLM: @LysandreJik
+ GPT2: @LysandreJik, @patrickvonplaten
  tokenizers: @mfuntowicz
  Trainer: @sgugger
- Speed and Memory Benchmarks: @patrickvonplaten
+ Benchmarks: @patrickvonplaten
  Model Cards: @julien-c
  Translation: @sshleifer
  Summarization: @sshleifer
- TextGeneration: @TevenLeScao 
  examples/distillation: @VictorSanh
  nlp datasets: [different repo](https://github.com/huggingface/nlp)
  rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)
- Text Generation: @TevenLeScao
+ Text Generation: @patrickvonplaten, @TevenLeScao
  Blenderbot, Bart, Marian, Pegasus: @sshleifer
  T5: @patrickvonplaten
- Longformer/Reformer: @patrickvonplaten
- TransfoXL/XLNet: @TevenLeScao 
+ Rag: @patrickvonplaten, @lhoestq
+ EncoderDecoder: @patrickvonplaten
+ Longformer, Reformer: @patrickvonplaten
+ TransfoXL, XLNet: @TevenLeScao, @patrickvonplaten
  examples/seq2seq: @sshleifer
  examples/bert-loses-patience: @JetRunner
  tensorflow: @jplu
  examples/token-classification: @stefan-it
  documentation: @sgugger
- -->
+ -->

.github/workflows/github-torch-hub.yml

@@ -30,8 +30,7 @@ jobs:
       run: |
         pip install --upgrade pip
         pip install torch
-        pip install numpy filelock protobuf requests tqdm regex sentencepiece sacremoses packaging
-        pip install tokenizers==0.9.0.rc2
+        pip install numpy filelock protobuf requests tqdm regex sentencepiece sacremoses tokenizers packaging
 
     - name: Torch hub list
       run: |

.github/workflows/self-push.yml

@@ -59,7 +59,6 @@ jobs:
         TF_FORCE_GPU_ALLOW_GROWTH: "true"
         # TF_GPU_MEMORY_LIMIT: 4096
         OMP_NUM_THREADS: 1
-        USE_CUDA: yes
       run: |
         source .env/bin/activate
         python -m pytest -n 2 --dist=loadfile -s ./tests/
@@ -110,7 +109,6 @@ jobs:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
           # TF_GPU_MEMORY_LIMIT: 4096
           OMP_NUM_THREADS: 1
-          USE_CUDA: yes
         run: |
           source .env/bin/activate
           python -m pytest -n 2 --dist=loadfile -s ./tests/

.github/workflows/self-scheduled.yml

@@ -57,21 +57,19 @@ jobs:
         TF_FORCE_GPU_ALLOW_GROWTH: "true"
         OMP_NUM_THREADS: 1
         RUN_SLOW: yes
-        USE_CUDA: yes
       run: |
         source .env/bin/activate
-        python -m pytest -n 1 --dist=loadfile -s ./tests/
+        python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0
 
     - name: Run examples tests on GPU
       env:
         TF_FORCE_GPU_ALLOW_GROWTH: "true"
         OMP_NUM_THREADS: 1
         RUN_SLOW: yes
-        USE_CUDA: yes
       run: |
         source .env/bin/activate
         pip install -r examples/requirements.txt
-        python -m pytest -n 1 --dist=loadfile -s examples
+        python -m pytest -n 1 --dist=loadfile -s examples --durations=0
 
   run_all_tests_torch_and_tf_multiple_gpu:
     runs-on: [self-hosted, multi-gpu]
@@ -120,18 +118,16 @@ jobs:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
           OMP_NUM_THREADS: 1
           RUN_SLOW: yes
-          USE_CUDA: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/
+          python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0
 
       - name: Run examples tests on GPU
         env:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
           OMP_NUM_THREADS: 1
           RUN_SLOW: yes
-          USE_CUDA: yes
         run: |
           source .env/bin/activate
           pip install -r examples/requirements.txt
-          python -m pytest -n 1 --dist=loadfile -s examples
+          python -m pytest -n 1 --dist=loadfile -s examples --durations=0

.gitignore

@@ -9,9 +9,11 @@ __pycache__/
 *.so
 
 # tests and logs
-tests/fixtures
+tests/fixtures/*
+!tests/fixtures/sample_text_no_unicode.txt
 logs/
 lightning_logs/
+lang_code_data/
 
 # Distribution / packaging
 .Python
@@ -155,3 +157,6 @@ debug.env
 
 #ctags
 tags
+
+# pre-commit
+.pre-commit*

README.md

@@ -81,7 +81,7 @@ This is another example of pipeline used for that can extract question answers f
 
 On top of the answer, the pretrained model used here returned its confidence score, along with the start position and its end position in the tokenized sentence. You can learn more about the tasks supported by the `pipeline` API in [this tutorial](https://huggingface.co/transformers/task_summary.html).
 
-To download and use any of the pretrained models on your given task, you just need to use those three lines of codes (PyTorch verison):
+To download and use any of the pretrained models on your given task, you just need to use those three lines of codes (PyTorch version):
 ```python
 >>> from transformers import AutoTokenizer, AutoModel
 
@@ -161,28 +161,28 @@ If you'd like to play with the examples, you must [install the library from sour
 1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.
 1. **[BART](https://huggingface.co/transformers/model_doc/bart.html)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://arxiv.org/pdf/1910.13461.pdf) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer.
 1. **[BERT](https://huggingface.co/transformers/model_doc/bert.html)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.
-1. **[BERT For Sequence Generation](https://tfhub.dev/s?module-type=text-generation&subtype=module,placeholder)** (from Google) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn.
+1. **[BERT For Sequence Generation](https://huggingface.co/transformers/model_doc/bertgeneration.html)** (from Google) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn.
+1. **[Blenderbot](https://huggingface.co/transformers/model_doc/blenderbot.html)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
 1. **[CamemBERT](https://huggingface.co/transformers/model_doc/camembert.html)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
 1. **[CTRL](https://huggingface.co/transformers/model_doc/ctrl.html)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
 1. **[DeBERTa](https://huggingface.co/transformers/model_doc/deberta.html)** (from Microsoft Research) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
 1. **[DialoGPT](https://huggingface.co/transformers/model_doc/dialogpt.html)** (from Microsoft Research) released with the paper [DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation](https://arxiv.org/abs/1911.00536) by Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan.
 1. **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/master/examples/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT.
-1. **[DPR](https://github.com/facebookresearch/DPR)** (from Facebook) released with the paper [Dense Passage Retrieval
+1. **[DPR](https://huggingface.co/transformers/model_doc/dpr.html)** (from Facebook) released with the paper [Dense Passage Retrieval
 for Open-Domain Question Answering](https://arxiv.org/abs/2004.04906) by Vladimir Karpukhin, Barlas Oğuz, Sewon
 Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
 1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning.
 1. **[FlauBERT](https://huggingface.co/transformers/model_doc/flaubert.html)** (from CNRS) released with the paper [FlauBERT: Unsupervised Language Model Pre-training for French](https://arxiv.org/abs/1912.05372) by Hang Le, Loïc Vial, Jibril Frej, Vincent Segonne, Maximin Coavoux, Benjamin Lecouteux, Alexandre Allauzen, Benoît Crabbé, Laurent Besacier, Didier Schwab.
-1. **[Funnel Transformer](https://github.com/laiguokun/Funnel-Transformer)** (from CMU/Google Brain) released with the paper [Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing](https://arxiv.org/abs/2006.03236) by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
+1. **[Funnel Transformer](https://huggingface.co/transformers/model_doc/funnel.html)** (from CMU/Google Brain) released with the paper [Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing](https://arxiv.org/abs/2006.03236) by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
 1. **[GPT](https://huggingface.co/transformers/model_doc/gpt.html)** (from OpenAI) released with the paper [Improving Language Understanding by Generative Pre-Training](https://blog.openai.com/language-unsupervised/) by Alec Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever.
 1. **[GPT-2](https://huggingface.co/transformers/model_doc/gpt2.html)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**.
-1. **[LayoutLM](https://github.com/microsoft/unilm/tree/master/layoutlm)** (from Microsoft Research Asia) released with the paper [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou.
+1. **[LayoutLM](https://huggingface.co/transformers/model_doc/layoutlm.html)** (from Microsoft Research Asia) released with the paper [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou.
 1. **[Longformer](https://huggingface.co/transformers/model_doc/longformer.html)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan.
-1. **[LXMERT](https://github.com/airsplay/lxmert)** (from UNC Chapel Hill) released with the paper [LXMERT: Learning Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal.
+1. **[LXMERT](https://huggingface.co/transformers/model_doc/lxmert.html)** (from UNC Chapel Hill) released with the paper [LXMERT: Learning Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal.
 1. **[MarianMT](https://huggingface.co/transformers/model_doc/marian.html)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team.
-1. **[MBart](https://github.com/pytorch/fairseq/tree/master/examples/mbart)** (from Facebook) released with the paper  [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-1. **[MMBT](https://github.com/facebookresearch/mmbt/)** (from Facebook), released together with the paper a [Supervised Multimodal Bitransformers for Classifying Images and Text](https://arxiv.org/pdf/1909.02950.pdf) by Douwe Kiela, Suvrat Bhooshan, Hamed Firooz, Davide Testuggine.
-1. **[Pegasus](https://github.com/google-research/pegasus)** (from Google) released with the paper [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization](https://arxiv.org/abs/1912.08777)> by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
-1. **[ProphetNet](https://github.com/microsoft/ProphetNet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
+1. **[MBart](https://huggingface.co/transformers/model_doc/mbart.html)** (from Facebook) released with the paper  [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
+1. **[Pegasus](https://huggingface.co/transformers/model_doc/pegasus.html)** (from Google) released with the paper [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization](https://arxiv.org/abs/1912.08777)> by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
+1. **[ProphetNet](https://huggingface.co/transformers/model_doc/prophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
 1. **[Reformer](https://huggingface.co/transformers/model_doc/reformer.html)** (from Google Research) released with the paper [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
 1. **[RoBERTa](https://huggingface.co/transformers/model_doc/roberta.html)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
 ultilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT.
@@ -190,7 +190,7 @@ ultilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/
 1. **[T5](https://huggingface.co/transformers/model_doc/t5.html)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
 1. **[Transformer-XL](https://huggingface.co/transformers/model_doc/transformerxl.html)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
 1. **[XLM](https://huggingface.co/transformers/model_doc/xlm.html)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau.
-1. **[XLMProphetNet](https://github.com/microsoft/ProphetNet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
+1. **[XLM-ProphetNet](https://huggingface.co/transformers/model_doc/xlmprophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
 1. **[XLM-RoBERTa](https://huggingface.co/transformers/model_doc/xlmroberta.html)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov.
 1. **[XLNet](https://huggingface.co/transformers/model_doc/xlnet.html)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
 1. **[Other community models](https://huggingface.co/models)**, contributed by the [community](https://huggingface.co/users).

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'3.3.1'
+release = u'3.4.0'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/glossary.rst

@@ -218,6 +218,52 @@ positional embeddings.
 Absolute positional embeddings are selected in the range ``[0, config.max_position_embeddings - 1]``. Some models
 use other types of positional embeddings, such as sinusoidal position embeddings or relative position embeddings.
 
+.. _labels:
+
+Labels
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The labels are an optional argument which can be passed in order for the model to compute the loss itself. These labels
+should be the expected prediction of the model: it will use the standard loss in order to compute the loss between
+its predictions and the expected value (the label).
+
+These labels are different according to the model head, for example:
+
+- For sequence classification models (e.g., :class:`~transformers.BertForSequenceClassification`), the model expects
+  a tensor of dimension :obj:`(batch_size)` with each value of the batch corresponding to the expected label of the
+  entire sequence.
+- For token classification models (e.g., :class:`~transformers.BertForTokenClassification`), the model expects
+  a tensor of dimension :obj:`(batch_size, seq_length)` with each value corresponding to the expected label of each
+  individual token.
+- For masked language modeling (e.g., :class:`~transformers.BertForMaskedLM`), the model expects
+  a tensor of dimension :obj:`(batch_size, seq_length)` with each value corresponding to the expected label of each
+  individual token: the labels being the token ID for the masked token, and values to be ignored for the rest (usually
+  -100).
+- For sequence to sequence tasks,(e.g., :class:`~transformers.BartForConditionalGeneration`,
+  :class:`~transformers.MBartForConditionalGeneration`), the model expects a tensor of dimension
+  :obj:`(batch_size, tgt_seq_length)` with each value corresponding to the target sequences associated with each
+  input sequence. During training, both `BART` and `T5` will make the appropriate `decoder_input_ids` and decoder
+  attention masks internally. They usually do not need to be supplied. This does not apply to models leveraging the
+  Encoder-Decoder framework.
+  See the documentation of each model for more information on each specific model's labels.
+
+The base models (e.g., :class:`~transformers.BertModel`) do not accept labels, as these are the base transformer models,
+simply outputting features.
+
+.. _decoder-input-ids:
+
+Decoder input IDs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This input is specific to encoder-decoder models, and contains the input IDs that will be fed to the decoder.
+These inputs should be used for sequence to sequence tasks, such as translation or summarization, and are usually
+built in a way specific to each model.
+
+Most encoder-decoder models (BART, T5) create their :obj:`decoder_input_ids` on their own from the :obj:`labels`.
+In such models, passing the :obj:`labels` is the preferred way to handle training.
+
+Please check each model's docs to see how they handle these input IDs for sequence to sequence training.
+
 .. _feed-forward-chunking:
 
 Feed Forward Chunking

docs/source/index.rst

@@ -57,118 +57,108 @@ conversion utilities for the following models:
 ..
     This list is updated automatically from the README with `make fix-copies`. Do not update manually!
 
-1. `ALBERT <https://huggingface.co/transformers/model_doc/albert.html>`__ (from Google Research and the Toyota
-   Technological Institute at Chicago) released with the paper `ALBERT: A Lite BERT for Self-supervised Learning of
-   Language Representations <https://arxiv.org/abs/1909.11942>`__, by Zhenzhong Lan, Mingda Chen, Sebastian Goodman,
-   Kevin Gimpel, Piyush Sharma, Radu Soricut.
-2. `BART <https://huggingface.co/transformers/model_doc/bart.html>`__ (from Facebook) released with the paper `BART:
-   Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension
+1. :doc:`ALBERT <model_doc/albert>` (from Google Research and the Toyota Technological Institute at Chicago) released
+   with the paper `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations
+   <https://arxiv.org/abs/1909.11942>`__, by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush
+   Sharma, Radu Soricut.
+2. :doc:`BART <model_doc/bart>` (from Facebook) released with the paper `BART: Denoising Sequence-to-Sequence
+   Pre-training for Natural Language Generation, Translation, and Comprehension
    <https://arxiv.org/pdf/1910.13461.pdf>`__ by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman
    Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer.
-3. `BERT <https://huggingface.co/transformers/model_doc/bert.html>`__ (from Google) released with the paper `BERT:
-   Pre-training of Deep Bidirectional Transformers for Language Understanding <https://arxiv.org/abs/1810.04805>`__ by
-   Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.
-4. `BERT For Sequence Generation <https://tfhub.dev/s?module-type=text-generation&subtype=module,placeholder>`__ (from
-   Google) released with the paper `Leveraging Pre-trained Checkpoints for Sequence Generation Tasks
-   <https://arxiv.org/abs/1907.12461>`__ by Sascha Rothe, Shashi Narayan, Aliaksei Severyn.
-5. `CamemBERT <https://huggingface.co/transformers/model_doc/camembert.html>`__ (from Inria/Facebook/Sorbonne) released
-   with the paper `CamemBERT: a Tasty French Language Model <https://arxiv.org/abs/1911.03894>`__ by Louis Martin*,
-   Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé
-   Seddah and Benoît Sagot.
-6. `CTRL <https://huggingface.co/transformers/model_doc/ctrl.html>`__ (from Salesforce) released with the paper `CTRL:
-   A Conditional Transformer Language Model for Controllable Generation <https://arxiv.org/abs/1909.05858>`__ by Nitish
-   Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
-7. `DeBERTa <https://huggingface.co/transformers/model_doc/deberta.html>`__ (from Microsoft Research) released with the
-   paper `DeBERTa: Decoding-enhanced BERT with Disentangled Attention <https://arxiv.org/abs/2006.03654>`__ by
-   Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
-8. `DialoGPT <https://huggingface.co/transformers/model_doc/dialogpt.html>`__ (from Microsoft Research) released with
-   the paper `DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation
-   <https://arxiv.org/abs/1911.00536>`__ by Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang
-   Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan.
-9. `DistilBERT <https://huggingface.co/transformers/model_doc/distilbert.html>`__ (from HuggingFace), released together
-   with the paper `DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter
-   <https://arxiv.org/abs/1910.01108>`__ by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been
-   applied to compress GPT2 into `DistilGPT2
-   <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__, RoBERTa into `DistilRoBERTa
-   <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__, Multilingual BERT into
-   `DistilmBERT <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German version
-   of DistilBERT.
-10. `DPR <https://github.com/facebookresearch/DPR>`__ (from Facebook) released with the paper `Dense Passage Retrieval
-    for Open-Domain Question Answering <https://arxiv.org/abs/2004.04906>`__ by Vladimir Karpukhin, Barlas Oğuz, Sewon
-    Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
-11. `ELECTRA <https://huggingface.co/transformers/model_doc/electra.html>`__ (from Google Research/Stanford University)
-    released with the paper `ELECTRA: Pre-training text encoders as discriminators rather than generators
-    <https://arxiv.org/abs/2003.10555>`__ by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning.
-12. `FlauBERT <https://huggingface.co/transformers/model_doc/flaubert.html>`__ (from CNRS) released with the paper
-    `FlauBERT: Unsupervised Language Model Pre-training for French <https://arxiv.org/abs/1912.05372>`__ by Hang Le,
-    Loïc Vial, Jibril Frej, Vincent Segonne, Maximin Coavoux, Benjamin Lecouteux, Alexandre Allauzen, Benoît Crabbé,
-    Laurent Besacier, Didier Schwab.
-13. `Funnel Transformer <https://github.com/laiguokun/Funnel-Transformer>`__ (from CMU/Google Brain) released with the
-    paper `Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing
-    <https://arxiv.org/abs/2006.03236>`__ by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
-14. `GPT <https://huggingface.co/transformers/model_doc/gpt.html>`__ (from OpenAI) released with the paper `Improving
-    Language Understanding by Generative Pre-Training <https://blog.openai.com/language-unsupervised/>`__ by Alec
-    Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever.
-15. `GPT-2 <https://huggingface.co/transformers/model_doc/gpt2.html>`__ (from OpenAI) released with the paper `Language
-    Models are Unsupervised Multitask Learners <https://blog.openai.com/better-language-models/>`__ by Alec Radford*,
-    Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**.
-16. `LayoutLM <https://github.com/microsoft/unilm/tree/master/layoutlm>`__ (from Microsoft Research Asia) released with
-    the paper `LayoutLM: Pre-training of Text and Layout for Document Image Understanding
-    <https://arxiv.org/abs/1912.13318>`__ by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou.
-17. `Longformer <https://huggingface.co/transformers/model_doc/longformer.html>`__ (from AllenAI) released with the
-    paper `Longformer: The Long-Document Transformer <https://arxiv.org/abs/2004.05150>`__ by Iz Beltagy, Matthew E.
-    Peters, Arman Cohan.
-18. `LXMERT <https://github.com/airsplay/lxmert>`__ (from UNC Chapel Hill) released with the paper `LXMERT: Learning
-    Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering
-    <https://arxiv.org/abs/1908.07490>`__ by Hao Tan and Mohit Bansal.
-19. `MarianMT <https://huggingface.co/transformers/model_doc/marian.html>`__ Machine translation models trained using
-    `OPUS <http://opus.nlpl.eu/>`__ data by Jörg Tiedemann. The `Marian Framework <https://marian-nmt.github.io/>`__ is
-    being developed by the Microsoft Translator Team.
-20. `MBart <https://github.com/pytorch/fairseq/tree/master/examples/mbart>`__ (from Facebook) released with the paper 
-    `Multilingual Denoising Pre-training for Neural Machine Translation <https://arxiv.org/abs/2001.08210>`__ by Yinhan
-    Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-21. `MMBT <https://github.com/facebookresearch/mmbt/>`__ (from Facebook), released together with the paper a
-    `Supervised Multimodal Bitransformers for Classifying Images and Text <https://arxiv.org/pdf/1909.02950.pdf>`__ by
-    Douwe Kiela, Suvrat Bhooshan, Hamed Firooz, Davide Testuggine.
-22. `Pegasus <https://github.com/google-research/pegasus>`__ (from Google) released with the paper `PEGASUS:
-    Pre-training with Extracted Gap-sentences for Abstractive Summarization <https://arxiv.org/abs/1912.08777>`__> by
-    Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
-23. `ProphetNet <https://github.com/microsoft/ProphetNet>`__ (from Microsoft Research) released with the paper
-    `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__
-    by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-24. `Reformer <https://huggingface.co/transformers/model_doc/reformer.html>`__ (from Google Research) released with the
-    paper `Reformer: The Efficient Transformer <https://arxiv.org/abs/2001.04451>`__ by Nikita Kitaev, Łukasz Kaiser,
-    Anselm Levskaya.
-25. `RoBERTa <https://huggingface.co/transformers/model_doc/roberta.html>`__ (from Facebook), released together with
-    the paper a `Robustly Optimized BERT Pretraining Approach <https://arxiv.org/abs/1907.11692>`__ by Yinhan Liu, Myle
-    Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
-    ultilingual BERT into `DistilmBERT
+3. :doc:`BERT <model_doc/bert>` (from Google) released with the paper `BERT: Pre-training of Deep Bidirectional
+   Transformers for Language Understanding <https://arxiv.org/abs/1810.04805>`__ by Jacob Devlin, Ming-Wei Chang,
+   Kenton Lee and Kristina Toutanova.
+4. :doc:`BERT For Sequence Generation <model_doc/bertgeneration>` (from Google) released with the paper `Leveraging
+   Pre-trained Checkpoints for Sequence Generation Tasks <https://arxiv.org/abs/1907.12461>`__ by Sascha Rothe, Shashi
+   Narayan, Aliaksei Severyn.
+5. :doc:`Blenderbot <model_doc/blenderbot>` (from Facebook) released with the paper `Recipes for building an
+   open-domain chatbot <https://arxiv.org/abs/2004.13637>`__ by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary
+   Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
+6. :doc:`CamemBERT <model_doc/camembert>` (from Inria/Facebook/Sorbonne) released with the paper `CamemBERT: a Tasty
+   French Language Model <https://arxiv.org/abs/1911.03894>`__ by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz
+   Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
+7. :doc:`CTRL <model_doc/ctrl>` (from Salesforce) released with the paper `CTRL: A Conditional Transformer Language
+   Model for Controllable Generation <https://arxiv.org/abs/1909.05858>`__ by Nitish Shirish Keskar*, Bryan McCann*,
+   Lav R. Varshney, Caiming Xiong and Richard Socher.
+8. :doc:`DeBERTa <model_doc/deberta>` (from Microsoft Research) released with the paper `DeBERTa: Decoding-enhanced
+   BERT with Disentangled Attention <https://arxiv.org/abs/2006.03654>`__ by Pengcheng He, Xiaodong Liu, Jianfeng Gao,
+   Weizhu Chen.
+9. :doc:`DialoGPT <model_doc/dialogpt>` (from Microsoft Research) released with the paper `DialoGPT: Large-Scale
+   Generative Pre-training for Conversational Response Generation <https://arxiv.org/abs/1911.00536>`__ by Yizhe Zhang,
+   Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan.
+10. :doc:`DistilBERT <model_doc/distilbert>` (from HuggingFace), released together with the paper `DistilBERT, a
+    distilled version of BERT: smaller, faster, cheaper and lighter <https://arxiv.org/abs/1910.01108>`__ by Victor
+    Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into `DistilGPT2
+    <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__, RoBERTa into `DistilRoBERTa
+    <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__, Multilingual BERT into
+    `DistilmBERT <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German
+    version of DistilBERT.
+11. :doc:`DPR <model_doc/dpr>` (from Facebook) released with the paper `Dense Passage Retrieval for Open-Domain
+    Question Answering <https://arxiv.org/abs/2004.04906>`__ by Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick
+    Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
+12. :doc:`ELECTRA <model_doc/electra>` (from Google Research/Stanford University) released with the paper `ELECTRA:
+    Pre-training text encoders as discriminators rather than generators <https://arxiv.org/abs/2003.10555>`__ by Kevin
+    Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning.
+13. :doc:`FlauBERT <model_doc/flaubert>` (from CNRS) released with the paper `FlauBERT: Unsupervised Language Model
+    Pre-training for French <https://arxiv.org/abs/1912.05372>`__ by Hang Le, Loïc Vial, Jibril Frej, Vincent Segonne,
+    Maximin Coavoux, Benjamin Lecouteux, Alexandre Allauzen, Benoît Crabbé, Laurent Besacier, Didier Schwab.
+14. :doc:`Funnel Transformer <model_doc/funnel>` (from CMU/Google Brain) released with the paper `Funnel-Transformer:
+    Filtering out Sequential Redundancy for Efficient Language Processing <https://arxiv.org/abs/2006.03236>`__ by
+    Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
+15. :doc:`GPT <model_doc/gpt>` (from OpenAI) released with the paper `Improving Language Understanding by Generative
+    Pre-Training <https://blog.openai.com/language-unsupervised/>`__ by Alec Radford, Karthik Narasimhan, Tim Salimans
+    and Ilya Sutskever.
+16. :doc:`GPT-2 <model_doc/gpt2>` (from OpenAI) released with the paper `Language Models are Unsupervised Multitask
+    Learners <https://blog.openai.com/better-language-models/>`__ by Alec Radford*, Jeffrey Wu*, Rewon Child, David
+    Luan, Dario Amodei** and Ilya Sutskever**.
+17. :doc:`LayoutLM <model_doc/layoutlm>` (from Microsoft Research Asia) released with the paper `LayoutLM: Pre-training
+    of Text and Layout for Document Image Understanding <https://arxiv.org/abs/1912.13318>`__ by Yiheng Xu, Minghao Li,
+    Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou.
+18. :doc:`Longformer <model_doc/longformer>` (from AllenAI) released with the paper `Longformer: The Long-Document
+    Transformer <https://arxiv.org/abs/2004.05150>`__ by Iz Beltagy, Matthew E. Peters, Arman Cohan.
+19. :doc:`LXMERT <model_doc/lxmert>` (from UNC Chapel Hill) released with the paper `LXMERT: Learning Cross-Modality
+    Encoder Representations from Transformers for Open-Domain Question Answering <https://arxiv.org/abs/1908.07490>`__
+    by Hao Tan and Mohit Bansal.
+20. :doc:`MarianMT <model_doc/marian>` Machine translation models trained using `OPUS <http://opus.nlpl.eu/>`__ data by
+    Jörg Tiedemann. The `Marian Framework <https://marian-nmt.github.io/>`__ is being developed by the Microsoft
+    Translator Team.
+21. :doc:`MBart <model_doc/mbart>` (from Facebook) released with the paper  `Multilingual Denoising Pre-training for
+    Neural Machine Translation <https://arxiv.org/abs/2001.08210>`__ by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li,
+    Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
+22. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
+    Gap-sentences for Abstractive Summarization <https://arxiv.org/abs/1912.08777>`__> by Jingqing Zhang, Yao Zhao,
+    Mohammad Saleh and Peter J. Liu.
+23. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
+    Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi,
+    Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
+24. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
+    Transformer <https://arxiv.org/abs/2001.04451>`__ by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
+25. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
+    Pretraining Approach <https://arxiv.org/abs/1907.11692>`__ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar
+    Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. ultilingual BERT into `DistilmBERT
     <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German version of
     DistilBERT.
-26. `SqueezeBert <https://huggingface.co/transformers/model_doc/squeezebert.html>`__ released with the paper
-    `SqueezeBERT: What can computer vision teach NLP about efficient neural networks?
-    <https://arxiv.org/abs/2006.11316>`__ by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
-27. `T5 <https://huggingface.co/transformers/model_doc/t5.html>`__ (from Google AI) released with the paper `Exploring
-    the Limits of Transfer Learning with a Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by
-    Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi
-    Zhou and Wei Li and Peter J. Liu.
-28. `Transformer-XL <https://huggingface.co/transformers/model_doc/transformerxl.html>`__ (from Google/CMU) released
-    with the paper `Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context
-    <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le,
-    Ruslan Salakhutdinov.
-29. `XLM <https://huggingface.co/transformers/model_doc/xlm.html>`__ (from Facebook) released together with the paper
-    `Cross-lingual Language Model Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis
-    Conneau.
-30. `XLMProphetNet <https://github.com/microsoft/ProphetNet>`__ (from Microsoft Research) released with the paper
-    `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__
-    by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-31. `XLM-RoBERTa <https://huggingface.co/transformers/model_doc/xlmroberta.html>`__ (from Facebook AI), released
-    together with the paper `Unsupervised Cross-lingual Representation Learning at Scale
-    <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary,
-    Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov.
-32. `XLNet <https://huggingface.co/transformers/model_doc/xlnet.html>`__ (from Google/CMU) released with the paper
-    `​XLNet: Generalized Autoregressive Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by
-    Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
+26. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
+    about efficient neural networks? <https://arxiv.org/abs/2006.11316>`__ by Forrest N. Iandola, Albert E. Shaw, Ravi
+    Krishna, and Kurt W. Keutzer.
+27. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
+    Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by Colin Raffel and Noam Shazeer and Adam
+    Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
+28. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
+    Attentive Language Models Beyond a Fixed-Length Context <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*,
+    Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
+29. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
+    Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis Conneau.
+30. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
+    Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan,
+    Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
+31. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
+    Cross-lingual Representation Learning at Scale <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay
+    Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke
+    Zettlemoyer and Veselin Stoyanov.
+32. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `​XLNet: Generalized Autoregressive
+    Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by Zhilin Yang*, Zihang Dai*, Yiming
+    Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
 33. `Other community models <https://huggingface.co/models>`__, contributed by the `community
     <https://huggingface.co/users>`__.
 
@@ -260,6 +250,7 @@ conversion utilities for the following models:
     model_doc/gpt
     model_doc/gpt2
     model_doc/pegasus
+    model_doc/prophetnet
     model_doc/rag
     model_doc/reformer
     model_doc/retribert
@@ -268,6 +259,7 @@ conversion utilities for the following models:
     model_doc/t5
     model_doc/transformerxl
     model_doc/xlm
+    model_doc/xlmprophetnet
     model_doc/xlmroberta
     model_doc/xlnet
 

docs/source/internal/trainer_utils.rst

@@ -19,3 +19,9 @@ Callbacks internals
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.trainer_callback.CallbackHandler
+
+Distributed Evaluation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.trainer_pt_utils.DistributedTensorGatherer
+    :members:

docs/source/main_classes/callback.rst

@@ -12,7 +12,7 @@ subclass :class:`~transformers.Trainer` and override the methods you need (see :
 
 By default a :class:`~transformers.Trainer` will use the following callbacks:
 
-- :class:`~transformers.DefaultFlowCallback` which handles the default beahvior for logging, saving and evaluation.
+- :class:`~transformers.DefaultFlowCallback` which handles the default behavior for logging, saving and evaluation.
 - :class:`~transformers.PrinterCallback` or :class:`~transformers.ProrgressCallback` to display progress and print the
   logs (the first one is used if you deactivate tqdm through the :class:`~transformers.TrainingArguments`, otherwise
   it's the second one).

docs/source/main_classes/trainer.rst

@@ -15,7 +15,7 @@ Both :class:`~transformers.Trainer` and :class:`~transformers.TFTrainer` contain
 previous features. To inject custom behavior you can subclass them and override the following methods:
 
 - **get_train_dataloader**/**get_train_tfdataset** -- Creates the training DataLoader (PyTorch) or TF Dataset.
-- **get_eval_dataloader**/**get_eval_tfdataset** -- Creates the evaulation DataLoader (PyTorch) or TF Dataset.
+- **get_eval_dataloader**/**get_eval_tfdataset** -- Creates the evaluation DataLoader (PyTorch) or TF Dataset.
 - **get_test_dataloader**/**get_test_tfdataset** -- Creates the test DataLoader (PyTorch) or TF Dataset.
 - **log** -- Logs information on the various objects watching training.
 - **create_optimizer_and_scheduler** -- Setups the optimizer and learning rate scheduler if they were not passed at

docs/source/model_doc/blenderbot.rst

@@ -1,5 +1,5 @@
 Blenderbot
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+-----------------------------------------------------------------------------------------------------------------------
 **DISCLAIMER:** If you see something strange,
 file a `Github Issue <https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ .
 

docs/source/model_doc/encoderdecoder.rst

@@ -27,4 +27,4 @@ EncoderDecoderModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.EncoderDecoderModel
-    :members: forward
+    :members: forward, from_encoder_decoder_pretrained

docs/source/model_doc/gpt.rst

@@ -104,6 +104,13 @@ OpenAIGPTDoubleHeadsModel
     :members: forward
 
 
+OpenAIGPTForSequenceClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.OpenAIGPTForSequenceClassification
+    :members: forward
+
+
 TFOpenAIGPTModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/prophetnet.rst

@@ -0,0 +1,83 @@
+ProphetNet
+-----------------------------------------------------------------------------------------------------------------------
+
+**DISCLAIMER:** If you see something strange, file a `Github Issue
+<https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ and assign
+@patrickvonplaten
+
+Overview
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ProphetNet model was proposed in `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou on 13 Jan, 2020.
+
+ProphetNet is an encoder-decoder model and can predict n-future tokens for "ngram" language modeling instead of just the next token.
+
+The abstract from the paper is the following:
+
+*In this paper, we present a new sequence-to-sequence pre-training model called ProphetNet, which introduces a novel self-supervised objective named future n-gram prediction and the proposed n-stream self-attention mechanism. Instead of the optimization of one-step ahead prediction in traditional sequence-to-sequence model, the ProphetNet is optimized by n-step ahead prediction which predicts the next n tokens simultaneously based on previous context tokens at each time step. The future n-gram prediction explicitly encourages the model to plan for the future tokens and prevent overfitting on strong local correlations. We pre-train ProphetNet using a base scale dataset (16GB) and a large scale dataset (160GB) respectively. Then we conduct experiments on CNN/DailyMail, Gigaword, and SQuAD 1.1 benchmarks for abstractive summarization and question generation tasks. Experimental results show that ProphetNet achieves new state-of-the-art results on all these datasets compared to the models using the same scale pre-training corpus.*
+
+The Authors' code can be found `here <https://github.com/microsoft/ProphetNet>`__.
+
+
+ProphetNetConfig
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ProphetNetConfig
+    :members:
+
+
+ProphetNetTokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ProphetNetTokenizer
+    :members:
+
+
+ProphetNet specific outputs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqLMOutput
+    :members:
+
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqModelOutput
+    :members:
+
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderModelOutput
+    :members:
+
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderLMOutput
+    :members:
+
+ProphetNetModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ProphetNetModel
+    :members: forward
+
+
+ProphetNetEncoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ProphetNetEncoder
+    :members: forward
+
+
+ProphetNetDecoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ProphetNetDecoder
+    :members: forward
+
+
+ProphetNetForConditionalGeneration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ProphetNetForConditionalGeneration
+    :members: forward
+
+
+ProphetNetForCausalLM
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ProphetNetForCausalLM
+    :members: forward

docs/source/model_doc/rag.rst

@@ -62,8 +62,7 @@ Rag specific outputs
 .. autoclass:: transformers.modeling_rag.RetrievAugLMOutput
     :members:
 
-
-RAGRetriever
+RagRetriever
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.RagRetriever

docs/source/model_doc/t5.rst

@@ -62,10 +62,10 @@ token. T5 can be trained / fine-tuned both in a supervised and unsupervised fash
 
 .. code-block::
 
-  input_ids = tokenizer.encode('The <extra_id_0> walks in <extra_id_1> park', return_tensors='pt')
-  labels = tokenizer.encode('<extra_id_0> cute dog <extra_id_1> the <extra_id_2> </s>', return_tensors='pt')
+  input_ids = tokenizer('The <extra_id_0> walks in <extra_id_1> park', return_tensors='pt').input_ids
+  labels = tokenizer('<extra_id_0> cute dog <extra_id_1> the <extra_id_2>', return_tensors='pt').input_ids
   # the forward function automatically creates the correct decoder_input_ids
-  model(input_ids=input_ids, labels=labels)
+  loss = model(input_ids=input_ids, labels=labels, return_dict=True).loss
 
 - Supervised training
 
@@ -75,10 +75,10 @@ token. T5 can be trained / fine-tuned both in a supervised and unsupervised fash
   
 .. code-block::
 
-  input_ids = tokenizer.encode('translate English to German: The house is wonderful. </s>', return_tensors='pt')
-  labels = tokenizer.encode('Das Haus ist wunderbar. </s>', return_tensors='pt')
+  input_ids = tokenizer('translate English to German: The house is wonderful.', return_tensors='pt').input_ids
+  labels = tokenizer('Das Haus ist wunderbar.', return_tensors='pt').input_ids
   # the forward function automatically creates the correct decoder_input_ids
-  model(input_ids=input_ids, labels=labels)
+  loss = model(input_ids=input_ids, labels=labels, return_dict=True).loss
 
 
 T5Config

docs/source/model_doc/xlmprophetnet.rst

@@ -0,0 +1,63 @@
+XLM-ProphetNet
+-----------------------------------------------------------------------------------------------------------------------
+
+**DISCLAIMER:** If you see something strange, file a `Github Issue
+<https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ and assign
+@patrickvonplaten
+
+
+Overview
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The XLM-ProphetNet model was proposed in `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou on 13 Jan, 2020.
+
+XLM-ProphetNet is an encoder-decoder model and can predict n-future tokens for "ngram" language modeling instead of just the next token. Its architecture is identical to ProhpetNet, but the model was trained on the multi-lingual "wiki100" Wikipedia dump.
+
+The abstract from the paper is the following:
+
+*In this paper, we present a new sequence-to-sequence pre-training model called ProphetNet, which introduces a novel self-supervised objective named future n-gram prediction and the proposed n-stream self-attention mechanism. Instead of the optimization of one-step ahead prediction in traditional sequence-to-sequence model, the ProphetNet is optimized by n-step ahead prediction which predicts the next n tokens simultaneously based on previous context tokens at each time step. The future n-gram prediction explicitly encourages the model to plan for the future tokens and prevent overfitting on strong local correlations. We pre-train ProphetNet using a base scale dataset (16GB) and a large scale dataset (160GB) respectively. Then we conduct experiments on CNN/DailyMail, Gigaword, and SQuAD 1.1 benchmarks for abstractive summarization and question generation tasks. Experimental results show that ProphetNet achieves new state-of-the-art results on all these datasets compared to the models using the same scale pre-training corpus.*
+
+The Authors' code can be found `here <https://github.com/microsoft/ProphetNet>`__.
+
+XLMProphetNetConfig
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLMProphetNetConfig
+    :members:
+
+
+XLMProphetNetTokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLMProphetNetTokenizer
+    :members:
+
+
+XLMProphetNetModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLMProphetNetModel
+
+
+XLMProphetNetEncoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLMProphetNetEncoder
+
+
+XLMProphetNetDecoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLMProphetNetDecoder
+
+
+XLMProphetNetForConditionalGeneration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLMProphetNetForConditionalGeneration
+
+
+XLMProphetNetForCausalLM
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.XLMProphetNetForCausalLM

docs/source/model_summary.rst

@@ -612,6 +612,43 @@ The `mbart-large-cc25 <https://huggingface.co/facebook/mbart-large-cc25>`_ check
 
 .. _multimodal-models:
 
+ProphetNet
+-----------------------------------------------------------------------------------------------------------------------
+
+.. raw:: html
+
+   <a href="https://huggingface.co/models?filter=prophetnet">
+       <img alt="Models" src="https://img.shields.io/badge/All_model_pages-prophetnet-blueviolet">
+   </a>
+   <a href="model_doc/prophetnet.html">
+       <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-prophetnet-blueviolet">
+   </a>
+
+`ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.
+
+ProphetNet introduces a novel *sequence-to-sequence* pre-training objective, called *future n-gram prediction*. In future n-gram prediction, the model predicts the next n tokens simultaneously based on previous context tokens at each time step instead instead of just the single next token. The future n-gram prediction explicitly encourages the model to plan for the future tokens and prevent overfitting on strong local correlations.
+The model architecture is based on the original Transformer, but replaces the "standard" self-attention mechanism in the decoder by a a main self-attention mechanism and a self and n-stream (predict) self-attention mechanism.
+
+The library provides a pre-trained version of this model for conditional generation and a fine-tuned version for summarization.
+
+XLM-ProphetNet
+-----------------------------------------------------------------------------------------------------------------------
+
+.. raw:: html
+
+   <a href="https://huggingface.co/models?filter=xprophetnet">
+       <img alt="Models" src="https://img.shields.io/badge/All_model_pages-xprophetnet-blueviolet">
+   </a>
+   <a href="model_doc/xlmprophetnet.html">
+       <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-xprophetnet-blueviolet">
+   </a>
+
+`ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.
+
+XLM-ProphetNet's model architecture and pre-training objective is same as ProphetNet, but XLM-ProphetNet was pre-trained on the cross-lingual dataset `XGLUE <https://arxiv.org/abs/2004.01401>`__.
+
+The library provides a pre-trained version of this model for multi-lingual conditional generation and fine-tuned versions for headline generation and question generation, respectively.
+
 Multimodal models
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/source/philosophy.rst

@@ -66,7 +66,7 @@ The library is built around three types of classes for each model:
 All these classes can be instantiated from pretrained instances and saved locally using two methods:
 
 - :obj:`from_pretrained()` lets you instantiate a model/configuration/tokenizer from a pretrained version either
-  provided by the library itself (the suported models are provided in the list :doc:`here <pretrained_models>`
+  provided by the library itself (the supported models are provided in the list :doc:`here <pretrained_models>`
   or stored locally (or on a server) by the user,
 - :obj:`save_pretrained()` lets you save a model/configuration/tokenizer locally so that it can be reloaded using
   :obj:`from_pretrained()`.

docs/source/pretrained_models.rst

@@ -294,10 +294,10 @@ For a list that includes community-uploaded models, refer to `https://huggingfac
 |                    | ``t5-11B``                                                 | | ~11B parameters with 24-layers, 1024-hidden-state, 65536 feed-forward hidden-state, 128-heads,                                      |
 |                    |                                                            | | Trained on English text: the Colossal Clean Crawled Corpus (C4)                                                                     |
 +--------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
-| XLM-RoBERTa        | ``xlm-roberta-base``                                       | | ~125M parameters with 12-layers, 768-hidden-state, 3072 feed-forward hidden-state, 8-heads,                                         |
+| XLM-RoBERTa        | ``xlm-roberta-base``                                       | | ~270M parameters with 12-layers, 768-hidden-state, 3072 feed-forward hidden-state, 8-heads,                                         |
 |                    |                                                            | | Trained on on 2.5 TB of newly created clean CommonCrawl data in 100 languages                                                       |
 |                    +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
-|                    | ``xlm-roberta-large``                                      | | ~355M parameters with 24-layers, 1027-hidden-state, 4096 feed-forward hidden-state, 16-heads,                                       |
+|                    | ``xlm-roberta-large``                                      | | ~550M parameters with 24-layers, 1024-hidden-state, 4096 feed-forward hidden-state, 16-heads,                                       |
 |                    |                                                            | | Trained on 2.5 TB of newly created clean CommonCrawl data in 100 languages                                                          |
 +--------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 | FlauBERT           | ``flaubert/flaubert_small_cased``                          | | 6-layer, 512-hidden, 8-heads, 54M parameters                                                                                        |

docs/source/task_summary.rst

@@ -758,8 +758,8 @@ Here is an example of using the pipelines to do summarization. It leverages a Ba
     ... If convicted, Barrientos faces up to four years in prison.  Her next court appearance is scheduled for May 18.
     ... """
 
-Because the summarization pipeline depends on the ``PretrainedModel.generate()`` method, we can override the default arguments
-of ``PretrainedModel.generate()`` directly in the pipeline for ``max_length`` and ``min_length`` as shown below.
+Because the summarization pipeline depends on the ``PreTrainedModel.generate()`` method, we can override the default arguments
+of ``PreTrainedModel.generate()`` directly in the pipeline for ``max_length`` and ``min_length`` as shown below.
 This outputs the following summary:
 
 .. code-block::
@@ -772,7 +772,7 @@ Here is an example of doing summarization using a model and a tokenizer. The pro
 1. Instantiate a tokenizer and a model from the checkpoint name. Summarization is usually done using an encoder-decoder model, such as ``Bart`` or ``T5``.
 2. Define the article that should be summarized.
 3. Add the T5 specific prefix "summarize: ".
-4. Use the ``PretrainedModel.generate()`` method to generate the summary.
+4. Use the ``PreTrainedModel.generate()`` method to generate the summary.
 
 In this example we use Google`s T5 model. Even though it was pre-trained only on a multi-task mixed dataset (including CNN / Daily Mail), it yields very good results.
 
@@ -819,15 +819,15 @@ translation results.
     >>> print(translator("Hugging Face is a technology company based in New York and Paris", max_length=40))
     [{'translation_text': 'Hugging Face ist ein Technologieunternehmen mit Sitz in New York und Paris.'}]
 
-Because the translation pipeline depends on the ``PretrainedModel.generate()`` method, we can override the default arguments
-of ``PretrainedModel.generate()`` directly in the pipeline as is shown for ``max_length`` above.
+Because the translation pipeline depends on the ``PreTrainedModel.generate()`` method, we can override the default arguments
+of ``PreTrainedModel.generate()`` directly in the pipeline as is shown for ``max_length`` above.
 
 Here is an example of doing translation using a model and a tokenizer. The process is the following:
 
 1. Instantiate a tokenizer and a model from the checkpoint name. Summarization is usually done using an encoder-decoder model, such as ``Bart`` or ``T5``.
 2. Define the article that should be summarizaed.
 3. Add the T5 specific prefix "translate English to German: "
-4. Use the ``PretrainedModel.generate()`` method to perform the translation.
+4. Use the ``PreTrainedModel.generate()`` method to perform the translation.
 
 .. code-block::
 

docs/source/testing.rst

@@ -22,12 +22,12 @@ How transformers are tested
 
    * `self-hosted (push) <https://github.com/huggingface/transformers/blob/master/.github/workflows/self-push.yml>`__: runs fast tests on GPU only on commits on ``master``. It only runs if a commit on ``master`` has updated the code in one of the following folders: ``src``, ``tests``, ``.github`` (to prevent running on added model cards, notebooks, etc.)
      
-   * `self-hosted runner <https://github.com/huggingface/transformers/blob/master/.github/workflows/self-scheduled.yml>`__: runs slow tests on ``tests`` and ``examples``:
+   * `self-hosted runner <https://github.com/huggingface/transformers/blob/master/.github/workflows/self-scheduled.yml>`__: runs normal and slow tests on GPU in ``tests`` and ``examples``:
 
    .. code-block:: bash
 
-    RUN_SLOW=1 USE_CUDA=1 pytest tests/
-    RUN_SLOW=1 USE_CUDA=1 pytest examples/
+    RUN_SLOW=1 pytest tests/
+    RUN_SLOW=1 pytest examples/
 
    The results can be observed `here <https://github.com/huggingface/transformers/actions>`__.
 
@@ -393,36 +393,53 @@ On a GPU-enabled setup, to test in CPU-only mode add ``CUDA_VISIBLE_DEVICES=""``
                 
     CUDA_VISIBLE_DEVICES="" pytest tests/test_logging.py
 
-or if you have multiple gpus, you can tell which one to use in this test session, e.g. to use only the second gpu if you have gpus ``0`` and ``1``, you can run:
+or if you have multiple gpus, you can specify which one is to be used by ``pytest``. For example, to use only the second gpu if you have gpus ``0`` and ``1``, you can run:
 
 .. code-block:: bash
                 
     CUDA_VISIBLE_DEVICES="1" pytest tests/test_logging.py
 
 This is handy when you want to run different tasks on different GPUs.
-    
-And we have these decorators that require the condition described by the marker.
 
-``
-@require_torch
-@require_tf
-@require_multigpu
-@require_non_multigpu
-@require_torch_tpu
-@require_torch_and_cuda
-``
+Some tests must be run on CPU-only, others on either CPU or GPU or TPU, yet others on multiple-GPUs. The following skip decorators are used to set the requirements of tests CPU/GPU/TPU-wise:
+
+* ``require_torch`` - this test will run only under torch
+* ``require_torch_gpu`` - as ``require_torch`` plus requires at least 1 GPU
+* ``require_torch_multigpu`` - as ``require_torch`` plus requires at least 2 GPUs
+* ``require_torch_non_multigpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
+* ``require_torch_tpu`` - as ``require_torch`` plus requires at least 1 TPU
+
+For example, here is a test that must be run only when there are 2 or more GPUs available and pytorch is installed:
+
+.. code-block:: python
+
+    @require_torch_multigpu
+    def test_example_with_multigpu():
+
+If a test requires ``tensorflow`` use the ``require_tf`` decorator. For example:
+
+.. code-block:: python
+
+    @require_tf
+    def test_tf_thing_with_tensorflow():
+
+These decorators can be stacked. For example, if a test is slow and requires at least one GPU under pytorch, here is how to set it up:
+
+.. code-block:: python
+
+    @require_torch_gpu
+    @slow
+    def test_example_slow_on_gpu():
 
 Some decorators like ``@parametrized`` rewrite test names, therefore ``@require_*`` skip decorators have to be listed last for them to work correctly. Here is an example of the correct usage:
 
 .. code-block:: python
 
     @parameterized.expand(...)
-    @require_multigpu
+    @require_torch_multigpu
     def test_integration_foo():
-    
-There is no problem whatsoever with ``@pytest.mark.parametrize`` (but it only works with non-unittests) - can use it in any order.
 
-This section will be expanded soon once our work in progress on those decorators is finished.
+This order problem doesn't exist with ``@pytest.mark.parametrize``, you can put it first or last and it will still work. But it only works with non-unittests.
 
 Inside tests:
 

docs/source/training.rst

@@ -109,9 +109,9 @@ The following is equivalent to the previous example:
 .. code-block:: python
 
     from torch.nn import functional as F
-    labels = torch.tensor([1,0]).unsqueeze(0)
+    labels = torch.tensor([1,0])
     outputs = model(input_ids, attention_mask=attention_mask)
-    loss = F.cross_entropy(labels, outputs.logitd)
+    loss = F.cross_entropy(outputs.logits, labels)
     loss.backward()
     optimizer.step()
 

examples/conftest.py

@@ -2,6 +2,7 @@
 # by pytest before any tests are run
 
 import sys
+import warnings
 from os.path import abspath, dirname, join
 
 
@@ -9,3 +10,7 @@ from os.path import abspath, dirname, join
 # 'pip install -e .[dev]' when switching between checkouts and running tests.
 git_repo_path = abspath(join(dirname(dirname(__file__)), "src"))
 sys.path.insert(1, git_repo_path)
+
+# silence FutureWarning warnings in tests since often we can't act on them until
+# they become normal warnings - i.e. the tests still need to test the current functionality
+warnings.simplefilter(action="ignore", category=FutureWarning)

examples/language-modeling/run_language_modeling.py

@@ -24,8 +24,11 @@ import logging
 import math
 import os
 from dataclasses import dataclass, field
+from glob import glob
 from typing import Optional
 
+from torch.utils.data import ConcatDataset
+
 from transformers import (
     CONFIG_MAPPING,
     MODEL_WITH_LM_HEAD_MAPPING,
@@ -87,6 +90,13 @@ class DataTrainingArguments:
     train_data_file: Optional[str] = field(
         default=None, metadata={"help": "The input training data file (a text file)."}
     )
+    train_data_files: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The input training data files (multiple files in glob format). "
+            "Very often splitting large files to smaller files can prevent tokenizer going out of memory"
+        },
+    )
     eval_data_file: Optional[str] = field(
         default=None,
         metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
@@ -131,17 +141,24 @@ def get_dataset(
     evaluate: bool = False,
     cache_dir: Optional[str] = None,
 ):
-    file_path = args.eval_data_file if evaluate else args.train_data_file
-    if args.line_by_line:
-        return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
+    def _dataset(file_path):
+        if args.line_by_line:
+            return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
+        else:
+            return TextDataset(
+                tokenizer=tokenizer,
+                file_path=file_path,
+                block_size=args.block_size,
+                overwrite_cache=args.overwrite_cache,
+                cache_dir=cache_dir,
+            )
+
+    if evaluate:
+        return _dataset(args.eval_data_file)
+    elif args.train_data_files:
+        return ConcatDataset([_dataset(f) for f in glob(args.train_data_files)])
     else:
-        return TextDataset(
-            tokenizer=tokenizer,
-            file_path=file_path,
-            block_size=args.block_size,
-            overwrite_cache=args.overwrite_cache,
-            cache_dir=cache_dir,
-        )
+        return _dataset(args.train_data_file)
 
 
 def main():

examples/lightning_base.py

@@ -119,7 +119,7 @@ class BaseTransformer(pl.LightningModule):
     def get_lr_scheduler(self):
         get_schedule_func = arg_to_scheduler[self.hparams.lr_scheduler]
         scheduler = get_schedule_func(
-            self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=self.total_steps
+            self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=self.total_steps()
         )
         scheduler = {"scheduler": scheduler, "interval": "step", "frequency": 1}
         return scheduler
@@ -159,19 +159,20 @@ class BaseTransformer(pl.LightningModule):
     def test_epoch_end(self, outputs):
         return self.validation_end(outputs)
 
-    @property
     def total_steps(self) -> int:
         """The number of total training steps that will be run. Used for lr scheduler purposes."""
         num_devices = max(1, self.hparams.gpus)  # TODO: consider num_tpu_cores
         effective_batch_size = self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
-        dataset_size = len(self.train_loader.dataset)
-        return (dataset_size / effective_batch_size) * self.hparams.max_epochs
+        return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs
 
     def setup(self, mode):
-        if mode == "fit":
+        if mode == "test":
+            self.dataset_size = len(self.test_dataloader().dataset)
+        else:
             self.train_loader = self.get_dataloader("train", self.hparams.train_batch_size, shuffle=True)
+            self.dataset_size = len(self.train_loader.dataset)
 
-    def get_dataloader(self, type_path, batch_size, shuffle=False):
+    def get_dataloader(self, type_path: str, batch_size: int, shuffle: bool = False):
         raise NotImplementedError("You must implement this for your task")
 
     def train_dataloader(self):
@@ -290,7 +291,8 @@ class LoggingCallback(pl.Callback):
 
 
 def add_generic_args(parser, root_dir) -> None:
-    #  TODO(SS): allow all pl args? parser = pl.Trainer.add_argparse_args(parser)
+    #  To allow all pl args uncomment the following line
+    #  parser = pl.Trainer.add_argparse_args(parser)
     parser.add_argument(
         "--output_dir",
         default=None,

examples/rag/README.md

@@ -65,26 +65,41 @@ Does He Love You	Does He Love You	Red Sandy Spika dress of Reba McEntire	Greates
 We demonstrate how to evaluate retrieval against DPR evaluation data. You can download respective files from links listed [here](https://github.com/facebookresearch/DPR/blob/master/data/download_data.py#L39-L45).
 
 1. Download and unzip the gold data file. We use the `biencoder-nq-dev` from https://dl.fbaipublicfiles.com/dpr/data/retriever/biencoder-nq-dev.json.gz.
+    ```bash
+    wget https://dl.fbaipublicfiles.com/dpr/data/retriever/biencoder-nq-dev.json.gz && gzip -d biencoder-nq-dev.json.gz
+   ```
+
 2. Parse the unziped file using the `parse_dpr_relevance_data.py`
     ```bash
+    mkdir output # or wherever you want to save this
     python examples/rag/parse_dpr_relevance_data.py \
-        --src_path path/to/unziped/biencoder-nq-dev.json \
-        --evaluation_set path/to/output/biencoder-nq-dev.questions \
-        --gold_data_path path/to/output/biencoder-nq-dev.pages
+        --src_path biencoder-nq-dev.json \
+        --evaluation_set output/biencoder-nq-dev.questions \
+        --gold_data_path output/biencoder-nq-dev.pages
     ```
 3. Run evaluation:
-    ```bash
+    ```bash    
+    python examples/rag/eval_rag.py \
+        --model_name_or_path facebook/rag-sequence-nq \
+        --model_type rag_sequence \
+        --evaluation_set output/biencoder-nq-dev.questions \
+        --gold_data_path output/biencoder-nq-dev.pages \
+        --predictions_path output/retrieval_preds.tsv  \
+        --eval_mode retrieval \
+        --k 1
+    ```
+   ```bash
+   # EXPLANATION
     python examples/rag/eval_rag.py \
         --model_name_or_path facebook/rag-sequence-nq \ # model name or path of the model we're evaluating
         --model_type rag_sequence \ # RAG model type (rag_token or rag_sequence)
-        --evaluation_set path/to/output/biencoder-nq-dev.questions \ # an input dataset for evaluation
-        --gold_data_path path/to/output/biencoder-nq-dev.pages \ # a dataset containing ground truth answers for samples from the evaluation_set
-        --predictions_path path/to/retrieval_preds.tsv  \ # name of file where predictions will be stored
+        --evaluation_set output/biencoder-nq-dev.questions \ # an input dataset for evaluation
+        --gold_data_path poutput/biencoder-nq-dev.pages \ # a dataset containing ground truth answers for samples from the evaluation_set
+        --predictions_path output/retrieval_preds.tsv  \ # name of file where predictions will be stored
         --eval_mode retrieval \ # indicates whether we're performing retrieval evaluation or e2e evaluation
         --k 1 # parameter k for the precision@k metric
+   
     ```
-
-
 ## End-to-end evaluation
 
 We support two formats of the gold data file (controlled by the `gold_data_mode` parameter):
@@ -97,7 +112,9 @@ who is the owner of reading football club	['Xiu Li Dai', 'Dai Yongge', 'Dai Xiul
 Xiu Li Dai
 ```
 
-Predictions of the model for the samples from the `evaluation_set` will be saved under the path specified by the `predictions_path` parameter. If this path already exists, the script will use saved predictions to calculate metrics. Add `--recalculate` parameter to force the script to perform inference from scratch.
+Predictions of the model for the samples from the `evaluation_set` will be saved under the path specified by the `predictions_path` parameter. 
+If this path already exists, the script will use saved predictions to calculate metrics. 
+Add `--recalculate` parameter to force the script to perform inference from scratch.
 
 An example e2e evaluation run could look as follows:
 ```bash

examples/rag/__init__.py

@@ -0,0 +1,5 @@
+import os
+import sys
+
+
+sys.path.insert(1, os.path.dirname(os.path.realpath(__file__)))

examples/rag/distributed_retriever.py

@@ -27,13 +27,18 @@ class RagPyTorchDistributedRetriever(RagRetriever):
             It is used to decode the question and then use the generator_tokenizer.
         generator_tokenizer (:class:`~transformers.PretrainedTokenizer`):
             The tokenizer used for the generator part of the RagModel.
+        index (:class:`~transformers.retrieval_rag.Index`, optional, defaults to the one defined by the configuration):
+            If specified, use this index instead of the one built using the configuration
     """
 
     _init_retrieval = False
 
-    def __init__(self, config, question_encoder_tokenizer, generator_tokenizer):
+    def __init__(self, config, question_encoder_tokenizer, generator_tokenizer, index=None):
         super().__init__(
-            config, question_encoder_tokenizer=question_encoder_tokenizer, generator_tokenizer=generator_tokenizer
+            config,
+            question_encoder_tokenizer=question_encoder_tokenizer,
+            generator_tokenizer=generator_tokenizer,
+            index=index,
         )
 
         self.process_group = None

examples/rag/eval_rag.py

@@ -15,7 +15,7 @@ from transformers import logging as transformers_logging
 
 
 sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # isort:skip
-from examples.rag.utils import exact_match_score, f1_score  # noqa: E402 # isort:skip
+from utils import exact_match_score, f1_score  # noqa: E402 # isort:skip
 
 
 logger = logging.getLogger(__name__)
@@ -72,7 +72,7 @@ def get_precision_at_k(args, preds_path, gold_data_path):
     em = total = 0
     for hypo, reference in zip(hypos, references):
         hypo_provenance = set(hypo.split("\t")[:k])
-        ref_provenance = set(reference.split("\t")[1 : (k + 1)])
+        ref_provenance = set(reference.split("\t"))
         total += 1
         em += len(hypo_provenance & ref_provenance) / k
 

examples/rag/finetune.py

@@ -31,16 +31,13 @@ from transformers import (
 from transformers import logging as transformers_logging
 
 
-sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip
-
-from examples.lightning_base import BaseTransformer, add_generic_args, generic_train  # noqa: E402 # isort:skip
-from examples.rag.callbacks import (  # noqa: E402 # isort:skip
+from callbacks import (  # noqa: E402 # isort:skipq
     get_checkpoint_callback,
     get_early_stopping_callback,
     Seq2SeqLoggingCallback,
 )
-from examples.rag.distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
-from examples.rag.utils import (  # noqa: E402 # isort:skip
+from distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
+from utils import (  # noqa: E402 # isort:skip
     calculate_exact_match,
     flatten_list,
     get_git_info,
@@ -53,6 +50,11 @@ from examples.rag.utils import (  # noqa: E402 # isort:skip
     Seq2SeqDataset,
 )
 
+# need the parent dir module
+sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
+from lightning_base import BaseTransformer, add_generic_args, generic_train  # noqa
+
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
@@ -88,6 +90,11 @@ class GenerativeQAModule(BaseTransformer):
         config_class = RagConfig if self.is_rag_model else AutoConfig
         config = config_class.from_pretrained(hparams.model_name_or_path)
 
+        # set retriever parameters
+        config.index_name = args.index_name or config.index_name
+        config.passages_path = args.passages_path or config.passages_path
+        config.index_path = args.index_path or config.index_path
+
         # set extra_model_params for generator configs and load_model
         extra_model_params = ("encoder_layerdrop", "decoder_layerdrop", "attention_dropout", "dropout")
         if self.is_rag_model:
@@ -95,7 +102,7 @@ class GenerativeQAModule(BaseTransformer):
                 config.generator.prefix = args.prefix
             config.label_smoothing = hparams.label_smoothing
             hparams, config.generator = set_extra_model_params(extra_model_params, hparams, config.generator)
-            retriever = RagPyTorchDistributedRetriever.from_pretrained(hparams.model_name_or_path)
+            retriever = RagPyTorchDistributedRetriever.from_pretrained(hparams.model_name_or_path, config=config)
             model = self.model_class.from_pretrained(hparams.model_name_or_path, config=config, retriever=retriever)
             prefix = config.question_encoder.prefix
         else:
@@ -403,6 +410,28 @@ class GenerativeQAModule(BaseTransformer):
         )
         return parser
 
+    @staticmethod
+    def add_retriever_specific_args(parser):
+        parser.add_argument(
+            "--index_name",
+            type=str,
+            default=None,
+            help="Name of the index to use: 'hf' for a canonical dataset from the datasets library (default), 'custom' for a local index, or 'legacy' for the orignal one)",
+        )
+        parser.add_argument(
+            "--passages_path",
+            type=str,
+            default=None,
+            help="Path to the dataset of passages for custom index. More info about custom indexes in the RagRetriever documentation as well as in `examples/rag/use_own_knowledge_dataset.py`",
+        )
+        parser.add_argument(
+            "--index_path",
+            type=str,
+            default=None,
+            help="Path to the faiss index for custom index. More info about custom indexes in the RagRetriever documentation as well as in `examples/rag/use_own_knowledge_dataset.py`",
+        )
+        return parser
+
 
 def main(args, model=None) -> GenerativeQAModule:
     Path(args.output_dir).mkdir(exist_ok=True)
@@ -463,6 +492,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser = pl.Trainer.add_argparse_args(parser)
     parser = GenerativeQAModule.add_model_specific_args(parser, os.getcwd())
+    parser = GenerativeQAModule.add_retriever_specific_args(parser)
 
     args = parser.parse_args()
 

examples/rag/test_data/my_knowledge_dataset.csv

@@ -0,0 +1,2 @@
+Aaron	Aaron Aaron ( or ; "Ahärôn") is a prophet, high priest, and the brother of Moses in the Abrahamic religions. Knowledge of Aaron, along with his brother Moses, comes exclusively from religious texts, such as the Bible and Quran. The Hebrew Bible relates that, unlike Moses, who grew up in the Egyptian royal court, Aaron and his elder sister Miriam remained with their kinsmen in the eastern border-land of Egypt (Goshen). When Moses first confronted the Egyptian king about the Israelites, Aaron served as his brother's spokesman ("prophet") to the Pharaoh. Part of the Law (Torah) that Moses received from God at Sinai granted Aaron the priesthood for himself and his male descendants, and he became the first High Priest of the Israelites. Aaron died before the Israelites crossed the North Jordan river and he was buried on Mount Hor (Numbers 33:39; Deuteronomy 10:6 says he died and was buried at Moserah). Aaron is also mentioned in the New Testament of the Bible. According to the Book of Exodus, Aaron first functioned as Moses' assistant. Because Moses complained that he could not speak well, God appointed Aaron as Moses' "prophet" (Exodus 4:10-17; 7:1). At the command of Moses, he let his rod turn into a snake. Then he stretched out his rod in order to bring on the first three plagues. After that, Moses tended to act and speak for himself. During the journey in the wilderness, Aaron was not always prominent or active. At the battle with Amalek, he was chosen with Hur to support the hand of Moses that held the "rod of God". When the revelation was given to Moses at biblical Mount Sinai, he headed the elders of Israel who accompanied Moses on the way to the summit.
+"Pokémon"	Pokémon , also known as in Japan, is a media franchise managed by The Pokémon Company, a Japanese consortium between Nintendo, Game Freak, and Creatures. The franchise copyright is shared by all three companies, but Nintendo is the sole owner of the trademark. The franchise was created by Satoshi Tajiri in 1995, and is centered on fictional creatures called "Pokémon", which humans, known as Pokémon Trainers, catch and train to battle each other for sport. The English slogan for the franchise is "Gotta Catch 'Em All". Works within the franchise are set in the Pokémon universe. The franchise began as "Pokémon Red" and "Green" (released outside of Japan as "Pokémon Red" and "Blue"), a pair of video games for the original Game Boy that were developed by Game Freak and published by Nintendo in February 1996. "Pokémon" has since gone on to become the highest-grossing media franchise of all time, with over in revenue up until March 2017. The original video game series is the second best-selling video game franchise (behind Nintendo's "Mario" franchise) with more than 300million copies sold and over 800million mobile downloads. In addition, the "Pokémon" franchise includes the world's top-selling toy brand, the top-selling trading card game with over 25.7billion cards sold, an anime television series that has become the most successful video game adaptation with over 20 seasons and 1,000 episodes in 124 countries, as well as an anime film series, a , books, manga comics, music, and merchandise. The franchise is also represented in other Nintendo media, such as the "Super Smash Bros." series. In November 2005, 4Kids Entertainment, which had managed the non-game related licensing of "Pokémon", announced that it had agreed not to renew the "Pokémon" representation agreement. The Pokémon Company International oversees all "Pokémon" licensing outside Asia.

examples/rag/test_distributed_retriever.py

@@ -15,6 +15,7 @@ from transformers.configuration_bart import BartConfig
 from transformers.configuration_dpr import DPRConfig
 from transformers.configuration_rag import RagConfig
 from transformers.file_utils import is_datasets_available, is_faiss_available, is_psutil_available, is_torch_available
+from transformers.retrieval_rag import CustomHFIndex
 from transformers.tokenization_bart import BartTokenizer
 from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
 from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer
@@ -23,7 +24,7 @@ from transformers.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FI
 
 sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip
 
-from examples.rag.distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
+from distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
 
 
 def require_distributed_retrieval(test_case):
@@ -114,7 +115,7 @@ class RagRetrieverTest(TestCase):
     def tearDown(self):
         shutil.rmtree(self.tmpdirname)
 
-    def get_dummy_pytorch_distributed_retriever(self, init_retrieval, port=12345) -> RagPyTorchDistributedRetriever:
+    def get_dummy_dataset(self):
         dataset = Dataset.from_dict(
             {
                 "id": ["0", "1"],
@@ -124,6 +125,12 @@ class RagRetrieverTest(TestCase):
             }
         )
         dataset.add_faiss_index("embeddings", string_factory="Flat", metric_type=faiss.METRIC_INNER_PRODUCT)
+        return dataset
+
+    def get_dummy_pytorch_distributed_retriever(
+        self, init_retrieval: bool, port=12345
+    ) -> RagPyTorchDistributedRetriever:
+        dataset = self.get_dummy_dataset()
         config = RagConfig(
             retrieval_vector_size=self.retrieval_vector_size,
             question_encoder=DPRConfig().to_dict(),
@@ -140,6 +147,37 @@ class RagRetrieverTest(TestCase):
                 retriever.init_retrieval(port)
         return retriever
 
+    def get_dummy_custom_hf_index_retriever(self, init_retrieval: bool, from_disk: bool, port=12345):
+        dataset = self.get_dummy_dataset()
+        config = RagConfig(
+            retrieval_vector_size=self.retrieval_vector_size,
+            question_encoder=DPRConfig().to_dict(),
+            generator=BartConfig().to_dict(),
+            index_name="custom",
+        )
+        if from_disk:
+            config.passages_path = os.path.join(self.tmpdirname, "dataset")
+            config.index_path = os.path.join(self.tmpdirname, "index.faiss")
+            dataset.get_index("embeddings").save(os.path.join(self.tmpdirname, "index.faiss"))
+            dataset.drop_index("embeddings")
+            dataset.save_to_disk(os.path.join(self.tmpdirname, "dataset"))
+            del dataset
+            retriever = RagPyTorchDistributedRetriever(
+                config,
+                question_encoder_tokenizer=self.get_dpr_tokenizer(),
+                generator_tokenizer=self.get_bart_tokenizer(),
+            )
+        else:
+            retriever = RagPyTorchDistributedRetriever(
+                config,
+                question_encoder_tokenizer=self.get_dpr_tokenizer(),
+                generator_tokenizer=self.get_bart_tokenizer(),
+                index=CustomHFIndex(config.retrieval_vector_size, dataset),
+            )
+        if init_retrieval:
+            retriever.init_retrieval(port)
+        return retriever
+
     def test_pytorch_distributed_retriever_retrieve(self):
         n_docs = 1
         retriever = self.get_dummy_pytorch_distributed_retriever(init_retrieval=True)
@@ -154,3 +192,33 @@ class RagRetrieverTest(TestCase):
         self.assertEqual(doc_dicts[0]["id"][0], "1")  # max inner product is reached with second doc
         self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
         self.assertListEqual(doc_ids.tolist(), [[1], [0]])
+
+    def test_custom_hf_index_retriever_retrieve(self):
+        n_docs = 1
+        retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=False)
+        hidden_states = np.array(
+            [np.ones(self.retrieval_vector_size), -np.ones(self.retrieval_vector_size)], dtype=np.float32
+        )
+        retrieved_doc_embeds, doc_ids, doc_dicts = retriever.retrieve(hidden_states, n_docs=n_docs)
+        self.assertEqual(retrieved_doc_embeds.shape, (2, n_docs, self.retrieval_vector_size))
+        self.assertEqual(len(doc_dicts), 2)
+        self.assertEqual(sorted(doc_dicts[0]), ["embeddings", "id", "text", "title"])
+        self.assertEqual(len(doc_dicts[0]["id"]), n_docs)
+        self.assertEqual(doc_dicts[0]["id"][0], "1")  # max inner product is reached with second doc
+        self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
+        self.assertListEqual(doc_ids.tolist(), [[1], [0]])
+
+    def test_custom_pytorch_distributed_retriever_retrieve_from_disk(self):
+        n_docs = 1
+        retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=True)
+        hidden_states = np.array(
+            [np.ones(self.retrieval_vector_size), -np.ones(self.retrieval_vector_size)], dtype=np.float32
+        )
+        retrieved_doc_embeds, doc_ids, doc_dicts = retriever.retrieve(hidden_states, n_docs=n_docs)
+        self.assertEqual(retrieved_doc_embeds.shape, (2, n_docs, self.retrieval_vector_size))
+        self.assertEqual(len(doc_dicts), 2)
+        self.assertEqual(sorted(doc_dicts[0]), ["embeddings", "id", "text", "title"])
+        self.assertEqual(len(doc_dicts[0]["id"]), n_docs)
+        self.assertEqual(doc_dicts[0]["id"][0], "1")  # max inner product is reached with second doc
+        self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
+        self.assertListEqual(doc_ids.tolist(), [[1], [0]])

examples/rag/use_own_knowledge_dataset.py

@@ -0,0 +1,199 @@
+import logging
+import os
+from dataclasses import dataclass, field
+from functools import partial
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import List, Optional
+
+import torch
+from datasets import load_dataset
+
+import faiss
+from transformers import (
+    DPRContextEncoder,
+    DPRContextEncoderTokenizerFast,
+    HfArgumentParser,
+    RagRetriever,
+    RagSequenceForGeneration,
+    RagTokenizer,
+)
+
+
+logger = logging.getLogger(__name__)
+torch.set_grad_enabled(False)
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+def split_text(text: str, n=100, character=" ") -> List[str]:
+    """Split the text every ``n``-th occurence of ``character``"""
+    text = text.split(character)
+    return [character.join(text[i : i + n]).strip() for i in range(0, len(text), n)]
+
+
+def split_documents(documents: dict) -> dict:
+    """Split documents into passages"""
+    titles, texts = [], []
+    for title, text in zip(documents["title"], documents["text"]):
+        for passage in split_text(text):
+            titles.append(title)
+            texts.append(passage)
+    return {"title": titles, "text": texts}
+
+
+def embed(documents: dict, ctx_encoder: DPRContextEncoder, ctx_tokenizer: DPRContextEncoderTokenizerFast) -> dict:
+    """Compute the DPR embeddings of document passages"""
+    input_ids = ctx_tokenizer(
+        documents["title"], documents["text"], truncation=True, padding="longest", return_tensors="pt"
+    )["input_ids"]
+    embeddings = ctx_encoder(input_ids.to(device=device), return_dict=True).pooler_output
+    return {"embeddings": embeddings.detach().cpu().numpy()}
+
+
+def main(
+    rag_example_args: "RagExampleArguments",
+    processing_args: "ProcessingArguments",
+    index_hnsw_args: "IndexHnswArguments",
+):
+
+    ######################################
+    logger.info("Step 1 - Create the dataset")
+    ######################################
+
+    # The dataset needed for RAG must have three columns:
+    # - title (string): title of the document
+    # - text (string): text of a passage of the document
+    # - embeddings (array of dimension d): DPR representation of the passage
+
+    # Let's say you have documents in tab-separated csv files with columns "title" and "text"
+    assert os.path.isfile(rag_example_args.csv_path), "Please provide a valid path to a csv file"
+
+    # You can load a Dataset object this way
+    dataset = load_dataset(
+        "csv", data_files=[rag_example_args.csv_path], split="train", delimiter="\t", column_names=["title", "text"]
+    )
+
+    # More info about loading csv files in the documentation: https://huggingface.co/docs/datasets/loading_datasets.html?highlight=csv#csv-files
+
+    # Then split the documents into passages of 100 words
+    dataset = dataset.map(split_documents, batched=True, num_proc=processing_args.num_proc)
+
+    # And compute the embeddings
+    ctx_encoder = DPRContextEncoder.from_pretrained(rag_example_args.dpr_ctx_encoder_model_name).to(device=device)
+    ctx_tokenizer = DPRContextEncoderTokenizerFast.from_pretrained(rag_example_args.dpr_ctx_encoder_model_name)
+    dataset = dataset.map(
+        partial(embed, ctx_encoder=ctx_encoder, ctx_tokenizer=ctx_tokenizer),
+        batched=True,
+        batch_size=processing_args.batch_size,
+    )
+
+    # And finally save your dataset
+    passages_path = os.path.join(rag_example_args.output_dir, "my_knowledge_dataset")
+    dataset.save_to_disk(passages_path)
+    # from datasets import load_from_disk
+    # dataset = load_from_disk(passages_path)  # to reload the dataset
+
+    ######################################
+    logger.info("Step 2 - Index the dataset")
+    ######################################
+
+    # Let's use the Faiss implementation of HNSW for fast approximate nearest neighbor search
+    index = faiss.IndexHNSWFlat(index_hnsw_args.d, index_hnsw_args.m, faiss.METRIC_INNER_PRODUCT)
+    dataset.add_faiss_index("embeddings", custom_index=index)
+
+    # And save the index
+    index_path = os.path.join(rag_example_args.output_dir, "my_knowledge_dataset_hnsw_index.faiss")
+    dataset.get_index("embeddings").save(index_path)
+    # dataset.load_faiss_index("embeddings", index_path)  # to reload the index
+
+    ######################################
+    logger.info("Step 3 - Load RAG")
+    ######################################
+
+    # Easy way to load the model
+    retriever = RagRetriever.from_pretrained(
+        rag_example_args.rag_model_name, index_name="custom", indexed_dataset=dataset
+    )
+    model = RagSequenceForGeneration.from_pretrained(rag_example_args.rag_model_name, retriever=retriever)
+    tokenizer = RagTokenizer.from_pretrained(rag_example_args.rag_model_name)
+
+    # For distributed fine-tuning you'll need to provide the paths instead, as the dataset and the index are loaded separately.
+    # retriever = RagRetriever.from_pretrained(rag_model_name, index_name="custom", passages_path=passages_path, index_path=index_path)
+
+    ######################################
+    logger.info("Step 4 - Have fun")
+    ######################################
+
+    question = rag_example_args.question or "What does Moses' rod turn into ?"
+    input_ids = tokenizer.question_encoder(question, return_tensors="pt")["input_ids"]
+    generated = model.generate(input_ids)
+    generated_string = tokenizer.batch_decode(generated, skip_special_tokens=True)[0]
+    logger.info("Q: " + question)
+    logger.info("A: " + generated_string)
+
+
+@dataclass
+class RagExampleArguments:
+    csv_path: str = field(
+        default=str(Path(__file__).parent / "test_data" / "my_knowledge_dataset.csv"),
+        metadata={"help": "Path to a tab-separated csv file with columns 'title' and 'text'"},
+    )
+    question: Optional[str] = field(
+        default=None,
+        metadata={"help": "Question that is passed as input to RAG. Default is 'What does Moses' rod turn into ?'."},
+    )
+    rag_model_name: str = field(
+        default="facebook/rag-sequence-nq",
+        metadata={"help": "The RAG model to use. Either 'facebook/rag-sequence-nq' or 'facebook/rag-token-nq'"},
+    )
+    dpr_ctx_encoder_model_name: str = field(
+        default="facebook/dpr-ctx_encoder-multiset-base",
+        metadata={
+            "help": "The DPR context encoder model to use. Either 'facebook/dpr-ctx_encoder-single-nq-base' or 'facebook/dpr-ctx_encoder-multiset-base'"
+        },
+    )
+    output_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Path to a directory where the dataset passages and the index will be saved"},
+    )
+
+
+@dataclass
+class ProcessingArguments:
+    num_proc: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "The number of processes to use to split the documents into passages. Default is single process."
+        },
+    )
+    batch_size: int = field(
+        default=16,
+        metadata={
+            "help": "The batch size to use when computing the passages embeddings using the DPR context encoder."
+        },
+    )
+
+
+@dataclass
+class IndexHnswArguments:
+    d: int = field(
+        default=768,
+        metadata={"help": "The dimension of the embeddings to pass to the HNSW Faiss index."},
+    )
+    m: int = field(
+        default=128,
+        metadata={
+            "help": "The number of bi-directional links created for every new element during the HNSW index construction."
+        },
+    )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.WARNING)
+    logger.setLevel(logging.INFO)
+
+    parser = HfArgumentParser((RagExampleArguments, ProcessingArguments, IndexHnswArguments))
+    rag_example_args, processing_args, index_hnsw_args = parser.parse_args_into_dataclasses()
+    with TemporaryDirectory() as tmp_dir:
+        rag_example_args.output_dir = rag_example_args.output_dir or tmp_dir
+        main(rag_example_args, processing_args, index_hnsw_args)

examples/requirements.txt

@@ -5,7 +5,7 @@ psutil
 sacrebleu
 rouge-score
 tensorflow_datasets
-pytorch-lightning==0.8.5
+pytorch-lightning==0.9.0
 matplotlib
 git-python==1.0.3
 faiss-cpu
@@ -17,3 +17,4 @@ datasets
 fire
 pytest
 conllu
+sentencepiece != 0.1.92

examples/seq2seq/README.md

@@ -12,14 +12,13 @@ For `bertabs` instructions, see [`bertabs/README.md`](bertabs/README.md).
 - `MBartForConditionalGeneration`
 - `FSMTForConditionalGeneration`
 - `T5ForConditionalGeneration`
-    
 
 ## Datasets
 
 #### XSUM:
 ```bash
 cd examples/seq2seq
-wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/xsum.tar.gz
+wget https://cdn-datasets.huggingface.co/summarization/xsum.tar.gz
 tar -xzvf xsum.tar.gz
 export XSUM_DIR=${PWD}/xsum
 ```
@@ -29,7 +28,7 @@ To use your own data, copy that files format. Each article to be summarized is o
 #### CNN/DailyMail
 ```bash
 cd examples/seq2seq
-wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/cnn_dm_v2.tgz
+wget https://cdn-datasets.huggingface.co/summarization/cnn_dm_v2.tgz
 tar -xzvf cnn_dm_v2.tgz  # empty lines removed
 mv cnn_cln cnn_dm
 export CNN_DIR=${PWD}/cnn_dm
@@ -39,7 +38,7 @@ this should make a directory called `cnn_dm/` with 6 files.
 #### WMT16 English-Romanian Translation Data:
 download with this command:
 ```bash
-wget https://s3.amazonaws.com/datasets.huggingface.co/translation/wmt_en_ro.tar.gz
+wget https://cdn-datasets.huggingface.co/translation/wmt_en_ro.tar.gz
 tar -xzvf wmt_en_ro.tar.gz
 export ENRO_DIR=${PWD}/wmt_en_ro
 ```
@@ -47,7 +46,7 @@ this should make a directory called `wmt_en_ro/` with 6 files.
 
 #### WMT English-German:
 ```bash
-wget https://s3.amazonaws.com/datasets.huggingface.co/translation/wmt_en_de.tgz
+wget https://cdn-datasets.huggingface.co/translation/wmt_en_de.tgz
 tar -xzvf wmt_en_de.tgz
 export DATA_DIR=${PWD}/wmt_en_de
 ```
@@ -100,7 +99,7 @@ All finetuning bash scripts call finetune.py (or distillation.py) with reasonabl
 To see all the possible command line options, run:
 
 ```bash
- ./finetune.py --help 
+./finetune.py --help
 ```
 
 ### Finetuning Training Params
@@ -192,7 +191,7 @@ model = AutoModelForSeq2SeqLM.from_pretrained(f'{output_dir}/best_tfmr')
 ### Fine-tuning using Seq2SeqTrainer
 To use `Seq2SeqTrainer` for fine-tuning you should use the `finetune_trainer.py` script. It subclasses `Trainer` to extend it for seq2seq training. Except the `Trainer` releated `TrainingArguments`, it shares the same argument names as that of `finetune.py` file. One notable difference is that, calculating generative metrics (BLEU, ROUGE) is optional and is controlled using the `--predict_with_generate` argument, set this argument to calculate BLEU and ROUGE metrics.
 
-With PyTorch 1.6+ it'll automatically use `native AMP` when `--fp16` is set. 
+With PyTorch 1.6+ it'll automatically use `native AMP` when `--fp16` is set.
 
 To see all the possible command line options, run:
 
@@ -265,6 +264,7 @@ export DATA_DIR=cnn_dm
     --fp16 \
     --bs 32
 ```
+
 ### Multi-GPU Evaluation
 here is a command to run xsum evaluation on 8 GPUS. It is more than linearly faster than run_eval.py in some cases 
 because it uses SortishSampler to minimize padding. You can also use it on 1 GPU. `data_dir` must have 
@@ -391,6 +391,17 @@ runtime: 13H on V-100 16GB GPU.
 pytest examples/seq2seq/
 ```
 
+### Converting pytorch-lightning checkpoints
+pytorch lightning ``-do_predict`` often fails, after you are done training, the best way to evaluate your model is to convert it.
+
+This should be done for you, with a file called `{save_dir}/best_tfmr`. 
+
+If that file doesn't exist but you have a lightning `.ckpt` file, you can run
+```bash
+python convert_pl_checkpoint_to_hf.py PATH_TO_CKPT  randomly_initialized_hf_model_path save_dir/best_tfmr
+```
+Then either `run_eval` or `run_distributed_eval` with `save_dir/best_tfmr` (see previous sections)
+
 
 ## Experimental Features 
 These features are harder to use and not always useful.
@@ -419,4 +430,3 @@ uses 12,723 batches of length 48 and takes slightly more time 9.5 minutes.
 The feature is still experimental, because:
 + we can make it much more robust if we have memory mapped/preprocessed datasets.
 + The speedup over sortish sampler is not that large at the moment.
-

examples/seq2seq/bertabs/README.md

@@ -39,7 +39,7 @@ python run_summarization.py \
     --compute_rouge true
 ```
 
-The scripts executes on GPU if one is available and if `no_cuda` is not set to `true`. Inference on multiple GPUs is not suported yet. The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. The script takes 30 hours to compute with a single Tesla V100 GPU and a batch size of 10 (300,000 texts to summarize).
+The scripts executes on GPU if one is available and if `no_cuda` is not set to `true`. Inference on multiple GPUs is not supported yet. The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. The script takes 30 hours to compute with a single Tesla V100 GPU and a batch size of 10 (300,000 texts to summarize).
 
 ## Summarize any text
 

examples/seq2seq/distillation.py

@@ -17,7 +17,7 @@ from finetune import main as ft_main
 from make_student import create_student_by_copying_alternating_layers, get_layers_to_supervise
 from transformers import AutoModelForSeq2SeqLM, MBartTokenizer, T5ForConditionalGeneration
 from transformers.modeling_bart import shift_tokens_right
-from utils import calculate_bleu, freeze_params, label_smoothed_nll_loss, pickle_load, use_task_specific_params
+from utils import calculate_bleu, freeze_params, label_smoothed_nll_loss, use_task_specific_params
 
 
 # need the parent dir module
@@ -264,30 +264,6 @@ def create_module(args):
     return model
 
 
-def evaluate_checkpoint(ckpt_path: Path, dest_dir=None):
-    # TODO(SS): DELETE? Better to convert_pl_ckpt_to_hf and run_eval.py
-    exp_dir = ckpt_path.parent
-    if dest_dir is None:
-        dest_dir = exp_dir
-    clash = list(dest_dir.glob("test_generations*"))
-    if clash:
-        print(f"SKIPPING to avoid overwriting {clash}")
-    ckpt = torch.load(ckpt_path, map_location="cpu")
-    if "hparams" in ckpt:
-        args = argparse.Namespace(**ckpt["hparams"])
-    else:
-        args = argparse.Namespace(**pickle_load(exp_dir / "hparams.pkl"))
-    args.resume_from_checkpoint = str(ckpt_path)
-    args.do_train = False
-    args.output_dir = str(dest_dir)
-    args.n_gpu = 1
-    args.eval_batch_size = 16
-    Path(args.output_dir).mkdir(exist_ok=True)
-    model = create_module(args)
-    trainer: pl.Trainer = generic_train(model, args, early_stopping_callback=False)
-    trainer.test(model)
-
-
 def distill_main(args):
     Path(args.output_dir).mkdir(exist_ok=True)
     if len(os.listdir(args.output_dir)) > 3 and args.do_train:

examples/seq2seq/finetune.py

@@ -181,6 +181,7 @@ class SummarizationModule(BaseTransformer):
         return self._generative_step(batch)
 
     def validation_epoch_end(self, outputs, prefix="val") -> Dict:
+
         self.step_count += 1
         losses = {k: torch.stack([x[k] for x in outputs]).mean() for k in self.loss_names}
         loss = losses["loss"]

examples/seq2seq/finetune_bart_tiny.sh

@@ -1,7 +1,7 @@
 # Script for verifying that run_bart_sum can be invoked from its directory
 
 # Get tiny dataset with cnn_dm format (4 examples for train, val, test)
-wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/cnn_tiny.tgz
+wget https://cdn-datasets.huggingface.co/summarization/cnn_tiny.tgz
 tar -xzvf cnn_tiny.tgz
 rm cnn_tiny.tgz
 

examples/seq2seq/finetune_trainer.py

@@ -4,7 +4,7 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional
 
-from seq2seq_trainer import Seq2SeqTrainer
+from seq2seq_trainer import Seq2SeqTrainer, arg_to_scheduler_choices
 from transformers import (
     AutoConfig,
     AutoModelForSeq2SeqLM,
@@ -63,6 +63,9 @@ class Seq2SeqTrainingArguments(TrainingArguments):
     attention_dropout: Optional[float] = field(
         default=None, metadata={"help": "Attention dropout probability. Goes into model.config."}
     )
+    lr_scheduler: Optional[str] = field(
+        default="linear", metadata={"help": f"Which lr scheduler to use. Selected in {arg_to_scheduler_choices}"}
+    )
 
 
 @dataclass

examples/seq2seq/precomputed_pseudo_labels.md

@@ -4,24 +4,24 @@ These are the generations of various large models on various large **training**
 ### Available Pseudo-labels
 | Dataset | Model                       | Link                                                                                   | Rouge Scores       | Notes                                                                                                       
 |---------|-----------------------------|----------------------------------------------------------------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------
-| XSUM    | `facebook/bart-large-xsum`    | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/xsum/bart_xsum_pl.tgz)          | 49.8/28.0/42.5     |                                                                                                             
-| XSUM    | `google/pegasus-xsum`         | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/xsum/pegasus_xsum.tgz)          | 53.3/32.7/46.5     |                                                                                                             
-| XSUM    | `facebook/bart-large-xsum`    | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/xsum/xsum_pl2_bart.tgz)         |                   | Bart pseudolabels filtered to those with Rouge2 > 10.0 w GT.                                                 
-| CNN/DM  | `sshleifer/pegasus-cnn-ft-v2` | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/cnn_dm/pegasus_cnn_cnn_pls.tgz) | 47.316/26.65/44.56 | do not worry about the fact that train.source is one line shorter.                                          
-| CNN/DM  | `facebook/bart-large-cnn`     | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/cnn_dm/cnn_bart_pl.tgz)         |                    | 5K (2%) are missing, there should be 282173                                                                 
-| CNN/DM  | `google/pegasus-xsum`         | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/cnn_dm/pegasus_xsum_on_cnn.tgz) | 21.5/6.76/25       | extra labels for xsum distillation  Used max_source_length=512, (and all other pegasus-xsum configuration). 
-| EN-RO   | `Helsinki-NLP/opus-mt-en-ro`  | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/wmt_en_ro/opus_mt_en_ro.tgz) |       |  
-| EN-RO   | `facebook/mbart-large-en-ro`  | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/wmt_en_ro/mbart_large_en_ro.tgz) |       |  
+| XSUM    | `facebook/bart-large-xsum`    | [download](https://cdn-datasets.huggingface.co/pseudo/xsum/bart_xsum_pl.tgz)          | 49.8/28.0/42.5     |                                                                                                             
+| XSUM    | `google/pegasus-xsum`         | [download](https://cdn-datasets.huggingface.co/pseudo/xsum/pegasus_xsum.tgz)          | 53.3/32.7/46.5     |                                                                                                             
+| XSUM    | `facebook/bart-large-xsum`    | [download](https://cdn-datasets.huggingface.co/pseudo/xsum/xsum_pl2_bart.tgz)         |                   | Bart pseudolabels filtered to those with Rouge2 > 10.0 w GT.                                                 
+| CNN/DM  | `sshleifer/pegasus-cnn-ft-v2` | [download](https://cdn-datasets.huggingface.co/pseudo/cnn_dm/pegasus_cnn_cnn_pls.tgz) | 47.316/26.65/44.56 | do not worry about the fact that train.source is one line shorter.                                          
+| CNN/DM  | `facebook/bart-large-cnn`     | [download](https://cdn-datasets.huggingface.co/pseudo/cnn_dm/cnn_bart_pl.tgz)         |                    | 5K (2%) are missing, there should be 282173                                                                 
+| CNN/DM  | `google/pegasus-xsum`         | [download](https://cdn-datasets.huggingface.co/pseudo/cnn_dm/pegasus_xsum_on_cnn.tgz) | 21.5/6.76/25       | extra labels for xsum distillation  Used max_source_length=512, (and all other pegasus-xsum configuration). 
+| EN-RO   | `Helsinki-NLP/opus-mt-en-ro`  | [download](https://cdn-datasets.huggingface.co/pseudo/wmt_en_ro/opus_mt_en_ro.tgz) |       |  
+| EN-RO   | `facebook/mbart-large-en-ro`  | [download](https://cdn-datasets.huggingface.co/pseudo/wmt_en_ro/mbart_large_en_ro.tgz) |       |  
 
 
 (EN_RO = WMT 2016 English-Romanian).
 
 Example Download Command:
 ```bash
-curl -S https://s3.amazonaws.com/datasets.huggingface.co/pseudo/xsum/bart_xsum_pl.tgz | tar -xvz -C .
+curl -S https://cdn-datasets.huggingface.co/pseudo/xsum/bart_xsum_pl.tgz | tar -xvz -C .
 ```
 ### Generating New Pseudolabels
-Here is the command I used to generate the pseudolabels in the second row of the table, after downloading XSUM from [here](https://s3.amazonaws.com/datasets.huggingface.co/summarization/xsum.tar.gz). 
+Here is the command I used to generate the pseudolabels in the second row of the table, after downloading XSUM from [here](https://cdn-datasets.huggingface.co/summarization/xsum.tar.gz). 
 
 ```bash                                                                         
 python -m torch.distributed.launch --nproc_per_node=8 run_distributed_eval.py \

examples/seq2seq/seq2seq_trainer.py

@@ -8,7 +8,16 @@ from torch.utils.data import DistributedSampler, RandomSampler
 from transformers import Trainer
 from transformers.configuration_fsmt import FSMTConfig
 from transformers.file_utils import is_torch_tpu_available
-from transformers.optimization import Adafactor, AdamW, get_linear_schedule_with_warmup
+from transformers.optimization import (
+    Adafactor,
+    AdamW,
+    get_constant_schedule,
+    get_constant_schedule_with_warmup,
+    get_cosine_schedule_with_warmup,
+    get_cosine_with_hard_restarts_schedule_with_warmup,
+    get_linear_schedule_with_warmup,
+    get_polynomial_decay_schedule_with_warmup,
+)
 from transformers.trainer_pt_utils import get_tpu_sampler
 
 
@@ -20,6 +29,16 @@ except ImportError:
 
 logger = logging.getLogger(__name__)
 
+arg_to_scheduler = {
+    "linear": get_linear_schedule_with_warmup,
+    "cosine": get_cosine_schedule_with_warmup,
+    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
+    "polynomial": get_polynomial_decay_schedule_with_warmup,
+    "constant": get_constant_schedule,
+    "constant_w_warmup": get_constant_schedule_with_warmup,
+}
+arg_to_scheduler_choices = sorted(arg_to_scheduler.keys())
+
 
 class Seq2SeqTrainer(Trainer):
     def __init__(self, config, data_args, *args, **kwargs):
@@ -62,9 +81,21 @@ class Seq2SeqTrainer(Trainer):
                 )
 
         if self.lr_scheduler is None:
-            self.lr_scheduler = get_linear_schedule_with_warmup(
+            self.lr_scheduler = self._get_lr_scheduler(num_training_steps)
+        else:  # ignoring --lr_scheduler
+            logger.warn("scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.")
+
+    def _get_lr_scheduler(self, num_training_steps):
+        schedule_func = arg_to_scheduler[self.args.lr_scheduler]
+        if self.args.lr_scheduler == "constant":
+            scheduler = schedule_func(self.optimizer)
+        elif self.args.lr_scheduler == "constant_w_warmup":
+            scheduler = schedule_func(self.optimizer, num_warmup_steps=self.args.warmup_steps)
+        else:
+            scheduler = schedule_func(
                 self.optimizer, num_warmup_steps=self.args.warmup_steps, num_training_steps=num_training_steps
             )
+        return scheduler
 
     def _get_train_sampler(self) -> Optional[torch.utils.data.sampler.Sampler]:
         if isinstance(self.train_dataset, torch.utils.data.IterableDataset):
@@ -143,7 +174,7 @@ class Seq2SeqTrainer(Trainer):
             # Call forward again to get loss # TODO: avoidable?
             outputs = model(**inputs, use_cache=False)
             loss = self._compute_loss(outputs[1], labels_out)
-            loss = loss.mean().item()
+            loss = loss.mean().detach()
             if self.args.prediction_loss_only:
                 return (loss, None, None)
 

examples/seq2seq/test_bash_script.py

@@ -3,7 +3,6 @@
 import argparse
 import os
 import sys
-import tempfile
 from pathlib import Path
 from unittest.mock import patch
 
@@ -16,172 +15,172 @@ from distillation import BartSummarizationDistiller, distill_main
 from finetune import SummarizationModule, main
 from test_seq2seq_examples import CUDA_AVAILABLE, MBART_TINY
 from transformers import BartForConditionalGeneration, MarianMTModel
-from transformers.testing_utils import slow
+from transformers.testing_utils import TestCasePlus, slow
 from utils import load_json
 
 
 MODEL_NAME = MBART_TINY
-# TODO(SS): MODEL_NAME = "sshleifer/student_mbart_en_ro_1_1"
 MARIAN_MODEL = "sshleifer/student_marian_en_ro_6_1"
 
 
-@slow
-@pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
-def test_model_download():
-    """This warms up the cache so that we can time the next test without including download time, which varies between machines."""
-    BartForConditionalGeneration.from_pretrained(MODEL_NAME)
-    MarianMTModel.from_pretrained(MARIAN_MODEL)
+class TestAll(TestCasePlus):
+    @slow
+    @pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
+    def test_model_download(self):
+        """This warms up the cache so that we can time the next test without including download time, which varies between machines."""
+        BartForConditionalGeneration.from_pretrained(MODEL_NAME)
+        MarianMTModel.from_pretrained(MARIAN_MODEL)
 
+    @timeout_decorator.timeout(120)
+    @slow
+    @pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
+    def test_train_mbart_cc25_enro_script(self):
+        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
+        env_vars_to_replace = {
+            "--fp16_opt_level=O1": "",
+            "$MAX_LEN": 128,
+            "$BS": 4,
+            "$GAS": 1,
+            "$ENRO_DIR": data_dir,
+            "facebook/mbart-large-cc25": MODEL_NAME,
+            # Download is 120MB in previous test.
+            "val_check_interval=0.25": "val_check_interval=1.0",
+        }
 
-@timeout_decorator.timeout(120)
-@slow
-@pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
-def test_train_mbart_cc25_enro_script():
-    data_dir = "examples/seq2seq/test_data/wmt_en_ro"
-    env_vars_to_replace = {
-        "--fp16_opt_level=O1": "",
-        "$MAX_LEN": 128,
-        "$BS": 4,
-        "$GAS": 1,
-        "$ENRO_DIR": data_dir,
-        "facebook/mbart-large-cc25": MODEL_NAME,
-        # Download is 120MB in previous test.
-        "val_check_interval=0.25": "val_check_interval=1.0",
-    }
+        # Clean up bash script
+        bash_script = Path("examples/seq2seq/train_mbart_cc25_enro.sh").open().read().split("finetune.py")[1].strip()
+        bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
+        for k, v in env_vars_to_replace.items():
+            bash_script = bash_script.replace(k, str(v))
+        output_dir = self.get_auto_remove_tmp_dir()
 
-    # Clean up bash script
-    bash_script = Path("examples/seq2seq/train_mbart_cc25_enro.sh").open().read().split("finetune.py")[1].strip()
-    bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
-    for k, v in env_vars_to_replace.items():
-        bash_script = bash_script.replace(k, str(v))
-    output_dir = tempfile.mkdtemp(prefix="output_mbart")
+        bash_script = bash_script.replace("--fp16 ", "")
+        testargs = (
+            ["finetune.py"]
+            + bash_script.split()
+            + [
+                f"--output_dir={output_dir}",
+                "--gpus=1",
+                "--learning_rate=3e-1",
+                "--warmup_steps=0",
+                "--val_check_interval=1.0",
+                "--tokenizer_name=facebook/mbart-large-en-ro",
+            ]
+        )
+        with patch.object(sys, "argv", testargs):
+            parser = argparse.ArgumentParser()
+            parser = pl.Trainer.add_argparse_args(parser)
+            parser = SummarizationModule.add_model_specific_args(parser, os.getcwd())
+            args = parser.parse_args()
+            args.do_predict = False
+            # assert args.gpus == gpus THIS BREAKS for multigpu
+            model = main(args)
 
-    bash_script = bash_script.replace("--fp16 ", "")
-    testargs = (
-        ["finetune.py"]
-        + bash_script.split()
-        + [
-            f"--output_dir={output_dir}",
-            "--gpus=1",
-            "--learning_rate=3e-1",
-            "--warmup_steps=0",
-            "--val_check_interval=1.0",
-            "--tokenizer_name=facebook/mbart-large-en-ro",
-        ]
-    )
-    with patch.object(sys, "argv", testargs):
-        parser = argparse.ArgumentParser()
-        parser = pl.Trainer.add_argparse_args(parser)
-        parser = SummarizationModule.add_model_specific_args(parser, os.getcwd())
-        args = parser.parse_args()
-        args.do_predict = False
-        # assert args.gpus == gpus THIS BREAKS for multigpu
-        model = main(args)
+        # Check metrics
+        metrics = load_json(model.metrics_save_path)
+        first_step_stats = metrics["val"][0]
+        last_step_stats = metrics["val"][-1]
+        assert (
+            len(metrics["val"]) == (args.max_epochs / args.val_check_interval) + 1
+        )  # +1 accounts for val_sanity_check
 
-    # Check metrics
-    metrics = load_json(model.metrics_save_path)
-    first_step_stats = metrics["val"][0]
-    last_step_stats = metrics["val"][-1]
-    assert len(metrics["val"]) == (args.max_epochs / args.val_check_interval) + 1  # +1 accounts for val_sanity_check
+        assert last_step_stats["val_avg_gen_time"] >= 0.01
 
-    assert last_step_stats["val_avg_gen_time"] >= 0.01
+        assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
+        assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
+        assert isinstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
 
-    assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
-    assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
-    assert isinstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
+        # check lightning ckpt can be loaded and has a reasonable statedict
+        contents = os.listdir(output_dir)
+        ckpt_path = [x for x in contents if x.endswith(".ckpt")][0]
+        full_path = os.path.join(args.output_dir, ckpt_path)
+        ckpt = torch.load(full_path, map_location="cpu")
+        expected_key = "model.model.decoder.layers.0.encoder_attn_layer_norm.weight"
+        assert expected_key in ckpt["state_dict"]
+        assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.float32
 
-    # check lightning ckpt can be loaded and has a reasonable statedict
-    contents = os.listdir(output_dir)
-    ckpt_path = [x for x in contents if x.endswith(".ckpt")][0]
-    full_path = os.path.join(args.output_dir, ckpt_path)
-    ckpt = torch.load(full_path, map_location="cpu")
-    expected_key = "model.model.decoder.layers.0.encoder_attn_layer_norm.weight"
-    assert expected_key in ckpt["state_dict"]
-    assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.float32
+        # TODO: turn on args.do_predict when PL bug fixed.
+        if args.do_predict:
+            contents = {os.path.basename(p) for p in contents}
+            assert "test_generations.txt" in contents
+            assert "test_results.txt" in contents
+            # assert len(metrics["val"]) ==  desired_n_evals
+            assert len(metrics["test"]) == 1
 
-    # TODO(SS): turn on args.do_predict when PL bug fixed.
-    if args.do_predict:
-        contents = {os.path.basename(p) for p in contents}
-        assert "test_generations.txt" in contents
-        assert "test_results.txt" in contents
-        # assert len(metrics["val"]) ==  desired_n_evals
-        assert len(metrics["test"]) == 1
+    @timeout_decorator.timeout(600)
+    @slow
+    @pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
+    def test_opus_mt_distill_script(self):
+        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
+        env_vars_to_replace = {
+            "--fp16_opt_level=O1": "",
+            "$MAX_LEN": 128,
+            "$BS": 16,
+            "$GAS": 1,
+            "$ENRO_DIR": data_dir,
+            "$m": "sshleifer/student_marian_en_ro_6_1",
+            "val_check_interval=0.25": "val_check_interval=1.0",
+        }
 
+        # Clean up bash script
+        bash_script = (
+            Path("examples/seq2seq/distil_marian_no_teacher.sh").open().read().split("distillation.py")[1].strip()
+        )
+        bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
+        bash_script = bash_script.replace("--fp16 ", " ")
 
-@timeout_decorator.timeout(600)
-@slow
-@pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
-def test_opus_mt_distill_script():
-    data_dir = "examples/seq2seq/test_data/wmt_en_ro"
-    env_vars_to_replace = {
-        "--fp16_opt_level=O1": "",
-        "$MAX_LEN": 128,
-        "$BS": 16,
-        "$GAS": 1,
-        "$ENRO_DIR": data_dir,
-        "$m": "sshleifer/student_marian_en_ro_6_1",
-        "val_check_interval=0.25": "val_check_interval=1.0",
-    }
+        for k, v in env_vars_to_replace.items():
+            bash_script = bash_script.replace(k, str(v))
+        output_dir = self.get_auto_remove_tmp_dir()
+        bash_script = bash_script.replace("--fp16", "")
+        epochs = 6
+        testargs = (
+            ["distillation.py"]
+            + bash_script.split()
+            + [
+                f"--output_dir={output_dir}",
+                "--gpus=1",
+                "--learning_rate=1e-3",
+                f"--num_train_epochs={epochs}",
+                "--warmup_steps=10",
+                "--val_check_interval=1.0",
+            ]
+        )
+        with patch.object(sys, "argv", testargs):
+            parser = argparse.ArgumentParser()
+            parser = pl.Trainer.add_argparse_args(parser)
+            parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+            args = parser.parse_args()
+            args.do_predict = False
+            # assert args.gpus == gpus THIS BREAKS for multigpu
 
-    # Clean up bash script
-    bash_script = (
-        Path("examples/seq2seq/distil_marian_no_teacher.sh").open().read().split("distillation.py")[1].strip()
-    )
-    bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
-    bash_script = bash_script.replace("--fp16 ", " ")
+            model = distill_main(args)
 
-    for k, v in env_vars_to_replace.items():
-        bash_script = bash_script.replace(k, str(v))
-    output_dir = tempfile.mkdtemp(prefix="marian_output")
-    bash_script = bash_script.replace("--fp16", "")
-    epochs = 6
-    testargs = (
-        ["distillation.py"]
-        + bash_script.split()
-        + [
-            f"--output_dir={output_dir}",
-            "--gpus=1",
-            "--learning_rate=1e-3",
-            f"--num_train_epochs={epochs}",
-            "--warmup_steps=10",
-            "--val_check_interval=1.0",
-        ]
-    )
-    with patch.object(sys, "argv", testargs):
-        parser = argparse.ArgumentParser()
-        parser = pl.Trainer.add_argparse_args(parser)
-        parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
-        args = parser.parse_args()
-        args.do_predict = False
-        # assert args.gpus == gpus THIS BREAKS for multigpu
+        # Check metrics
+        metrics = load_json(model.metrics_save_path)
+        first_step_stats = metrics["val"][0]
+        last_step_stats = metrics["val"][-1]
+        assert len(metrics["val"]) >= (args.max_epochs / args.val_check_interval)  # +1 accounts for val_sanity_check
 
-        model = distill_main(args)
+        assert last_step_stats["val_avg_gen_time"] >= 0.01
 
-    # Check metrics
-    metrics = load_json(model.metrics_save_path)
-    first_step_stats = metrics["val"][0]
-    last_step_stats = metrics["val"][-1]
-    assert len(metrics["val"]) >= (args.max_epochs / args.val_check_interval)  # +1 accounts for val_sanity_check
+        assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
+        assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
+        assert isinstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
 
-    assert last_step_stats["val_avg_gen_time"] >= 0.01
+        # check lightning ckpt can be loaded and has a reasonable statedict
+        contents = os.listdir(output_dir)
+        ckpt_path = [x for x in contents if x.endswith(".ckpt")][0]
+        full_path = os.path.join(args.output_dir, ckpt_path)
+        ckpt = torch.load(full_path, map_location="cpu")
+        expected_key = "model.model.decoder.layers.0.encoder_attn_layer_norm.weight"
+        assert expected_key in ckpt["state_dict"]
+        assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.float32
 
-    assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
-    assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
-    assert isinstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
-
-    # check lightning ckpt can be loaded and has a reasonable statedict
-    contents = os.listdir(output_dir)
-    ckpt_path = [x for x in contents if x.endswith(".ckpt")][0]
-    full_path = os.path.join(args.output_dir, ckpt_path)
-    ckpt = torch.load(full_path, map_location="cpu")
-    expected_key = "model.model.decoder.layers.0.encoder_attn_layer_norm.weight"
-    assert expected_key in ckpt["state_dict"]
-    assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.float32
-
-    # TODO(SS): turn on args.do_predict when PL bug fixed.
-    if args.do_predict:
-        contents = {os.path.basename(p) for p in contents}
-        assert "test_generations.txt" in contents
-        assert "test_results.txt" in contents
-        # assert len(metrics["val"]) ==  desired_n_evals
-        assert len(metrics["test"]) == 1
+        # TODO: turn on args.do_predict when PL bug fixed.
+        if args.do_predict:
+            contents = {os.path.basename(p) for p in contents}
+            assert "test_generations.txt" in contents
+            assert "test_results.txt" in contents
+            # assert len(metrics["val"]) ==  desired_n_evals
+            assert len(metrics["test"]) == 1

examples/seq2seq/test_datasets.py

@@ -1,5 +1,4 @@
 import os
-import tempfile
 from pathlib import Path
 
 import numpy as np
@@ -7,11 +6,12 @@ import pytest
 from torch.utils.data import DataLoader
 
 from pack_dataset import pack_data_dir
+from parameterized import parameterized
 from save_len_file import save_len_file
 from test_seq2seq_examples import ARTICLES, BART_TINY, MARIAN_TINY, MBART_TINY, SUMMARIES, T5_TINY, make_test_data_dir
 from transformers import AutoTokenizer
 from transformers.modeling_bart import shift_tokens_right
-from transformers.testing_utils import slow
+from transformers.testing_utils import TestCasePlus, slow
 from utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeq2SeqDataset, Seq2SeqDataset
 
 
@@ -19,202 +19,198 @@ BERT_BASE_CASED = "bert-base-cased"
 PEGASUS_XSUM = "google/pegasus-xsum"
 
 
-@slow
-@pytest.mark.parametrize(
-    "tok_name",
-    [
-        MBART_TINY,
-        MARIAN_TINY,
-        T5_TINY,
-        BART_TINY,
-        PEGASUS_XSUM,
-    ],
-)
-def test_seq2seq_dataset_truncation(tok_name):
-    tokenizer = AutoTokenizer.from_pretrained(tok_name)
-    tmp_dir = make_test_data_dir()
-    max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
-    max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
-    max_src_len = 4
-    max_tgt_len = 8
-    assert max_len_target > max_src_len  # Will be truncated
-    assert max_len_source > max_src_len  # Will be truncated
-    src_lang, tgt_lang = "ro_RO", "de_DE"  # ignored for all but mbart, but never causes error.
-    train_dataset = Seq2SeqDataset(
-        tokenizer,
-        data_dir=tmp_dir,
-        type_path="train",
-        max_source_length=max_src_len,
-        max_target_length=max_tgt_len,  # ignored
-        src_lang=src_lang,
-        tgt_lang=tgt_lang,
+class TestAll(TestCasePlus):
+    @parameterized.expand(
+        [
+            MBART_TINY,
+            MARIAN_TINY,
+            T5_TINY,
+            BART_TINY,
+            PEGASUS_XSUM,
+        ],
     )
-    dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
-    for batch in dataloader:
-        assert isinstance(batch, dict)
-        assert batch["attention_mask"].shape == batch["input_ids"].shape
-        # show that articles were trimmed.
-        assert batch["input_ids"].shape[1] == max_src_len
-        # show that targets are the same len
-        assert batch["labels"].shape[1] == max_tgt_len
-        if tok_name != MBART_TINY:
-            continue
-        # check language codes in correct place
-        batch["decoder_input_ids"] = shift_tokens_right(batch["labels"], tokenizer.pad_token_id)
-        assert batch["decoder_input_ids"][0, 0].item() == tokenizer.lang_code_to_id[tgt_lang]
-        assert batch["decoder_input_ids"][0, -1].item() == tokenizer.eos_token_id
-        assert batch["input_ids"][0, -2].item() == tokenizer.eos_token_id
-        assert batch["input_ids"][0, -1].item() == tokenizer.lang_code_to_id[src_lang]
-
-        break  # No need to test every batch
-
-
-@pytest.mark.parametrize("tok", [BART_TINY, BERT_BASE_CASED])
-def test_legacy_dataset_truncation(tok):
-    tokenizer = AutoTokenizer.from_pretrained(tok)
-    tmp_dir = make_test_data_dir()
-    max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
-    max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
-    trunc_target = 4
-    train_dataset = LegacySeq2SeqDataset(
-        tokenizer,
-        data_dir=tmp_dir,
-        type_path="train",
-        max_source_length=20,
-        max_target_length=trunc_target,
-    )
-    dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
-    for batch in dataloader:
-        assert batch["attention_mask"].shape == batch["input_ids"].shape
-        # show that articles were trimmed.
-        assert batch["input_ids"].shape[1] == max_len_source
-        assert 20 >= batch["input_ids"].shape[1]  # trimmed significantly
-        # show that targets were truncated
-        assert batch["labels"].shape[1] == trunc_target  # Truncated
-        assert max_len_target > trunc_target  # Truncated
-        break  # No need to test every batch
-
-
-def test_pack_dataset():
-    tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25")
-
-    tmp_dir = Path(make_test_data_dir())
-    orig_examples = tmp_dir.joinpath("train.source").open().readlines()
-    save_dir = Path(tempfile.mkdtemp(prefix="packed_"))
-    pack_data_dir(tokenizer, tmp_dir, 128, save_dir)
-    orig_paths = {x.name for x in tmp_dir.iterdir()}
-    new_paths = {x.name for x in save_dir.iterdir()}
-    packed_examples = save_dir.joinpath("train.source").open().readlines()
-    # orig: [' Sam ate lunch today.\n', 'Sams lunch ingredients.']
-    # desired_packed: [' Sam ate lunch today.\n Sams lunch ingredients.']
-    assert len(packed_examples) < len(orig_examples)
-    assert len(packed_examples) == 1
-    assert len(packed_examples[0]) == sum(len(x) for x in orig_examples)
-    assert orig_paths == new_paths
-
-
-@pytest.mark.skipif(not FAIRSEQ_AVAILABLE, reason="This test requires fairseq")
-def test_dynamic_batch_size():
-    if not FAIRSEQ_AVAILABLE:
-        return
-    ds, max_tokens, tokenizer = _get_dataset(max_len=64)
-    required_batch_size_multiple = 64
-    batch_sampler = ds.make_dynamic_sampler(max_tokens, required_batch_size_multiple=required_batch_size_multiple)
-    batch_sizes = [len(x) for x in batch_sampler]
-    assert len(set(batch_sizes)) > 1  # it's not dynamic batch size if every batch is the same length
-    assert sum(batch_sizes) == len(ds)  # no dropped or added examples
-    data_loader = DataLoader(ds, batch_sampler=batch_sampler, collate_fn=ds.collate_fn, num_workers=2)
-    failures = []
-    num_src_per_batch = []
-    for batch in data_loader:
-        src_shape = batch["input_ids"].shape
-        bs = src_shape[0]
-        assert bs % required_batch_size_multiple == 0 or bs < required_batch_size_multiple
-        num_src_tokens = np.product(batch["input_ids"].shape)
-        num_src_per_batch.append(num_src_tokens)
-        if num_src_tokens > (max_tokens * 1.1):
-            failures.append(num_src_tokens)
-    assert num_src_per_batch[0] == max(num_src_per_batch)
-    if failures:
-        raise AssertionError(f"too many tokens in {len(failures)} batches")
-
-
-def test_sortish_sampler_reduces_padding():
-    ds, _, tokenizer = _get_dataset(max_len=512)
-    bs = 2
-    sortish_sampler = ds.make_sortish_sampler(bs, shuffle=False)
-
-    naive_dl = DataLoader(ds, batch_size=bs, collate_fn=ds.collate_fn, num_workers=2)
-    sortish_dl = DataLoader(ds, batch_size=bs, collate_fn=ds.collate_fn, num_workers=2, sampler=sortish_sampler)
-
-    pad = tokenizer.pad_token_id
-
-    def count_pad_tokens(data_loader, k="input_ids"):
-        return [batch[k].eq(pad).sum().item() for batch in data_loader]
-
-    assert sum(count_pad_tokens(sortish_dl, k="labels")) < sum(count_pad_tokens(naive_dl, k="labels"))
-    assert sum(count_pad_tokens(sortish_dl)) < sum(count_pad_tokens(naive_dl))
-    assert len(sortish_dl) == len(naive_dl)
-
-
-def _get_dataset(n_obs=1000, max_len=128):
-    if os.getenv("USE_REAL_DATA", False):
-        data_dir = "examples/seq2seq/wmt_en_ro"
-        max_tokens = max_len * 2 * 64
-        if not Path(data_dir).joinpath("train.len").exists():
-            save_len_file(MARIAN_TINY, data_dir)
-    else:
-        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
-        max_tokens = max_len * 4
-        save_len_file(MARIAN_TINY, data_dir)
-
-    tokenizer = AutoTokenizer.from_pretrained(MARIAN_TINY)
-    ds = Seq2SeqDataset(
-        tokenizer,
-        data_dir=data_dir,
-        type_path="train",
-        max_source_length=max_len,
-        max_target_length=max_len,
-        n_obs=n_obs,
-    )
-    return ds, max_tokens, tokenizer
-
-
-def test_distributed_sortish_sampler_splits_indices_between_procs():
-    ds, max_tokens, tokenizer = _get_dataset()
-    ids1 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=0, add_extra_examples=False))
-    ids2 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=1, add_extra_examples=False))
-    assert ids1.intersection(ids2) == set()
-
-
-@pytest.mark.parametrize(
-    "tok_name",
-    [
-        MBART_TINY,
-        MARIAN_TINY,
-        T5_TINY,
-        BART_TINY,
-        PEGASUS_XSUM,
-    ],
-)
-def test_dataset_kwargs(tok_name):
-    tokenizer = AutoTokenizer.from_pretrained(tok_name)
-    if tok_name == MBART_TINY:
+    @slow
+    def test_seq2seq_dataset_truncation(self, tok_name):
+        tokenizer = AutoTokenizer.from_pretrained(tok_name)
+        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
+        max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
+        max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
+        max_src_len = 4
+        max_tgt_len = 8
+        assert max_len_target > max_src_len  # Will be truncated
+        assert max_len_source > max_src_len  # Will be truncated
+        src_lang, tgt_lang = "ro_RO", "de_DE"  # ignored for all but mbart, but never causes error.
         train_dataset = Seq2SeqDataset(
             tokenizer,
-            data_dir=make_test_data_dir(),
+            data_dir=tmp_dir,
             type_path="train",
-            max_source_length=4,
-            max_target_length=8,
-            src_lang="EN",
-            tgt_lang="FR",
+            max_source_length=max_src_len,
+            max_target_length=max_tgt_len,  # ignored
+            src_lang=src_lang,
+            tgt_lang=tgt_lang,
         )
-        kwargs = train_dataset.dataset_kwargs
-        assert "src_lang" in kwargs and "tgt_lang" in kwargs
-    else:
-        train_dataset = Seq2SeqDataset(
-            tokenizer, data_dir=make_test_data_dir(), type_path="train", max_source_length=4, max_target_length=8
+        dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
+        for batch in dataloader:
+            assert isinstance(batch, dict)
+            assert batch["attention_mask"].shape == batch["input_ids"].shape
+            # show that articles were trimmed.
+            assert batch["input_ids"].shape[1] == max_src_len
+            # show that targets are the same len
+            assert batch["labels"].shape[1] == max_tgt_len
+            if tok_name != MBART_TINY:
+                continue
+            # check language codes in correct place
+            batch["decoder_input_ids"] = shift_tokens_right(batch["labels"], tokenizer.pad_token_id)
+            assert batch["decoder_input_ids"][0, 0].item() == tokenizer.lang_code_to_id[tgt_lang]
+            assert batch["decoder_input_ids"][0, -1].item() == tokenizer.eos_token_id
+            assert batch["input_ids"][0, -2].item() == tokenizer.eos_token_id
+            assert batch["input_ids"][0, -1].item() == tokenizer.lang_code_to_id[src_lang]
+
+            break  # No need to test every batch
+
+    @parameterized.expand([BART_TINY, BERT_BASE_CASED])
+    def test_legacy_dataset_truncation(self, tok):
+        tokenizer = AutoTokenizer.from_pretrained(tok)
+        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
+        max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
+        max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
+        trunc_target = 4
+        train_dataset = LegacySeq2SeqDataset(
+            tokenizer,
+            data_dir=tmp_dir,
+            type_path="train",
+            max_source_length=20,
+            max_target_length=trunc_target,
         )
-        kwargs = train_dataset.dataset_kwargs
-        assert "add_prefix_space" not in kwargs if tok_name != BART_TINY else "add_prefix_space" in kwargs
-        assert len(kwargs) == 1 if tok_name == BART_TINY else len(kwargs) == 0
+        dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
+        for batch in dataloader:
+            assert batch["attention_mask"].shape == batch["input_ids"].shape
+            # show that articles were trimmed.
+            assert batch["input_ids"].shape[1] == max_len_source
+            assert 20 >= batch["input_ids"].shape[1]  # trimmed significantly
+            # show that targets were truncated
+            assert batch["labels"].shape[1] == trunc_target  # Truncated
+            assert max_len_target > trunc_target  # Truncated
+            break  # No need to test every batch
+
+    def test_pack_dataset(self):
+        tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25")
+
+        tmp_dir = Path(make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()))
+        orig_examples = tmp_dir.joinpath("train.source").open().readlines()
+        save_dir = Path(make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()))
+        pack_data_dir(tokenizer, tmp_dir, 128, save_dir)
+        orig_paths = {x.name for x in tmp_dir.iterdir()}
+        new_paths = {x.name for x in save_dir.iterdir()}
+        packed_examples = save_dir.joinpath("train.source").open().readlines()
+        # orig: [' Sam ate lunch today.\n', 'Sams lunch ingredients.']
+        # desired_packed: [' Sam ate lunch today.\n Sams lunch ingredients.']
+        assert len(packed_examples) < len(orig_examples)
+        assert len(packed_examples) == 1
+        assert len(packed_examples[0]) == sum(len(x) for x in orig_examples)
+        assert orig_paths == new_paths
+
+    @pytest.mark.skipif(not FAIRSEQ_AVAILABLE, reason="This test requires fairseq")
+    def test_dynamic_batch_size(self):
+        if not FAIRSEQ_AVAILABLE:
+            return
+        ds, max_tokens, tokenizer = self._get_dataset(max_len=64)
+        required_batch_size_multiple = 64
+        batch_sampler = ds.make_dynamic_sampler(max_tokens, required_batch_size_multiple=required_batch_size_multiple)
+        batch_sizes = [len(x) for x in batch_sampler]
+        assert len(set(batch_sizes)) > 1  # it's not dynamic batch size if every batch is the same length
+        assert sum(batch_sizes) == len(ds)  # no dropped or added examples
+        data_loader = DataLoader(ds, batch_sampler=batch_sampler, collate_fn=ds.collate_fn, num_workers=2)
+        failures = []
+        num_src_per_batch = []
+        for batch in data_loader:
+            src_shape = batch["input_ids"].shape
+            bs = src_shape[0]
+            assert bs % required_batch_size_multiple == 0 or bs < required_batch_size_multiple
+            num_src_tokens = np.product(batch["input_ids"].shape)
+            num_src_per_batch.append(num_src_tokens)
+            if num_src_tokens > (max_tokens * 1.1):
+                failures.append(num_src_tokens)
+        assert num_src_per_batch[0] == max(num_src_per_batch)
+        if failures:
+            raise AssertionError(f"too many tokens in {len(failures)} batches")
+
+    def test_sortish_sampler_reduces_padding(self):
+        ds, _, tokenizer = self._get_dataset(max_len=512)
+        bs = 2
+        sortish_sampler = ds.make_sortish_sampler(bs, shuffle=False)
+
+        naive_dl = DataLoader(ds, batch_size=bs, collate_fn=ds.collate_fn, num_workers=2)
+        sortish_dl = DataLoader(ds, batch_size=bs, collate_fn=ds.collate_fn, num_workers=2, sampler=sortish_sampler)
+
+        pad = tokenizer.pad_token_id
+
+        def count_pad_tokens(data_loader, k="input_ids"):
+            return [batch[k].eq(pad).sum().item() for batch in data_loader]
+
+        assert sum(count_pad_tokens(sortish_dl, k="labels")) < sum(count_pad_tokens(naive_dl, k="labels"))
+        assert sum(count_pad_tokens(sortish_dl)) < sum(count_pad_tokens(naive_dl))
+        assert len(sortish_dl) == len(naive_dl)
+
+    def _get_dataset(self, n_obs=1000, max_len=128):
+        if os.getenv("USE_REAL_DATA", False):
+            data_dir = "examples/seq2seq/wmt_en_ro"
+            max_tokens = max_len * 2 * 64
+            if not Path(data_dir).joinpath("train.len").exists():
+                save_len_file(MARIAN_TINY, data_dir)
+        else:
+            data_dir = "examples/seq2seq/test_data/wmt_en_ro"
+            max_tokens = max_len * 4
+            save_len_file(MARIAN_TINY, data_dir)
+
+        tokenizer = AutoTokenizer.from_pretrained(MARIAN_TINY)
+        ds = Seq2SeqDataset(
+            tokenizer,
+            data_dir=data_dir,
+            type_path="train",
+            max_source_length=max_len,
+            max_target_length=max_len,
+            n_obs=n_obs,
+        )
+        return ds, max_tokens, tokenizer
+
+    def test_distributed_sortish_sampler_splits_indices_between_procs(self):
+        ds, max_tokens, tokenizer = self._get_dataset()
+        ids1 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=0, add_extra_examples=False))
+        ids2 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=1, add_extra_examples=False))
+        assert ids1.intersection(ids2) == set()
+
+    @parameterized.expand(
+        [
+            MBART_TINY,
+            MARIAN_TINY,
+            T5_TINY,
+            BART_TINY,
+            PEGASUS_XSUM,
+        ],
+    )
+    def test_dataset_kwargs(self, tok_name):
+        tokenizer = AutoTokenizer.from_pretrained(tok_name)
+        if tok_name == MBART_TINY:
+            train_dataset = Seq2SeqDataset(
+                tokenizer,
+                data_dir=make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()),
+                type_path="train",
+                max_source_length=4,
+                max_target_length=8,
+                src_lang="EN",
+                tgt_lang="FR",
+            )
+            kwargs = train_dataset.dataset_kwargs
+            assert "src_lang" in kwargs and "tgt_lang" in kwargs
+        else:
+            train_dataset = Seq2SeqDataset(
+                tokenizer,
+                data_dir=make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()),
+                type_path="train",
+                max_source_length=4,
+                max_target_length=8,
+            )
+            kwargs = train_dataset.dataset_kwargs
+            assert "add_prefix_space" not in kwargs if tok_name != BART_TINY else "add_prefix_space" in kwargs
+            assert len(kwargs) == 1 if tok_name == BART_TINY else len(kwargs) == 0

examples/seq2seq/test_finetune_trainer.py

@@ -1,9 +1,8 @@
 import os
 import sys
-import tempfile
 from unittest.mock import patch
 
-from transformers.testing_utils import slow
+from transformers.testing_utils import TestCasePlus, slow
 from transformers.trainer_callback import TrainerState
 from transformers.trainer_utils import set_seed
 
@@ -15,93 +14,71 @@ set_seed(42)
 MARIAN_MODEL = "sshleifer/student_marian_en_ro_6_1"
 
 
-def test_finetune_trainer():
-    output_dir = run_trainer(1, "12", MBART_TINY, 1)
-    logs = TrainerState.load_from_json(os.path.join(output_dir, "trainer_state.json")).log_history
-    eval_metrics = [log for log in logs if "eval_loss" in log.keys()]
-    first_step_stats = eval_metrics[0]
-    assert "eval_bleu" in first_step_stats
+class TestFinetuneTrainer(TestCasePlus):
+    def test_finetune_trainer(self):
+        output_dir = self.run_trainer(1, "12", MBART_TINY, 1)
+        logs = TrainerState.load_from_json(os.path.join(output_dir, "trainer_state.json")).log_history
+        eval_metrics = [log for log in logs if "eval_loss" in log.keys()]
+        first_step_stats = eval_metrics[0]
+        assert "eval_bleu" in first_step_stats
 
+    @slow
+    def test_finetune_trainer_slow(self):
+        # There is a missing call to __init__process_group somewhere
+        output_dir = self.run_trainer(eval_steps=2, max_len="128", model_name=MARIAN_MODEL, num_train_epochs=3)
 
-@slow
-def test_finetune_trainer_slow():
-    # TODO(SS): This will fail on devices with more than 1 GPU.
-    # There is a missing call to __init__process_group somewhere
-    output_dir = run_trainer(eval_steps=2, max_len="128", model_name=MARIAN_MODEL, num_train_epochs=3)
+        # Check metrics
+        logs = TrainerState.load_from_json(os.path.join(output_dir, "trainer_state.json")).log_history
+        eval_metrics = [log for log in logs if "eval_loss" in log.keys()]
+        first_step_stats = eval_metrics[0]
+        last_step_stats = eval_metrics[-1]
 
-    # Check metrics
-    logs = TrainerState.load_from_json(os.path.join(output_dir, "trainer_state.json")).log_history
-    eval_metrics = [log for log in logs if "eval_loss" in log.keys()]
-    first_step_stats = eval_metrics[0]
-    last_step_stats = eval_metrics[-1]
+        assert first_step_stats["eval_bleu"] < last_step_stats["eval_bleu"]  # model learned nothing
+        assert isinstance(last_step_stats["eval_bleu"], float)
 
-    assert first_step_stats["eval_bleu"] < last_step_stats["eval_bleu"]  # model learned nothing
-    assert isinstance(last_step_stats["eval_bleu"], float)
+        # test if do_predict saves generations and metrics
+        contents = os.listdir(output_dir)
+        contents = {os.path.basename(p) for p in contents}
+        assert "test_generations.txt" in contents
+        assert "test_results.json" in contents
 
-    # test if do_predict saves generations and metrics
-    contents = os.listdir(output_dir)
-    contents = {os.path.basename(p) for p in contents}
-    assert "test_generations.txt" in contents
-    assert "test_results.json" in contents
+    def run_trainer(self, eval_steps: int, max_len: str, model_name: str, num_train_epochs: int):
+        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
+        output_dir = self.get_auto_remove_tmp_dir()
+        argv = f"""
+            --model_name_or_path {model_name}
+            --data_dir {data_dir}
+            --output_dir {output_dir}
+            --overwrite_output_dir
+            --n_train 8
+            --n_val 8
+            --max_source_length {max_len}
+            --max_target_length {max_len}
+            --val_max_target_length {max_len}
+            --do_train
+            --do_eval
+            --do_predict
+            --num_train_epochs {str(num_train_epochs)}
+            --per_device_train_batch_size 4
+            --per_device_eval_batch_size 4
+            --learning_rate 3e-4
+            --warmup_steps 8
+            --evaluate_during_training
+            --predict_with_generate
+            --logging_steps 0
+            --save_steps {str(eval_steps)}
+            --eval_steps {str(eval_steps)}
+            --sortish_sampler
+            --label_smoothing 0.1
+            --adafactor
+            --task translation
+            --tgt_lang ro_RO
+            --src_lang en_XX
+        """.split()
+        # --eval_beams  2
 
+        testargs = ["finetune_trainer.py"] + argv
+        with patch.object(sys, "argv", testargs):
+            main()
 
-def run_trainer(eval_steps: int, max_len: str, model_name: str, num_train_epochs: int):
-    data_dir = "examples/seq2seq/test_data/wmt_en_ro"
-    output_dir = tempfile.mkdtemp(prefix="test_output")
-    argv = [
-        "--model_name_or_path",
-        model_name,
-        "--data_dir",
-        data_dir,
-        "--output_dir",
-        output_dir,
-        "--overwrite_output_dir",
-        "--n_train",
-        "8",
-        "--n_val",
-        "8",
-        "--max_source_length",
-        max_len,
-        "--max_target_length",
-        max_len,
-        "--val_max_target_length",
-        max_len,
-        "--do_train",
-        "--do_eval",
-        "--do_predict",
-        "--num_train_epochs",
-        str(num_train_epochs),
-        "--per_device_train_batch_size",
-        "4",
-        "--per_device_eval_batch_size",
-        "4",
-        "--learning_rate",
-        "3e-4",
-        "--warmup_steps",
-        "8",
-        "--evaluate_during_training",
-        "--predict_with_generate",
-        "--logging_steps",
-        0,
-        "--save_steps",
-        str(eval_steps),
-        "--eval_steps",
-        str(eval_steps),
-        "--sortish_sampler",
-        "--label_smoothing",
-        "0.1",
-        # "--eval_beams",
-        # "2",
-        "--adafactor",
-        "--task",
-        "translation",
-        "--tgt_lang",
-        "ro_RO",
-        "--src_lang",
-        "en_XX",
-    ]
-    testargs = ["finetune_trainer.py"] + argv
-    with patch.object(sys, "argv", testargs):
-        main()
-
-    return output_dir
+        return output_dir

examples/seq2seq/test_seq2seq_examples.py

@@ -3,7 +3,6 @@ import logging
 import os
 import sys
 import tempfile
-import unittest
 from pathlib import Path
 from unittest.mock import patch
 
@@ -13,13 +12,14 @@ import torch
 
 import lightning_base
 from convert_pl_checkpoint_to_hf import convert_pl_to_hf
-from distillation import distill_main, evaluate_checkpoint
+from distillation import distill_main
 from finetune import SummarizationModule, main
+from parameterized import parameterized
 from run_eval import generate_summaries_or_translations, run_generate
 from run_eval_search import run_search
 from transformers import AutoConfig, AutoModelForSeq2SeqLM
 from transformers.hf_api import HfApi
-from transformers.testing_utils import CaptureStderr, CaptureStdout, require_multigpu, require_torch_and_cuda, slow
+from transformers.testing_utils import CaptureStderr, CaptureStdout, TestCasePlus, require_torch_gpu, slow
 from utils import ROUGE_KEYS, label_smoothed_nll_loss, lmap, load_json
 
 
@@ -88,6 +88,7 @@ CHEAP_ARGS = {
     "student_encoder_layers": 1,
     "freeze_encoder": False,
     "auto_scale_batch_size": False,
+    "overwrite_output_dir": False,
 }
 
 
@@ -110,24 +111,23 @@ logger.addHandler(stream_handler)
 logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
 
 
-def make_test_data_dir(**kwargs):
-    tmp_dir = Path(tempfile.mkdtemp(**kwargs))
+def make_test_data_dir(tmp_dir):
     for split in ["train", "val", "test"]:
-        _dump_articles((tmp_dir / f"{split}.source"), ARTICLES)
-        _dump_articles((tmp_dir / f"{split}.target"), SUMMARIES)
+        _dump_articles(os.path.join(tmp_dir, f"{split}.source"), ARTICLES)
+        _dump_articles(os.path.join(tmp_dir, f"{split}.target"), SUMMARIES)
     return tmp_dir
 
 
-class TestSummarizationDistiller(unittest.TestCase):
+class TestSummarizationDistiller(TestCasePlus):
     @classmethod
     def setUpClass(cls):
         logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
         return cls
 
     @slow
-    @require_torch_and_cuda
+    @require_torch_gpu
     def test_hub_configs(self):
-        """I put require_torch_and_cuda cause I only want this to run with self-scheduled."""
+        """I put require_torch_gpu cause I only want this to run with self-scheduled."""
 
         model_list = HfApi().model_list()
         org = "sshleifer"
@@ -143,17 +143,6 @@ class TestSummarizationDistiller(unittest.TestCase):
                 failures.append(m)
         assert not failures, f"The following models could not be loaded through AutoConfig: {failures}"
 
-    @require_multigpu
-    @unittest.skip("Broken at the moment")
-    def test_multigpu(self):
-        updates = dict(
-            no_teacher=True,
-            freeze_encoder=True,
-            gpus=2,
-            sortish_sampler=True,
-        )
-        self._test_distiller_cli(updates, check_contents=False)
-
     def test_distill_no_teacher(self):
         updates = dict(student_encoder_layers=2, student_decoder_layers=1, no_teacher=True)
         self._test_distiller_cli(updates)
@@ -173,13 +162,12 @@ class TestSummarizationDistiller(unittest.TestCase):
         self.assertEqual(1, len(ckpts))
         transformer_ckpts = list(Path(model.output_dir).glob("**/*.bin"))
         self.assertEqual(len(transformer_ckpts), 2)
-        examples = lmap(str.strip, model.hparams.data_dir.joinpath("test.source").open().readlines())
-        out_path = tempfile.mktemp()
+        examples = lmap(str.strip, Path(model.hparams.data_dir).joinpath("test.source").open().readlines())
+        out_path = tempfile.mktemp()  # XXX: not being cleaned up
         generate_summaries_or_translations(examples, out_path, str(model.output_dir / "best_tfmr"))
         self.assertTrue(Path(out_path).exists())
 
-        evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
-        out_path_new = tempfile.mkdtemp()
+        out_path_new = self.get_auto_remove_tmp_dir()
         convert_pl_to_hf(ckpts[0], transformer_ckpts[0].parent, out_path_new)
         assert os.path.exists(os.path.join(out_path_new, "pytorch_model.bin"))
 
@@ -227,8 +215,6 @@ class TestSummarizationDistiller(unittest.TestCase):
         assert len(all_files) > 2
         self.assertEqual(len(transformer_ckpts), 2)
 
-        evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
-
     def test_distill_t5(self):
         updates = dict(
             student_encoder_layers=1,
@@ -256,8 +242,8 @@ class TestSummarizationDistiller(unittest.TestCase):
         )
         default_updates.update(updates)
         args_d: dict = CHEAP_ARGS.copy()
-        tmp_dir = make_test_data_dir()
-        output_dir = tempfile.mkdtemp(prefix="output_")
+        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
+        output_dir = self.get_auto_remove_tmp_dir()
 
         args_d.update(data_dir=tmp_dir, output_dir=output_dir, **default_updates)
         model = distill_main(argparse.Namespace(**args_d))
@@ -282,256 +268,253 @@ class TestSummarizationDistiller(unittest.TestCase):
         return model
 
 
-def run_eval_tester(model):
-    input_file_name = Path(tempfile.mkdtemp()) / "utest_input.source"
-    output_file_name = input_file_name.parent / "utest_output.txt"
-    assert not output_file_name.exists()
-    articles = [" New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County."]
-    _dump_articles(input_file_name, articles)
-    score_path = str(Path(tempfile.mkdtemp()) / "scores.json")
-    task = "translation_en_to_de" if model == T5_TINY else "summarization"
-    testargs = f"""
-        run_eval_search.py
-        {model}
-        {input_file_name}
-        {output_file_name}
-        --score_path {score_path}
-        --task {task}
-        --num_beams 2
-        --length_penalty 2.0
-        """.split()
+class TestTheRest(TestCasePlus):
+    def run_eval_tester(self, model):
+        input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
+        output_file_name = input_file_name.parent / "utest_output.txt"
+        assert not output_file_name.exists()
+        articles = [" New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County."]
+        _dump_articles(input_file_name, articles)
 
-    with patch.object(sys, "argv", testargs):
-        run_generate()
-        assert Path(output_file_name).exists()
-        os.remove(Path(output_file_name))
+        score_path = str(Path(self.get_auto_remove_tmp_dir()) / "scores.json")
+        task = "translation_en_to_de" if model == T5_TINY else "summarization"
+        testargs = f"""
+            run_eval_search.py
+            {model}
+            {input_file_name}
+            {output_file_name}
+            --score_path {score_path}
+            --task {task}
+            --num_beams 2
+            --length_penalty 2.0
+            """.split()
 
+        with patch.object(sys, "argv", testargs):
+            run_generate()
+            assert Path(output_file_name).exists()
+            # os.remove(Path(output_file_name))
 
-# test one model to quickly (no-@slow) catch simple problems and do an
-# extensive testing of functionality with multiple models as @slow separately
-def test_run_eval():
-    run_eval_tester(T5_TINY)
+    # test one model to quickly (no-@slow) catch simple problems and do an
+    # extensive testing of functionality with multiple models as @slow separately
+    def test_run_eval(self):
+        self.run_eval_tester(T5_TINY)
 
+    # any extra models should go into the list here - can be slow
+    @parameterized.expand([BART_TINY, MBART_TINY])
+    @slow
+    def test_run_eval_slow(self, model):
+        self.run_eval_tester(model)
 
-# any extra models should go into the list here - can be slow
-@slow
-@pytest.mark.parametrize("model", [BART_TINY, MBART_TINY])
-def test_run_eval_slow(model):
-    run_eval_tester(model)
+    # testing with 2 models to validate: 1. translation (t5) 2. summarization (mbart)
+    @parameterized.expand([T5_TINY, MBART_TINY])
+    @slow
+    def test_run_eval_search(self, model):
+        input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
+        output_file_name = input_file_name.parent / "utest_output.txt"
+        assert not output_file_name.exists()
 
+        text = {
+            "en": ["Machine learning is great, isn't it?", "I like to eat bananas", "Tomorrow is another great day!"],
+            "de": [
+                "Maschinelles Lernen ist großartig, oder?",
+                "Ich esse gerne Bananen",
+                "Morgen ist wieder ein toller Tag!",
+            ],
+        }
 
-# testing with 2 models to validate: 1. translation (t5) 2. summarization (mbart)
-@slow
-@pytest.mark.parametrize("model", [T5_TINY, MBART_TINY])
-def test_run_eval_search(model):
-    input_file_name = Path(tempfile.mkdtemp()) / "utest_input.source"
-    output_file_name = input_file_name.parent / "utest_output.txt"
-    assert not output_file_name.exists()
+        tmp_dir = Path(self.get_auto_remove_tmp_dir())
+        score_path = str(tmp_dir / "scores.json")
+        reference_path = str(tmp_dir / "val.target")
+        _dump_articles(input_file_name, text["en"])
+        _dump_articles(reference_path, text["de"])
+        task = "translation_en_to_de" if model == T5_TINY else "summarization"
+        testargs = f"""
+            run_eval_search.py
+            {model}
+            {str(input_file_name)}
+            {str(output_file_name)}
+            --score_path {score_path}
+            --reference_path {reference_path}
+            --task {task}
+            """.split()
+        testargs.extend(["--search", "num_beams=1:2 length_penalty=0.9:1.0"])
 
-    text = {
-        "en": ["Machine learning is great, isn't it?", "I like to eat bananas", "Tomorrow is another great day!"],
-        "de": [
-            "Maschinelles Lernen ist großartig, oder?",
-            "Ich esse gerne Bananen",
-            "Morgen ist wieder ein toller Tag!",
-        ],
-    }
+        with patch.object(sys, "argv", testargs):
+            with CaptureStdout() as cs:
+                run_search()
+            expected_strings = [" num_beams | length_penalty", model, "Best score args"]
+            un_expected_strings = ["Info"]
+            if "translation" in task:
+                expected_strings.append("bleu")
+            else:
+                expected_strings.extend(ROUGE_KEYS)
+            for w in expected_strings:
+                assert w in cs.out
+            for w in un_expected_strings:
+                assert w not in cs.out
+            assert Path(output_file_name).exists()
+            os.remove(Path(output_file_name))
 
-    tmp_dir = Path(tempfile.mkdtemp())
-    score_path = str(tmp_dir / "scores.json")
-    reference_path = str(tmp_dir / "val.target")
-    _dump_articles(input_file_name, text["en"])
-    _dump_articles(reference_path, text["de"])
-    task = "translation_en_to_de" if model == T5_TINY else "summarization"
-    testargs = f"""
-        run_eval_search.py
-        {model}
-        {str(input_file_name)}
-        {str(output_file_name)}
-        --score_path {score_path}
-        --reference_path {reference_path}
-        --task {task}
-        """.split()
-    testargs.extend(["--search", "num_beams=1:2 length_penalty=0.9:1.0"])
+    @parameterized.expand(
+        [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY],
+    )
+    def test_finetune(self, model):
+        args_d: dict = CHEAP_ARGS.copy()
+        task = "translation" if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else "summarization"
+        args_d["label_smoothing"] = 0.1 if task == "translation" else 0
 
-    with patch.object(sys, "argv", testargs):
-        with CaptureStdout() as cs:
-            run_search()
-        expected_strings = [" num_beams | length_penalty", model, "Best score args"]
-        un_expected_strings = ["Info"]
-        if "translation" in task:
-            expected_strings.append("bleu")
+        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
+        output_dir = self.get_auto_remove_tmp_dir()
+        args_d.update(
+            data_dir=tmp_dir,
+            model_name_or_path=model,
+            tokenizer_name=None,
+            train_batch_size=2,
+            eval_batch_size=2,
+            output_dir=output_dir,
+            do_predict=True,
+            task=task,
+            src_lang="en_XX",
+            tgt_lang="ro_RO",
+            freeze_encoder=True,
+            freeze_embeds=True,
+        )
+        assert "n_train" in args_d
+        args = argparse.Namespace(**args_d)
+        module = main(args)
+
+        input_embeds = module.model.get_input_embeddings()
+        assert not input_embeds.weight.requires_grad
+        if model == T5_TINY:
+            lm_head = module.model.lm_head
+            assert not lm_head.weight.requires_grad
+            assert (lm_head.weight == input_embeds.weight).all().item()
+        elif model == FSMT_TINY:
+            fsmt = module.model.model
+            embed_pos = fsmt.decoder.embed_positions
+            assert not embed_pos.weight.requires_grad
+            assert not fsmt.decoder.embed_tokens.weight.requires_grad
+            # check that embeds are not the same
+            assert fsmt.decoder.embed_tokens != fsmt.encoder.embed_tokens
         else:
-            expected_strings.extend(ROUGE_KEYS)
-        for w in expected_strings:
-            assert w in cs.out
-        for w in un_expected_strings:
-            assert w not in cs.out
-        assert Path(output_file_name).exists()
-        os.remove(Path(output_file_name))
+            bart = module.model.model
+            embed_pos = bart.decoder.embed_positions
+            assert not embed_pos.weight.requires_grad
+            assert not bart.shared.weight.requires_grad
+            # check that embeds are the same
+            assert bart.decoder.embed_tokens == bart.encoder.embed_tokens
+            assert bart.decoder.embed_tokens == bart.shared
 
+        example_batch = load_json(module.output_dir / "text_batch.json")
+        assert isinstance(example_batch, dict)
+        assert len(example_batch) >= 4
 
-@pytest.mark.parametrize(
-    "model",
-    [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY],
-)
-def test_finetune(model):
-    args_d: dict = CHEAP_ARGS.copy()
-    task = "translation" if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else "summarization"
-    args_d["label_smoothing"] = 0.1 if task == "translation" else 0
+    def test_finetune_extra_model_args(self):
+        args_d: dict = CHEAP_ARGS.copy()
 
-    tmp_dir = make_test_data_dir()
-    output_dir = tempfile.mkdtemp(prefix="output_")
-    args_d.update(
-        data_dir=tmp_dir,
-        model_name_or_path=model,
-        tokenizer_name=None,
-        train_batch_size=2,
-        eval_batch_size=2,
-        output_dir=output_dir,
-        do_predict=True,
-        task=task,
-        src_lang="en_XX",
-        tgt_lang="ro_RO",
-        freeze_encoder=True,
-        freeze_embeds=True,
-    )
-    assert "n_train" in args_d
-    args = argparse.Namespace(**args_d)
-    module = main(args)
+        task = "summarization"
+        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
 
-    input_embeds = module.model.get_input_embeddings()
-    assert not input_embeds.weight.requires_grad
-    if model == T5_TINY:
-        lm_head = module.model.lm_head
-        assert not lm_head.weight.requires_grad
-        assert (lm_head.weight == input_embeds.weight).all().item()
-    elif model == FSMT_TINY:
-        fsmt = module.model.model
-        embed_pos = fsmt.decoder.embed_positions
-        assert not embed_pos.weight.requires_grad
-        assert not fsmt.decoder.embed_tokens.weight.requires_grad
-        # check that embeds are not the same
-        assert fsmt.decoder.embed_tokens != fsmt.encoder.embed_tokens
-    else:
-        bart = module.model.model
-        embed_pos = bart.decoder.embed_positions
-        assert not embed_pos.weight.requires_grad
-        assert not bart.shared.weight.requires_grad
-        # check that embeds are the same
-        assert bart.decoder.embed_tokens == bart.encoder.embed_tokens
-        assert bart.decoder.embed_tokens == bart.shared
+        args_d.update(
+            data_dir=tmp_dir,
+            tokenizer_name=None,
+            train_batch_size=2,
+            eval_batch_size=2,
+            do_predict=False,
+            task=task,
+            src_lang="en_XX",
+            tgt_lang="ro_RO",
+            freeze_encoder=True,
+            freeze_embeds=True,
+        )
 
-    example_batch = load_json(module.output_dir / "text_batch.json")
-    assert isinstance(example_batch, dict)
-    assert len(example_batch) >= 4
-
-
-def test_finetune_extra_model_args():
-    args_d: dict = CHEAP_ARGS.copy()
-
-    task = "summarization"
-    tmp_dir = make_test_data_dir()
-
-    args_d.update(
-        data_dir=tmp_dir,
-        tokenizer_name=None,
-        train_batch_size=2,
-        eval_batch_size=2,
-        do_predict=False,
-        task=task,
-        src_lang="en_XX",
-        tgt_lang="ro_RO",
-        freeze_encoder=True,
-        freeze_embeds=True,
-    )
-
-    # test models whose config includes the extra_model_args
-    model = BART_TINY
-    output_dir = tempfile.mkdtemp(prefix="output_1_")
-    args_d1 = args_d.copy()
-    args_d1.update(
-        model_name_or_path=model,
-        output_dir=output_dir,
-    )
-    extra_model_params = ("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
-    for p in extra_model_params:
-        args_d1[p] = 0.5
-    args = argparse.Namespace(**args_d1)
-    model = main(args)
-    for p in extra_model_params:
-        assert getattr(model.config, p) == 0.5, f"failed to override the model config for param {p}"
-
-    # test models whose config doesn't include the extra_model_args
-    model = T5_TINY
-    output_dir = tempfile.mkdtemp(prefix="output_2_")
-    args_d2 = args_d.copy()
-    args_d2.update(
-        model_name_or_path=model,
-        output_dir=output_dir,
-    )
-    unsupported_param = "encoder_layerdrop"
-    args_d2[unsupported_param] = 0.5
-    args = argparse.Namespace(**args_d2)
-    with pytest.raises(Exception) as excinfo:
+        # test models whose config includes the extra_model_args
+        model = BART_TINY
+        output_dir = self.get_auto_remove_tmp_dir()
+        args_d1 = args_d.copy()
+        args_d1.update(
+            model_name_or_path=model,
+            output_dir=output_dir,
+        )
+        extra_model_params = ("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
+        for p in extra_model_params:
+            args_d1[p] = 0.5
+        args = argparse.Namespace(**args_d1)
         model = main(args)
-    assert str(excinfo.value) == f"model config doesn't have a `{unsupported_param}` attribute"
+        for p in extra_model_params:
+            assert getattr(model.config, p) == 0.5, f"failed to override the model config for param {p}"
 
+        # test models whose config doesn't include the extra_model_args
+        model = T5_TINY
+        output_dir = self.get_auto_remove_tmp_dir()
+        args_d2 = args_d.copy()
+        args_d2.update(
+            model_name_or_path=model,
+            output_dir=output_dir,
+        )
+        unsupported_param = "encoder_layerdrop"
+        args_d2[unsupported_param] = 0.5
+        args = argparse.Namespace(**args_d2)
+        with pytest.raises(Exception) as excinfo:
+            model = main(args)
+        assert str(excinfo.value) == f"model config doesn't have a `{unsupported_param}` attribute"
 
-def test_finetune_lr_schedulers():
-    args_d: dict = CHEAP_ARGS.copy()
+    def test_finetune_lr_schedulers(self):
+        args_d: dict = CHEAP_ARGS.copy()
 
-    task = "summarization"
-    tmp_dir = make_test_data_dir()
+        task = "summarization"
+        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
 
-    model = BART_TINY
-    output_dir = tempfile.mkdtemp(prefix="output_1_")
+        model = BART_TINY
+        output_dir = self.get_auto_remove_tmp_dir()
 
-    args_d.update(
-        data_dir=tmp_dir,
-        model_name_or_path=model,
-        output_dir=output_dir,
-        tokenizer_name=None,
-        train_batch_size=2,
-        eval_batch_size=2,
-        do_predict=False,
-        task=task,
-        src_lang="en_XX",
-        tgt_lang="ro_RO",
-        freeze_encoder=True,
-        freeze_embeds=True,
-    )
+        args_d.update(
+            data_dir=tmp_dir,
+            model_name_or_path=model,
+            output_dir=output_dir,
+            tokenizer_name=None,
+            train_batch_size=2,
+            eval_batch_size=2,
+            do_predict=False,
+            task=task,
+            src_lang="en_XX",
+            tgt_lang="ro_RO",
+            freeze_encoder=True,
+            freeze_embeds=True,
+        )
 
-    # emulate finetune.py
-    parser = argparse.ArgumentParser()
-    parser = pl.Trainer.add_argparse_args(parser)
-    parser = SummarizationModule.add_model_specific_args(parser, os.getcwd())
-    args = {"--help": True}
+        # emulate finetune.py
+        parser = argparse.ArgumentParser()
+        parser = pl.Trainer.add_argparse_args(parser)
+        parser = SummarizationModule.add_model_specific_args(parser, os.getcwd())
+        args = {"--help": True}
 
-    # --help test
-    with pytest.raises(SystemExit) as excinfo:
-        with CaptureStdout() as cs:
-            args = parser.parse_args(args)
-        assert False, "--help is expected to sys.exit"
-    assert excinfo.type == SystemExit
-    expected = lightning_base.arg_to_scheduler_metavar
-    assert expected in cs.out, "--help is expected to list the supported schedulers"
+        # --help test
+        with pytest.raises(SystemExit) as excinfo:
+            with CaptureStdout() as cs:
+                args = parser.parse_args(args)
+            assert False, "--help is expected to sys.exit"
+        assert excinfo.type == SystemExit
+        expected = lightning_base.arg_to_scheduler_metavar
+        assert expected in cs.out, "--help is expected to list the supported schedulers"
 
-    # --lr_scheduler=non_existing_scheduler test
-    unsupported_param = "non_existing_scheduler"
-    args = {f"--lr_scheduler={unsupported_param}"}
-    with pytest.raises(SystemExit) as excinfo:
-        with CaptureStderr() as cs:
-            args = parser.parse_args(args)
-        assert False, "invalid argument is expected to sys.exit"
-    assert excinfo.type == SystemExit
-    expected = f"invalid choice: '{unsupported_param}'"
-    assert expected in cs.err, f"should have bailed on invalid choice of scheduler {unsupported_param}"
+        # --lr_scheduler=non_existing_scheduler test
+        unsupported_param = "non_existing_scheduler"
+        args = {f"--lr_scheduler={unsupported_param}"}
+        with pytest.raises(SystemExit) as excinfo:
+            with CaptureStderr() as cs:
+                args = parser.parse_args(args)
+            assert False, "invalid argument is expected to sys.exit"
+        assert excinfo.type == SystemExit
+        expected = f"invalid choice: '{unsupported_param}'"
+        assert expected in cs.err, f"should have bailed on invalid choice of scheduler {unsupported_param}"
 
-    # --lr_scheduler=existing_scheduler test
-    supported_param = "cosine"
-    args_d1 = args_d.copy()
-    args_d1["lr_scheduler"] = supported_param
-    args = argparse.Namespace(**args_d1)
-    model = main(args)
-    assert getattr(model.hparams, "lr_scheduler") == supported_param, f"lr_scheduler={supported_param} shouldn't fail"
+        # --lr_scheduler=existing_scheduler test
+        supported_param = "cosine"
+        args_d1 = args_d.copy()
+        args_d1["lr_scheduler"] = supported_param
+        args = argparse.Namespace(**args_d1)
+        model = main(args)
+        assert (
+            getattr(model.hparams, "lr_scheduler") == supported_param
+        ), f"lr_scheduler={supported_param} shouldn't fail"

examples/seq2seq/test_tatoeba_conversion.py

@@ -0,0 +1,22 @@
+import tempfile
+import unittest
+
+from transformers.convert_marian_tatoeba_to_pytorch import TatoebaConverter
+from transformers.file_utils import cached_property
+from transformers.testing_utils import slow
+
+
+class TatoebaConversionTester(unittest.TestCase):
+    @cached_property
+    def resolver(self):
+        tmp_dir = tempfile.mkdtemp()
+        return TatoebaConverter(save_dir=tmp_dir)
+
+    @slow
+    def test_resolver(self):
+        self.resolver.convert_models(["heb-eng"])
+
+    @slow
+    def test_model_card(self):
+        content, mmeta = self.resolver.write_model_card("opus-mt-he-en", dry_run=True)
+        assert mmeta["long_pair"] == "heb-eng"

examples/seq2seq/utils.py

@@ -457,14 +457,22 @@ def load_json(path):
 
 
 def get_git_info():
-    repo = git.Repo(search_parent_directories=True)
-    repo_infos = {
-        "repo_id": str(repo),
-        "repo_sha": str(repo.head.object.hexsha),
-        "repo_branch": str(repo.active_branch),
-        "hostname": str(socket.gethostname()),
-    }
-    return repo_infos
+    try:
+        repo = git.Repo(search_parent_directories=True)
+        repo_infos = {
+            "repo_id": str(repo),
+            "repo_sha": str(repo.head.object.hexsha),
+            "repo_branch": str(repo.active_branch),
+            "hostname": str(socket.gethostname()),
+        }
+        return repo_infos
+    except TypeError:
+        return {
+            "repo_id": None,
+            "repo_sha": None,
+            "repo_branch": None,
+            "hostname": None,
+        }
 
 
 ROUGE_KEYS = ["rouge1", "rouge2", "rougeL", "rougeLsum"]

examples/test_examples.py

@@ -116,8 +116,8 @@ class ExamplesTests(TestCasePlus):
             testargs.append("--fp16")
 
         with patch.object(sys, "argv", testargs):
-            result = run_pl_glue.main()
-            # for now just testing that the script can run to a completion
+            result = run_pl_glue.main()[0]
+            # for now just testing that the script can run to completion
             self.assertGreater(result["acc"], 0.25)
             #
             # TODO: this fails on CI - doesn't get acc/f1>=0.75:

examples/test_xla_examples.py

@@ -59,7 +59,7 @@ class TorchXLAExamplesTests(unittest.TestCase):
             --model_name_or_path=bert-base-cased
             --per_device_train_batch_size=64
             --per_device_eval_batch_size=64
-            --evaluate_during_training
+            --evaluation_strategy steps
             --overwrite_cache
             """.split()
         with patch.object(sys, "argv", testargs):
@@ -80,4 +80,15 @@ class TorchXLAExamplesTests(unittest.TestCase):
                 self.assertGreaterEqual(value, 0.70)
 
             # Assert that the script takes less than 300 seconds to make sure it doesn't hang.
-            self.assertLess(end - start, 300)
+            self.assertLess(end - start, 500)
+
+    def test_trainer_tpu(self):
+        import xla_spawn
+
+        testargs = """
+            transformers/tests/test_trainer_tpu.py
+            --num_cores=8
+            transformers/tests/test_trainer_tpu.py
+            """.split()
+        with patch.object(sys, "argv", testargs):
+            xla_spawn.main()

examples/text-classification/README.md

@@ -43,7 +43,7 @@ python run_tf_text_classification.py \
   --do_eval \
   --do_predict \
   --logging_steps 10 \
-  --evaluate_during_training \
+  --evaluation_strategy steps \
   --save_steps 10 \
   --overwrite_output_dir \
   --max_seq_length 128

examples/text-classification/run_tf_text_classification.py

@@ -60,7 +60,7 @@ def get_tfds(
         for k in files.keys():
             transformed_ds[k] = ds[k].map(
                 lambda example: tokenizer.batch_encode_plus(
-                    (example[features_name[0]], features_name[1]),
+                    (example[features_name[0]], example[features_name[1]]),
                     truncation=True,
                     max_length=max_seq_length,
                     padding="max_length",
@@ -96,6 +96,9 @@ def get_tfds(
         else None
     )
 
+    if train_ds is not None:
+        train_ds = train_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TRAIN])))
+
     val_ds = (
         tf.data.Dataset.from_generator(
             gen_val,
@@ -106,6 +109,9 @@ def get_tfds(
         else None
     )
 
+    if val_ds is not None:
+        val_ds = val_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.VALIDATION])))
+
     test_ds = (
         tf.data.Dataset.from_generator(
             gen_test,
@@ -116,6 +122,9 @@ def get_tfds(
         else None
     )
 
+    if test_ds is not None:
+        test_ds = test_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TEST])))
+
     return train_ds, val_ds, test_ds, label2id
 
 

model_cards/allegro/herbert-base-cased/README.md

@@ -0,0 +1,51 @@
+---
+language: pl
+tags:
+- herbert
+license: cc-by-sa-4.0
+---
+
+# HerBERT 
+**[HerBERT](https://en.wikipedia.org/wiki/Zbigniew_Herbert)** is a BERT-based Language Model trained on Polish Corpora
+using MLM and SSO objectives with dynamic masking of whole words.
+Model training and experiments were conducted with [transformers](https://github.com/huggingface/transformers) in version 2.9.
+
+## Tokenizer
+The training dataset was tokenized into subwords using ``CharBPETokenizer`` a character level byte-pair encoding with
+a vocabulary size of 50k tokens. The tokenizer itself was trained with a [tokenizers](https://github.com/huggingface/tokenizers) library. 
+We kindly encourage you to use the **Fast** version of tokenizer, namely ``HerbertTokenizerFast``.
+
+## HerBERT usage
+
+
+Example code:
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("allegro/herbert-base-cased")
+model = AutoModel.from_pretrained("allegro/herbert-base-cased")
+
+output = model(
+    **tokenizer.batch_encode_plus(
+        [
+            (
+                "A potem szedł środkiem drogi w kurzawie, bo zamiatał nogami, ślepy dziad prowadzony przez tłustego kundla na sznurku.",
+                "A potem leciał od lasu chłopak z butelką, ale ten ujrzawszy księdza przy drodze okrążył go z dala i biegł na przełaj pól do karczmy."
+            )
+        ],
+    padding='longest',
+    add_special_tokens=True,
+    return_tensors='pt'
+    )
+)
+```
+
+
+## License
+CC BY-SA 4.0
+
+
+## Authors
+Model was trained by **Allegro Machine Learning Research** team.
+
+You can contact us at: <a href="mailto:klejbenchmark@allegro.pl">klejbenchmark@allegro.pl</a>

model_cards/allegro/herbert-large-cased/README.md

@@ -0,0 +1,50 @@
+---
+language: pl
+tags:
+- herbert
+license: cc-by-sa-4.0
+---
+# HerBERT 
+**[HerBERT](https://en.wikipedia.org/wiki/Zbigniew_Herbert)** is a BERT-based Language Model trained on Polish Corpora
+using MLM and SSO objectives with dynamic masking of whole words.
+Model training and experiments were conducted with [transformers](https://github.com/huggingface/transformers) in version 2.9.
+
+## Tokenizer
+The training dataset was tokenized into subwords using ``CharBPETokenizer`` a character level byte-pair encoding with
+a vocabulary size of 50k tokens. The tokenizer itself was trained with a [tokenizers](https://github.com/huggingface/tokenizers) library. 
+We kindly encourage you to use the **Fast** version of tokenizer, namely ``HerbertTokenizerFast``.
+
+## HerBERT usage
+
+
+Example code:
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("allegro/herbert-large-cased")
+model = AutoModel.from_pretrained("allegro/herbert-large-cased")
+
+output = model(
+    **tokenizer.batch_encode_plus(
+        [
+            (
+                "A potem szedł środkiem drogi w kurzawie, bo zamiatał nogami, ślepy dziad prowadzony przez tłustego kundla na sznurku.",
+                "A potem leciał od lasu chłopak z butelką, ale ten ujrzawszy księdza przy drodze okrążył go z dala i biegł na przełaj pól do karczmy."
+            )
+        ],
+    padding='longest',
+    add_special_tokens=True,
+    return_tensors='pt'
+    )
+)
+```
+
+
+## License
+CC BY-SA 4.0
+
+
+## Authors
+Model was trained by **Allegro Machine Learning Research** team.
+
+You can contact us at: <a href="mailto:klejbenchmark@allegro.pl">klejbenchmark@allegro.pl</a>

model_cards/cooelf/limitbert/README.md

@@ -0,0 +1,59 @@
+# LIMIT-BERT
+
+Code and model for the *EMNLP 2020 Findings* paper: 
+
+[LIMIT-BERT: Linguistic Informed Multi-task BERT](https://arxiv.org/abs/1910.14296)) 
+
+## Contents
+
+1. [Requirements](#Requirements)
+2. [Training](#Training)
+
+## Requirements
+
+* Python 3.6 or higher.
+* Cython 0.25.2 or any compatible version.
+* [PyTorch](http://pytorch.org/) 1.0.0+. 
+* [EVALB](http://nlp.cs.nyu.edu/evalb/). Before starting, run `make` inside the `EVALB/` directory to compile an `evalb` executable. This will be called from Python for evaluation. 
+* [pytorch-transformers](https://github.com/huggingface/pytorch-transformers) PyTorch 1.0.0+ or any compatible version.
+
+#### Pre-trained Models (PyTorch)
+The following pre-trained models are available for download from Google Drive:
+* [`LIMIT-BERT`](https://drive.google.com/open?id=1fm0cK2A91iLG3lCpwowCCQSALnWS2X4i): 
+  PyTorch version, same setting with BERT-Large-WWM，loading model with [pytorch-transformers](https://github.com/huggingface/pytorch-transformers).
+
+## How to use
+
+```
+from transformers import AutoTokenizer, AutoModel
+tokenizer = AutoTokenizer.from_pretrained("cooelf/limitbert")
+model = AutoModel.from_pretrained("cooelf/limitbert")
+```
+
+Please see our original repo for the training scripts.
+
+https://github.com/cooelf/LIMIT-BERT
+
+## Training
+
+To train LIMIT-BERT, simply run:
+```
+sh run_limitbert.sh
+```
+### Evaluation Instructions
+
+To test after setting model path:
+```
+sh test_bert.sh
+```
+
+## Citation
+
+```
+@article{zhou2019limit,
+  title={{LIMIT-BERT}: Linguistic informed multi-task {BERT}},
+  author={Zhou, Junru and Zhang, Zhuosheng and Zhao, Hai},
+  journal={arXiv preprint arXiv:1910.14296},
+  year={2019}
+}
+```

model_cards/distilbert-base-cased-README.md

@@ -0,0 +1,3 @@
+---
+license: apache-2.0
+---

model_cards/distilbert-base-cased-distilled-squad-README.md

@@ -4,4 +4,5 @@ datasets:
 - squad
 metrics:
 - squad
+license: apache-2.0
 ---

model_cards/distilbert-base-german-cased-README.md

@@ -1,4 +1,5 @@
 ---
 language: de
+license: apache-2.0
 ---
 ## distilbert-base-german-cased

model_cards/distilbert-base-uncased-distilled-squad-README.md

@@ -6,4 +6,5 @@ widget:
   context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
 - text: "How many square kilometers of rainforest is covered in the basin?"
   context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
+license: apache-2.0
 ---

model_cards/distilbert-base-uncased-finetuned-sst-2-english-README.md

@@ -0,0 +1,3 @@
+---
+license: apache-2.0
+---

model_cards/dslim/bert-base-NER/README.md

@@ -0,0 +1,114 @@
+---
+language: en
+datasets:
+- conll2003
+---
+# bert-base-NER
+
+## Model description
+
+**bert-base-NER** is a fine-tuned BERT model that is ready to use for **Named Entity Recognition** and achieves **state-of-the-art performance** for the NER task. It has been trained to recognize four types of entities: location (LOC), organizations (ORG), person (PER) and Miscellaneous (MISC). 
+
+Specifically, this model is a *bert-base-cased* model that was fine-tuned on the English version of the standard [CoNLL-2003 Named Entity Recognition](https://www.aclweb.org/anthology/W03-0419.pdf) dataset. 
+## Intended uses & limitations
+
+#### How to use
+
+You can use this model with Transformers *pipeline* for NER.
+
+```python
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+from transformers import pipeline
+
+tokenizer = AutoTokenizer.from_pretrained("dslim/bert-base-NER")
+model = AutoModelForTokenClassification.from_pretrained("dslim/bert-base-NER")
+
+nlp = pipeline("ner", model=model, tokenizer=tokenizer)
+example = "My name is Wolfgang and I live in Berlin"
+
+ner_results = nlp(example)
+print(ner_results)
+```
+
+#### Limitations and bias
+
+This model is limited by its training dataset of entity-annotated news articles from a specific span of time. This may not generalize well for all use cases in different domains. Furthermore, the model occassionally tags subword tokens as entities and post-processing of results may be necessary to handle those cases. 
+
+## Training data
+
+This model was fine-tuned on English version of the standard [CoNLL-2003 Named Entity Recognition](https://www.aclweb.org/anthology/W03-0419.pdf) dataset. 
+
+The training dataset distinguishes between the beginning and continuation of an entity so that if there are back-to-back entities of the same type, the model can output where the second entity begins. As in the dataset, each token will be classified as one of the following classes:
+Abbreviation|Description
+-|-
+O|Outside of a named entity
+B-MIS |Beginning of a miscellaneous entity right after another miscellaneous entity
+I-MIS |Miscellaneous entity
+B-PER |Beginning of a person’s name right after another person’s name
+I-PER |Person’s name
+B-ORG |Beginning of an organisation right after another organisation
+I-ORG |Organisation
+B-LOC |Beginning of a location right after another location
+I-LOC |Location
+
+
+### CoNLL-2003 English Dataset Statistics
+This dataset was derived from the Reuters corpus which consists of Reuters news stories. You can read more about how this dataset was created in the CoNLL-2003 paper. 
+#### # of training examples per entity type
+Dataset|LOC|MISC|ORG|PER
+-|-|-|-|-
+Train|7140|3438|6321|6600
+Dev|1837|922|1341|1842
+Test|1668|702|1661|1617
+#### # of articles/sentences/tokens per dataset
+Dataset |Articles |Sentences |Tokens
+-|-|-|-
+Train |946 |14,987 |203,621
+Dev |216 |3,466 |51,362
+Test |231 |3,684 |46,435
+
+## Training procedure
+
+This model was trained on a single NVIDIA V100 GPU with recommended hyperparameters from the [original BERT paper](https://arxiv.org/pdf/1810.04805) which trained & evaluated the model on CoNLL-2003 NER task. 
+
+## Eval results
+metric|dev|test
+-|-|-
+f1 |95.1 |91.3
+precision |95.0 |90.7
+recall |95.3 |91.9
+
+The test metrics are a little lower than the official Google BERT results which encoded document context & experimented with CRF. More on replicating the original results [here](https://github.com/google-research/bert/issues/223).
+
+### BibTeX entry and citation info
+
+```
+@article{DBLP:journals/corr/abs-1810-04805,
+  author    = {Jacob Devlin and
+               Ming{-}Wei Chang and
+               Kenton Lee and
+               Kristina Toutanova},
+  title     = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
+               Understanding},
+  journal   = {CoRR},
+  volume    = {abs/1810.04805},
+  year      = {2018},
+  url       = {http://arxiv.org/abs/1810.04805},
+  archivePrefix = {arXiv},
+  eprint    = {1810.04805},
+  timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},
+  biburl    = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+```
+```
+@inproceedings{tjong-kim-sang-de-meulder-2003-introduction,
+    title = "Introduction to the {C}o{NLL}-2003 Shared Task: Language-Independent Named Entity Recognition",
+    author = "Tjong Kim Sang, Erik F.  and
+      De Meulder, Fien",
+    booktitle = "Proceedings of the Seventh Conference on Natural Language Learning at {HLT}-{NAACL} 2003",
+    year = "2003",
+    url = "https://www.aclweb.org/anthology/W03-0419",
+    pages = "142--147",
+}
+```

model_cards/facebook/bart-large-cnn/README.md

@@ -3,4 +3,5 @@ tags:
 - summarization
 
 license: mit
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---

model_cards/facebook/bart-large-mnli/README.md

@@ -0,0 +1,5 @@
+---
+license: mit
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
+pipeline_tag: zero-shot-classification
+---

model_cards/facebook/bart-large/README.md

@@ -1,3 +1,8 @@
+---
+license: mit
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
+---
+
 The Bart model was proposed by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer on 29 Oct, 2019. According to the abstract,
 
 Bart uses a standard seq2seq/machine translation architecture with a bidirectional encoder (like BERT) and a left-to-right decoder (like GPT).

model_cards/facebook/rag-sequence-base/README.md

@@ -1,3 +1,7 @@
+---
+license: apache-2.0
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
+---
 ## RAG
 
 This is a non-finetuned version of the RAG-Sequence model of the the paper [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401.pdf) 

model_cards/facebook/rag-sequence-nq/README.md

@@ -3,6 +3,7 @@ language: en
 license: apache-2.0
 datasets:
 - wiki_dpr
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 ## RAG
 
@@ -23,9 +24,9 @@ The model can generate answers to any factoid question as follows:
 ```python
 from transformers import RagTokenizer, RagRetriever, RagSequenceForGeneration 
  
-tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-nq") 
-retriever = RagRetriever.from_pretrained("facebook/rag-token-nq", index_name="exact", use_dummy_dataset=True) 
-model = RagSequenceForGeneration.from_pretrained("facebook/rag-token-nq", retriever=retriever) 
+tokenizer = RagTokenizer.from_pretrained("facebook/rag-sequence-nq") 
+retriever = RagRetriever.from_pretrained("facebook/rag-sequence-nq", index_name="exact", use_dummy_dataset=True) 
+model = RagSequenceForGeneration.from_pretrained("facebook/rag-sequence-nq", retriever=retriever) 
  
 input_dict = tokenizer.prepare_seq2seq_batch("how many countries are in europe", return_tensors="pt") 
 

model_cards/facebook/rag-token-base/README.md

@@ -1,3 +1,10 @@
+---
+language: en
+license: apache-2.0
+datasets:
+- wiki_dpr
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
+---
 ## RAG
 
 This is a non-finetuned version of the RAG-Token model of the the paper [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401.pdf) 

model_cards/facebook/rag-token-nq/README.md

@@ -3,6 +3,7 @@ language: en
 license: apache-2.0
 datasets:
 - wiki_dpr
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 ## RAG
 

model_cards/facebook/wmt19-de-en/README.md

@@ -1,9 +1,7 @@
-
 ---
 language: 
 - de
 - en
-thumbnail:
 tags:
 - translation
 - wmt19
@@ -13,6 +11,7 @@ datasets:
 - wmt19
 metrics:
 - bleu
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 
 # FSMT

model_cards/facebook/wmt19-en-de/README.md

@@ -1,9 +1,7 @@
-
 ---
 language: 
 - en
 - de
-thumbnail:
 tags:
 - translation
 - wmt19
@@ -13,6 +11,7 @@ datasets:
 - wmt19
 metrics:
 - bleu
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 
 # FSMT

model_cards/facebook/wmt19-en-ru/README.md

@@ -1,9 +1,7 @@
-
 ---
 language: 
 - en
 - ru
-thumbnail:
 tags:
 - translation
 - wmt19
@@ -13,6 +11,7 @@ datasets:
 - wmt19
 metrics:
 - bleu
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 
 # FSMT

model_cards/facebook/wmt19-ru-en/README.md

@@ -1,9 +1,7 @@
-
 ---
 language: 
 - ru
 - en
-thumbnail:
 tags:
 - translation
 - wmt19
@@ -13,6 +11,7 @@ datasets:
 - wmt19
 metrics:
 - bleu
+thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 
 # FSMT

model_cards/google/roberta2roberta_L-24_discofuse/README.md

@@ -19,6 +19,8 @@ Disclaimer: The model card has been written by the Hugging Face team.
 
 You can use this model for sentence fusion, *e.g.*
 
+IMPORTANT: The model was not trained on the `"` (double quotation mark) character -> so the before tokenizing the text, it is advised to replace all `"` (double quotation marks) with a single `` ` `` (single back tick).
+
 ```python
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 

model_cards/google/roberta2roberta_L-24_wikisplit/README.md

@@ -17,6 +17,9 @@ Disclaimer: The model card has been written by the Hugging Face team.
 
 You can use this model for sentence splitting, *e.g.*
 
+**IMPORTANT**: The model was not trained on the `"` (double quotation mark) character -> so the before tokenizing the text, 
+it is advised to replace all `"` (double quotation marks) with two single `'` (single quotation mark).
+
 ```python
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 
@@ -25,10 +28,9 @@ model = AutoModelForSeq2SeqLM.from_pretrained("google/roberta2roberta_L-24_wikis
 
 long_sentence = """Due to the hurricane, Lobsterfest has been canceled, making Bob very happy about it and he decides to open Bob 's Burgers for customers who were planning on going to Lobsterfest."""
 
-input_ids = tokenizer(long_sentence, return_tensors="pt").input_ids
+input_ids = tokenizer(tokenizer.bos_token + long_sentence + tokenizer.eos_token, return_tensors="pt").input_ids
 output_ids = model.generate(input_ids)[0]
 print(tokenizer.decode(output_ids, skip_special_tokens=True))
 # should output
-# Due Due hurricane, Lobsterfest has been canceled, making Bob very happy about it. He decides to open B
-# ob's Burgers for customers who were planning on going to Lobsterfest.com.
+# Due to the hurricane, Lobsterfest has been canceled, making Bob very happy about it. He decides to open Bob's Burgers for customers who were planning on going to Lobsterfest. 
 ```

model_cards/huawei-noah/TinyBERT_General_4L_312D/README.md

@@ -0,0 +1,19 @@
+TinyBERT: Distilling BERT for Natural Language Understanding
+======== 
+TinyBERT is 7.5x smaller and 9.4x faster on inference than BERT-base and achieves competitive performances in the tasks of natural language understanding. It performs a novel transformer distillation at both the pre-training and task-specific learning stages. In general distillation, we use the original BERT-base without fine-tuning as the teacher and a large-scale text corpus as the learning data. By performing the Transformer distillation on the text from general domain, we obtain a general TinyBERT which provides a good initialization for the task-specific distillation. We here provide the general TinyBERT for your tasks at hand.
+
+For more details about the techniques of TinyBERT, refer to our paper:
+[TinyBERT: Distilling BERT for Natural Language Understanding](https://arxiv.org/abs/1909.10351)
+
+
+Citation
+========
+If you find TinyBERT useful in your research, please cite the following paper:
+```
+@article{jiao2019tinybert,
+  title={Tinybert: Distilling bert for natural language understanding},
+  author={Jiao, Xiaoqi and Yin, Yichun and Shang, Lifeng and Jiang, Xin and Chen, Xiao and Li, Linlin and Wang, Fang and Liu, Qun},
+  journal={arXiv preprint arXiv:1909.10351},
+  year={2019}
+}
+```

model_cards/joeddav/xlm-roberta-large-xnli/README.md

@@ -9,6 +9,7 @@ datasets:
 - xnli
 widget:
 - text: "За кого вы голосуете в 2020 году? <sep> This text is about politique."
+license: mit
 ---
 
 # xlm-roberta-large-xnli

model_cards/jordimas/julibert/README.md

@@ -0,0 +1,23 @@
+---
+language: ca
+---
+
+## Introduction
+
+
+Download the model here:
+
+* Catalan Roberta model: [julibert-2020-11-10.zip](https://www.softcatala.org/pub/softcatala/julibert/julibert-2020-11-10.zip)
+
+## What's this?
+
+Source code: https://github.com/Softcatala/julibert
+
+* Corpus: Oscar Catalan Corpus (3,8G)
+* Model type: Roberta
+* Vocabulary size: 50265
+* Steps: 500000
+
+
+
+

model_cards/microsoft/prophetnet-large-uncased-cnndm/README.md

@@ -4,7 +4,6 @@ ProphetNet is a new pre-trained language model for sequence-to-sequence learning
 ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
 
 ### Usage
-A quick usage is like: 
 ```
 from transformers import ProphetNetTokenizer, ProphetNetForConditionalGeneration, ProphetNetConfig
 
@@ -16,7 +15,7 @@ inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=100, return_tensors='pt')
 
 # Generate Summary
 summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=512, early_stopping=True)
-print([tokenizer.decode(g) for g in summary_ids])
+tokenizer.batch_decode(summary_ids.tolist())
 ```
 Here, [X_SEP] is used as a special token to seperate sentences.
 ### Citation

model_cards/nlptown/bert-base-multilingual-uncased-sentiment/README.md

@@ -6,6 +6,8 @@ language:
 - fr
 - it
 - es
+
+license: mit
 ---
 
 # bert-base-multilingual-uncased-sentiment

model_cards/sagorsarker/bangla-bert-base/README.md

@@ -49,6 +49,7 @@ Our final vocab file availabe at [https://github.com/sagorbrur/bangla-bert](http
 
 ## Evaluation Results
 
+### LM Evaluation Results
 After training 1 millions steps here is the evaluation resutls. 
 
 ```
@@ -61,9 +62,22 @@ next_sentence_loss = 0.040997364
 perplexity = numpy.exp(2.2406516) = 9.393331287442784
 Loss for final step: 2.426227
 
-
 ```
 
+### Downstream Task Evaluation Results
+Huge Thanks to [Nick Doiron](https://twitter.com/mapmeld) for providing evalution results of classification task.
+He used [Bengali Classification Benchmark](https://github.com/rezacsedu/Classification_Benchmarks_Benglai_NLP) datasets for classification task.
+Comparing to Nick's [Bengali electra](https://huggingface.co/monsoon-nlp/bangla-electra) and multi-lingual BERT, Bangla BERT Base achieves state of the art result.
+Here is the [evaluation script](https://github.com/sagorbrur/bangla-bert/blob/master/notebook/bangla-bert-evaluation-classification-task.ipynb).
+
+
+| Model | Sentiment Analysis | Hate Speech Task | News Topic Task | Average |
+| ----- | -------------------| ---------------- | --------------- | ------- |
+| mBERT | 68.15 | 52.32 | 72.27 | 64.25 |
+| Bengali Electra | 69.19 | 44.84 | 82.33 | 65.45 |
+| Bangla BERT Base | 70.37 | 71.83 | 89.19 | 77.13 |
+
+
 **NB: If you use this model for any nlp task please share evaluation results with us. We will add it here.** 
 
 
@@ -73,8 +87,8 @@ You can use this model directly with a pipeline for masked language modeling:
 ```py
 from transformers import BertForMaskedLM, BertTokenizer, pipeline
 
-model = BertForMaskedLM.from_pretrained("bangla-bert-base")
-tokenizer = BertTokenizer.from_pretrained("bangla-bert-base")
+model = BertForMaskedLM.from_pretrained("sagorsarker/bangla-bert-base")
+tokenizer = BertTokenizer.from_pretrained("sagorsarker/bangla-bert-base")
 nlp = pipeline('fill-mask', model=model, tokenizer=tokenizer)
 for pred in nlp(f"আমি বাংলায় {nlp.tokenizer.mask_token} গাই।"):
   print(pred)

model_cards/sarahlintang/IndoBERT/README.md

@@ -0,0 +1,43 @@
+---
+language: id
+datasets:
+- oscar
+---
+# IndoBERT (Indonesian BERT Model)
+
+## Model description
+IndoBERT is a pre-trained language model based on BERT architecture for the Indonesian Language. 
+
+This model is base-uncased version which use bert-base config.
+
+## Intended uses & limitations
+
+#### How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+tokenizer = AutoTokenizer.from_pretrained("sarahlintang/IndoBERT")
+model = AutoModel.from_pretrained("sarahlintang/IndoBERT")
+tokenizer.encode("hai aku mau makan.")
+[2, 8078, 1785, 2318, 1946, 18, 4]
+```
+
+
+## Training data
+
+This model was pre-trained on 16 GB of raw text ~2 B words from Oscar Corpus (https://oscar-corpus.com/). 
+
+This model is equal to bert-base model which has 32,000 vocabulary size. 
+
+## Training procedure
+
+The training of the model has been performed using Google’s original Tensorflow code on eight core Google Cloud TPU v2.
+We used a Google Cloud Storage bucket, for persistent storage of training data and models.
+
+## Eval results
+
+We evaluate this model on three Indonesian NLP downstream task:
+- some extractive summarization model
+- sentiment analysis
+- Part-of-Speech Tagger
+it was proven that this model outperforms multilingual BERT for all downstream tasks.

model_cards/sentence-transformers/LaBSE/README.md

@@ -0,0 +1,37 @@
+# LaBSE Pytorch Version 
+This is a pytorch port of the tensorflow version of [LaBSE](https://tfhub.dev/google/LaBSE/1).
+
+To get the sentence embeddings, you can  use the following code:
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/LaBSE")
+model = AutoModel.from_pretrained("sentence-transformers/LaBSE")
+
+sentences = ["Hello World", "Hallo Welt"]
+
+encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=64, return_tensors='pt')
+
+with torch.no_grad():
+    model_output = model(**encoded_input, return_dict=True)
+
+embeddings = model_output.pooler_output
+embeddings = torch.nn.functional.normalize(embeddings)
+print(embeddings)
+```
+
+
+When you have [sentence-transformers](https://www.sbert.net/) installed, you can use the model like this:
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["Hello World", "Hallo Welt"]
+
+model = SentenceTransformer('LaBSE')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+## Reference:
+Fangxiaoyu Feng, Yinfei Yang, Daniel Cer, Narveen Ari, Wei Wang. [Language-agnostic BERT Sentence Embedding](https://arxiv.org/abs/2007.01852). July 2020
+
+License: [https://tfhub.dev/google/LaBSE/1](https://tfhub.dev/google/LaBSE/1)

scripts/fsmt/tests-to-run.sh

@@ -2,5 +2,5 @@
 
 # these scripts need to be run before any changes to FSMT-related code - it should cover all bases
 
-USE_CUDA=0 RUN_SLOW=1 pytest --disable-warnings tests/test_tokenization_fsmt.py tests/test_configuration_auto.py tests/test_modeling_fsmt.py examples/seq2seq/test_fsmt_bleu_score.py
-USE_CUDA=1 RUN_SLOW=1 pytest --disable-warnings tests/test_tokenization_fsmt.py tests/test_configuration_auto.py tests/test_modeling_fsmt.py examples/seq2seq/test_fsmt_bleu_score.py
+CUDA_VISIBLE_DEVICES="" RUN_SLOW=1 pytest --disable-warnings tests/test_tokenization_fsmt.py tests/test_configuration_auto.py tests/test_modeling_fsmt.py examples/seq2seq/test_fsmt_bleu_score.py
+RUN_SLOW=1 pytest --disable-warnings tests/test_tokenization_fsmt.py tests/test_configuration_auto.py tests/test_modeling_fsmt.py examples/seq2seq/test_fsmt_bleu_score.py

scripts/pegasus/build_test_sample_spm_no_bos.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+# this script builds a small sample spm file tests/fixtures/test_sentencepiece_no_bos.model, with features needed by pegasus 
+
+# 1. pip install sentencepiece
+# 
+# 2. wget https://raw.githubusercontent.com/google/sentencepiece/master/data/botchan.txt
+
+# 3. build
+import sentencepiece as spm
+
+# pegasus:
+# 1. no bos
+# 2. eos_id is 1
+# 3. unk_id is 2
+# build a sample spm file accordingly
+spm.SentencePieceTrainer.train('--input=botchan.txt --model_prefix=test_sentencepiece_no_bos --bos_id=-1 --unk_id=2  --eos_id=1  --vocab_size=1000')
+
+# 4. now update the fixture
+# mv test_sentencepiece_no_bos.model ../../tests/fixtures/

scripts/tatoeba/README.md

@@ -0,0 +1,44 @@
+Setup transformers following instructions in README.md, (I would fork first).
+```bash
+git clone git@github.com:huggingface/transformers.git
+cd transformers
+pip install -e .
+pip install pandas
+```
+
+Get required metadata
+```
+curl https://cdn-datasets.huggingface.co/language_codes/language-codes-3b2.csv  > language-codes-3b2.csv
+curl https://cdn-datasets.huggingface.co/language_codes/iso-639-3.csv > iso-639-3.csv
+```
+
+Install Tatoeba-Challenge repo inside transformers
+```bash
+git clone git@github.com:Helsinki-NLP/Tatoeba-Challenge.git
+```
+
+To convert a few models, call the conversion script from command line:
+```bash
+python src/transformers/convert_marian_tatoeba_to_pytorch.py --models heb-eng eng-heb --save_dir converted
+```
+
+To convert lots of models you can pass your list of Tatoeba model names to `resolver.convert_models` in a python client or script.
+
+```python
+from transformers.convert_marian_tatoeba_to_pytorch import TatoebaConverter
+resolver = TatoebaConverter(save_dir='converted')
+resolver.convert_models(['heb-eng', 'eng-heb'])
+```
+
+
+### Upload converted models
+```bash
+cd converted
+transformers-cli login
+for FILE in *; do transformers-cli upload $FILE; done
+```
+
+
+### Modifications
+- To change naming logic, change the code near `os.rename`. The model card creation code may also need to change.
+- To change model card content, you must modify `TatoebaCodeResolver.write_model_card`

setup.py

@@ -3,7 +3,10 @@ Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/m
 
 To create the package for pypi.
 
-1. Change the version in __init__.py, setup.py as well as docs/source/conf.py.
+1. Change the version in __init__.py, setup.py as well as docs/source/conf.py. Remove the master from the links in
+   the new models of the README:
+   (https://huggingface.co/transformers/master/model_doc/ -> https://huggingface.co/transformers/model_doc/)
+   then run `make fix-copies` to fix the index of the documentation.
 
 2. Unpin specific versions from setup.py that use a git install.
 
@@ -84,21 +87,23 @@ extras["tf-cpu"] = [
     # "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx",
 ]
 extras["torch"] = ["torch>=1.0"]
+extras["flax"] = ["jaxlib==0.1.55", "jax>=0.2.0", "flax==0.2.2"]
 extras["onnxruntime"] = ["onnxruntime>=1.4.0", "onnxruntime-tools>=1.4.2"]
 
 extras["serving"] = ["pydantic", "uvicorn", "fastapi", "starlette"]
 extras["all"] = extras["serving"] + ["tensorflow", "torch"]
 
+extras["sentencepiece"] = ["sentencepiece!=0.1.92"]
 extras["retrieval"] = ["faiss-cpu", "datasets"]
 extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil"] + extras["retrieval"]
 # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
 extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
 extras["quality"] = ["black >= 20.8b1", "isort >= 5.5.4", "flake8 >= 3.8.3"]
-extras["dev"] = extras["testing"] + extras["quality"] + extras["ja"] + ["scikit-learn", "tensorflow", "torch"]
+extras["dev"] = extras["testing"] + extras["quality"] + extras["ja"] + ["scikit-learn", "tensorflow", "torch", "sentencepiece!=0.1.92"]
 
 setup(
     name="transformers",
-    version="3.3.1",
+    version="3.4.0",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Sylvain Gugger, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
@@ -111,7 +116,7 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.9.0.rc2",
+        "tokenizers == 0.9.2",
         # dataclasses for Python versions that don't have it
         "dataclasses;python_version<'3.7'",
         # utilities from PyPA to e.g. compare versions
@@ -131,9 +136,7 @@ setup(
         "sacremoses",
     ],
     extras_require=extras,
-    entry_points={
-        "console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]
-    },
+    entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
     python_requires=">=3.6.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",

src/transformers/__init__.py

@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-__version__ = "3.3.1"
+__version__ = "3.4.0"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -94,6 +94,7 @@ from .file_utils import (
     MODEL_CARD_NAME,
     PYTORCH_PRETRAINED_BERT_CACHE,
     PYTORCH_TRANSFORMERS_CACHE,
+    SPIECE_UNDERLINE,
     TF2_WEIGHTS_NAME,
     TF_WEIGHTS_NAME,
     TRANSFORMERS_CACHE,
@@ -104,10 +105,13 @@ from .file_utils import (
     is_apex_available,
     is_datasets_available,
     is_faiss_available,
+    is_flax_available,
     is_psutil_available,
     is_py3nvml_available,
+    is_sentencepiece_available,
     is_sklearn_available,
     is_tf_available,
+    is_tokenizers_available,
     is_torch_available,
     is_torch_tpu_available,
 )
@@ -154,49 +158,42 @@ from .pipelines import (
 from .retrieval_rag import RagRetriever
 
 # Tokenizers
-from .tokenization_albert import AlbertTokenizer, AlbertTokenizerFast
 from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
-from .tokenization_bart import BartTokenizer, BartTokenizerFast
-from .tokenization_bert import BasicTokenizer, BertTokenizer, BertTokenizerFast, WordpieceTokenizer
-from .tokenization_bert_generation import BertGenerationTokenizer
+from .tokenization_bart import BartTokenizer
+from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer
 from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer
 from .tokenization_bertweet import BertweetTokenizer
 from .tokenization_blenderbot import BlenderbotSmallTokenizer, BlenderbotTokenizer
-from .tokenization_camembert import CamembertTokenizer, CamembertTokenizerFast
 from .tokenization_ctrl import CTRLTokenizer
 from .tokenization_deberta import DebertaTokenizer
-from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFast
+from .tokenization_distilbert import DistilBertTokenizer
 from .tokenization_dpr import (
     DPRContextEncoderTokenizer,
-    DPRContextEncoderTokenizerFast,
     DPRQuestionEncoderTokenizer,
-    DPRQuestionEncoderTokenizerFast,
+    DPRReaderOutput,
     DPRReaderTokenizer,
-    DPRReaderTokenizerFast,
 )
-from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast
+from .tokenization_electra import ElectraTokenizer
 from .tokenization_flaubert import FlaubertTokenizer
 from .tokenization_fsmt import FSMTTokenizer
-from .tokenization_funnel import FunnelTokenizer, FunnelTokenizerFast
-from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
-from .tokenization_layoutlm import LayoutLMTokenizer, LayoutLMTokenizerFast
-from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast
-from .tokenization_lxmert import LxmertTokenizer, LxmertTokenizerFast
-from .tokenization_mbart import MBartTokenizer, MBartTokenizerFast
-from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFast
-from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
-from .tokenization_pegasus import PegasusTokenizer, PegasusTokenizerFast
+from .tokenization_funnel import FunnelTokenizer
+from .tokenization_gpt2 import GPT2Tokenizer
+from .tokenization_herbert import HerbertTokenizer
+from .tokenization_layoutlm import LayoutLMTokenizer
+from .tokenization_longformer import LongformerTokenizer
+from .tokenization_lxmert import LxmertTokenizer
+from .tokenization_mobilebert import MobileBertTokenizer
+from .tokenization_openai import OpenAIGPTTokenizer
 from .tokenization_phobert import PhobertTokenizer
 from .tokenization_prophetnet import ProphetNetTokenizer
 from .tokenization_rag import RagTokenizer
-from .tokenization_reformer import ReformerTokenizer, ReformerTokenizerFast
-from .tokenization_retribert import RetriBertTokenizer, RetriBertTokenizerFast
-from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
-from .tokenization_squeezebert import SqueezeBertTokenizer, SqueezeBertTokenizerFast
-from .tokenization_t5 import T5Tokenizer, T5TokenizerFast
+from .tokenization_retribert import RetriBertTokenizer
+from .tokenization_roberta import RobertaTokenizer
+from .tokenization_squeezebert import SqueezeBertTokenizer
 from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer
 from .tokenization_utils import PreTrainedTokenizer
 from .tokenization_utils_base import (
+    AddedToken,
     BatchEncoding,
     CharSpan,
     PreTrainedTokenizerBase,
@@ -204,11 +201,59 @@ from .tokenization_utils_base import (
     TensorType,
     TokenSpan,
 )
-from .tokenization_utils_fast import PreTrainedTokenizerFast
 from .tokenization_xlm import XLMTokenizer
-from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer
-from .tokenization_xlm_roberta import XLMRobertaTokenizer, XLMRobertaTokenizerFast
-from .tokenization_xlnet import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
+
+
+if is_sentencepiece_available():
+    from .tokenization_albert import AlbertTokenizer
+    from .tokenization_bert_generation import BertGenerationTokenizer
+    from .tokenization_camembert import CamembertTokenizer
+    from .tokenization_marian import MarianTokenizer
+    from .tokenization_mbart import MBartTokenizer
+    from .tokenization_pegasus import PegasusTokenizer
+    from .tokenization_reformer import ReformerTokenizer
+    from .tokenization_t5 import T5Tokenizer
+    from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer
+    from .tokenization_xlm_roberta import XLMRobertaTokenizer
+    from .tokenization_xlnet import XLNetTokenizer
+else:
+    from .utils.dummy_sentencepiece_objects import *
+
+if is_tokenizers_available():
+    from .tokenization_albert_fast import AlbertTokenizerFast
+    from .tokenization_bart_fast import BartTokenizerFast
+    from .tokenization_bert_fast import BertTokenizerFast
+    from .tokenization_camembert_fast import CamembertTokenizerFast
+    from .tokenization_distilbert_fast import DistilBertTokenizerFast
+    from .tokenization_dpr_fast import (
+        DPRContextEncoderTokenizerFast,
+        DPRQuestionEncoderTokenizerFast,
+        DPRReaderTokenizerFast,
+    )
+    from .tokenization_electra_fast import ElectraTokenizerFast
+    from .tokenization_funnel_fast import FunnelTokenizerFast
+    from .tokenization_gpt2_fast import GPT2TokenizerFast
+    from .tokenization_herbert_fast import HerbertTokenizerFast
+    from .tokenization_layoutlm_fast import LayoutLMTokenizerFast
+    from .tokenization_longformer_fast import LongformerTokenizerFast
+    from .tokenization_lxmert_fast import LxmertTokenizerFast
+    from .tokenization_mbart_fast import MBartTokenizerFast
+    from .tokenization_mobilebert_fast import MobileBertTokenizerFast
+    from .tokenization_openai_fast import OpenAIGPTTokenizerFast
+    from .tokenization_pegasus_fast import PegasusTokenizerFast
+    from .tokenization_reformer_fast import ReformerTokenizerFast
+    from .tokenization_retribert_fast import RetriBertTokenizerFast
+    from .tokenization_roberta_fast import RobertaTokenizerFast
+    from .tokenization_squeezebert_fast import SqueezeBertTokenizerFast
+    from .tokenization_t5_fast import T5TokenizerFast
+    from .tokenization_utils_fast import PreTrainedTokenizerFast
+    from .tokenization_xlm_roberta_fast import XLMRobertaTokenizerFast
+    from .tokenization_xlnet_fast import XLNetTokenizerFast
+
+    if is_sentencepiece_available():
+        from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS, convert_slow_tokenizer
+else:
+    from .utils.dummy_tokenizers_objects import *
 
 # Trainer
 from .trainer_callback import (
@@ -441,6 +486,7 @@ if is_torch_available():
     from .modeling_openai import (
         OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
         OpenAIGPTDoubleHeadsModel,
+        OpenAIGPTForSequenceClassification,
         OpenAIGPTLMHeadModel,
         OpenAIGPTModel,
         OpenAIGPTPreTrainedModel,
@@ -449,6 +495,9 @@ if is_torch_available():
     from .modeling_pegasus import PegasusForConditionalGeneration
     from .modeling_prophetnet import (
         PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
+        ProphetNetDecoder,
+        ProphetNetEncoder,
+        ProphetNetForCausalLM,
         ProphetNetForConditionalGeneration,
         ProphetNetModel,
         ProphetNetPreTrainedModel,
@@ -515,6 +564,9 @@ if is_torch_available():
     )
     from .modeling_xlm_prophetnet import (
         XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
+        XLMProphetNetDecoder,
+        XLMProphetNetEncoder,
+        XLMProphetNetForCausalLM,
         XLMProphetNetForConditionalGeneration,
         XLMProphetNetModel,
     )
@@ -552,7 +604,6 @@ if is_torch_available():
         get_linear_schedule_with_warmup,
         get_polynomial_decay_schedule_with_warmup,
     )
-    from .tokenization_marian import MarianTokenizer
 
     # Trainer
     from .trainer import Trainer
@@ -787,6 +838,15 @@ else:
     from .utils.dummy_tf_objects import *
 
 
+if is_flax_available():
+    from .modeling_flax_bert import FlaxBertModel
+    from .modeling_flax_roberta import FlaxRobertaModel
+else:
+    # Import the same objects as dummies to get them in the namespace.
+    # They will raise an import error if the user tries to instantiate / use them.
+    from .utils.dummy_flax_objects import *
+
+
 if not is_tf_available() and not is_torch_available():
     logger.warning(
         "Neither PyTorch nor TensorFlow >= 2.0 have been found."

src/transformers/activations.py

@@ -66,7 +66,3 @@ def get_activation(activation_string):
         return ACT2FN[activation_string]
     else:
         raise KeyError("function {} not found in ACT2FN mapping {}".format(activation_string, list(ACT2FN.keys())))
-
-
-def mish(x):
-    return x * torch.tanh(torch.nn.functional.softplus(x))

src/transformers/configuration_auto.py

@@ -274,7 +274,7 @@ class AutoConfig:
                       our S3, e.g., ``dbmdz/bert-base-german-cased``.
                     - A path to a `directory` containing a configuration file saved using the
                       :meth:`~transformers.PretrainedConfig.save_pretrained` method, or the
-                      :meth:`~transformers.PretrainedModel.save_pretrained` method, e.g., ``./my_model_directory/``.
+                      :meth:`~transformers.PreTrainedModel.save_pretrained` method, e.g., ``./my_model_directory/``.
                     - A path or url to a saved configuration JSON `file`, e.g.,
                       ``./my_model_directory/configuration.json``.
             cache_dir (:obj:`str`, `optional`):

src/transformers/configuration_dpr.py

@@ -24,6 +24,9 @@ DPR_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "facebook/dpr-ctx_encoder-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-ctx_encoder-single-nq-base/config.json",
     "facebook/dpr-question_encoder-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-question_encoder-single-nq-base/config.json",
     "facebook/dpr-reader-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-reader-single-nq-base/config.json",
+    "facebook/dpr-ctx_encoder-multiset-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-ctx_encoder-multiset-base/config.json",
+    "facebook/dpr-question_encoder-multiset-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-question_encoder-multiset-base/config.json",
+    "facebook/dpr-reader-multiset-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-reader-multiset-base/config.json",
 }
 
 

src/transformers/configuration_lxmert.py

@@ -15,12 +15,11 @@
 """ LXMERT model configuration """
 
 
-import logging
-
 from .configuration_utils import PretrainedConfig
+from .utils import logging
 
 
-logger = logging.getLogger(__name__)
+logger = logging.get_logger(__name__)
 
 LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "unc-nlp/lxmert-base-uncased": "",

src/transformers/configuration_mmbt.py

@@ -31,7 +31,7 @@ class MMBTConfig(object):
             Config of the underlying Transformer models. Its values are copied over to use a single config.
         num_labels (:obj:`int`, `optional`):
             Size of final Linear layer for classification.
-        modal_hidden_size (:obj:`int`, `optional`, defautls to 2048):
+        modal_hidden_size (:obj:`int`, `optional`, defaults to 2048):
             Embedding dimension of the non-text modality encoder.
     """
 

src/transformers/configuration_prophetnet.py

@@ -15,12 +15,11 @@
 """ ProphetNet model configuration """
 
 
-import logging
-
 from .configuration_utils import PretrainedConfig
+from .utils import logging
 
 
-logger = logging.getLogger(__name__)
+logger = logging.get_logger(__name__)
 
 PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "microsoft/prophetnet-large-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/microsoft/prophetnet-large-uncased/config.json",
@@ -29,7 +28,71 @@ PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
 
 class ProphetNetConfig(PretrainedConfig):
     r"""
-    Configuration class for ProphetNet. Parameters are renamed from the fairseq implementation
+    This is the configuration class to store the configuration of a :class:`~transformers.ProphetNetModel`. It is used to
+    instantiate a ProphetNet model according to the specified arguments, defining the model architecture.
+
+    Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
+    to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
+    for more information.
+
+    Args:
+        activation_dropout (:obj:`float`, `optional`, defaults to 0.1):
+            The dropout ratio for activations inside the fully connected layer.
+        activation_function (:obj:`str` or :obj:`function`, `optional`, defaults to :obj:`"gelu"`):
+            The non-linear activation function (function or string) in the encoder and pooler.
+            If string, :obj:`"gelu"`, :obj:`"relu"`, :obj:`"swish"` and :obj:`"gelu_new"` are supported.
+        vocab_size (:obj:`int`, `optional`, defaults to 30522):
+            Vocabulary size of the ProphetNET model. Defines the number of different tokens that can be represented by the
+            :obj:`inputs_ids` passed when calling :class:`~transformers.ProphetNetModel`.
+        hidden_size (:obj:`int`, `optional`, defaults to 1024):
+            Dimensionality of the layers and the pooler layer.
+        encoder_ffn_dim (:obj:`int`, `optional`, defaults to 4096):
+            Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
+        num_encoder_layers (:obj:`int`, `optional`, defaults to 12):
+            Number of encoder layers.
+        num_encoder_attention_heads (:obj:`int`, `optional`, defaults to 16):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        decoder_ffn_dim (:obj:`int`, `optional`, defaults to 4096):
+            Dimensionality of the ``intermediate`` (often named feed-forward) layer in decoder.
+        num_decoder_layers (:obj:`int`, `optional`, defaults to 12):
+            Number of decoder layers.
+        num_decoder_attention_heads (:obj:`int`, `optional`, defaults to 16):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        attention_dropout (:obj:`float`, `optional`, defaults to 0.1):
+            The dropout ratio for the attention probabilities.
+        dropout (:obj:`float`, `optional`, defaults to 0.1):
+            The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
+        max_position_embeddings (:obj:`int`, `optional`, defaults to 512):
+            The maximum sequence length that this model might ever be used with.
+            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+        init_std (:obj:`float`, `optional`, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        add_cross_attention (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether cross-attention layers should be added to the model.
+        is_encoder_decoder (:obj:`bool`, `optional`, defaults to :obj:`True`):
+            Whether this is an encoder/decoder model.
+        pad_token_id (:obj:`int`, `optional`, defaults to 1)
+            Padding token id.
+        bos_token_id (:obj:`int`, `optional`, defaults to 0)
+            Beginning of stream token id.
+        eos_token_id (:obj:`int`, `optional`, defaults to 2)
+            End of stream token id.
+        ngram (:obj:`int`, `optional`, defaults to 2)
+            Number of future tokens to predict.
+            Set to 1 to be same as traditional Language model to predict next first token.
+        num_buckets (:obj:`int`, `optional`, defaults to 32)
+            The number of buckets to use for each attention layer.
+            This is for relative position calculation. See the `T5 paper
+            <see https://arxiv.org/abs/1910.10683>`__ for more details.
+        relative_max_distance (:obj:`int`, `optional`, defaults to 128)
+            Relative distances greater than this number will be put into the last same bucket.
+            This is for relative position calculation. See the `T5 paper
+            <see https://arxiv.org/abs/1910.10683>`__ for more details.
+        disable_ngram_loss (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether be trained predicting only the next first token.
+        eps (:obj:`float`, `optional`, defaults to 0.0):
+            Controls the ``epsilon`` parameter value for label
+            smoothing in the loss calculation. If set to 0, no label smoothing is performed.
     """
     model_type = "prophetnet"
 
@@ -45,13 +108,12 @@ class ProphetNetConfig(PretrainedConfig):
         decoder_ffn_dim=4096,
         num_decoder_layers=12,
         num_decoder_attention_heads=16,
-        encoder_layerdrop=0.1,
-        decoder_layerdrop=0.1,
         attention_dropout=0.1,
         dropout=0.1,
         max_position_embeddings=512,
         init_std=0.02,
         is_encoder_decoder=True,
+        add_cross_attention=True,
         pad_token_id=0,
         bos_token_id=1,
         eos_token_id=2,
@@ -61,23 +123,22 @@ class ProphetNetConfig(PretrainedConfig):
         relative_max_distance=128,
         disable_ngram_loss=False,
         eps=0.0,
-        **common_kwargs
+        **kwargs
     ):
         super().__init__(
             pad_token_id=pad_token_id,
             bos_token_id=bos_token_id,
             eos_token_id=eos_token_id,
             is_encoder_decoder=is_encoder_decoder,
+            add_cross_attention=add_cross_attention,
             decoder_start_token_id=decoder_start_token_id,
-            **common_kwargs,
+            **kwargs,
         )
         self.vocab_size = vocab_size
         self.hidden_size = hidden_size
         self.encoder_ffn_dim = encoder_ffn_dim
         self.num_encoder_layers = num_encoder_layers
         self.num_encoder_attention_heads = num_encoder_attention_heads
-        self.encoder_layerdrop = encoder_layerdrop
-        self.decoder_layerdrop = decoder_layerdrop
         self.decoder_ffn_dim = decoder_ffn_dim
         self.num_decoder_layers = num_decoder_layers
         self.num_decoder_attention_heads = num_decoder_attention_heads

src/transformers/configuration_rag.py

@@ -41,7 +41,7 @@ RAG_CONFIG_DOC = r"""
             Retrieval batch size, defined as the number of queries issues concurrently to the faiss index excapsulated
             :class:`~transformers.RagRetriever`.
         dataset (:obj:`str`, `optional`, defaults to :obj:`"wiki_dpr"`):
-            A dataset identifier of the indexed dataset on HuggingFace AWS bucket (list all available datasets and
+            A dataset identifier of the indexed dataset in HuggingFace Datasets (list all available datasets and
             ids using :obj:`datasets.list_datasets()`).
         dataset_split (:obj:`str`, `optional`, defaults to :obj:`"train"`)
             Which split of the :obj:`dataset` to load.