Test model parallelization

Note: parellelism not yet introduced into t5. Just doing this to get past checks.

.circleci/config.yml

@@ -84,7 +84,7 @@ jobs:
                 key: v0.3-{{ checksum "setup.py" }}
                 paths:
                     - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ --cov --durations=0 | tee output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ --cov  | tee output.txt
             - run: codecov
             - store_artifacts:
                   path: ~/transformers/output.txt
@@ -139,31 +139,6 @@ jobs:
             - store_artifacts:
                path: ~/transformers/output.txt
                destination: test_output.txt
-    run_tests_flax:
-        working_directory: ~/transformers
-        docker:
-            - image: circleci/python:3.7
-        environment:
-            OMP_NUM_THREADS: 1
-        resource_class: xlarge
-        parallelism: 1
-        steps:
-            - checkout
-            - restore_cache:
-                keys:
-                    - v0.3-flax-{{ checksum "setup.py" }}
-                    - v0.3-{{ checksum "setup.py" }}
-            - run: pip install --upgrade pip
-            - run: pip install git+https://github.com/huggingface/datasets
-            - run: sudo pip install .[flax,sklearn,torch,testing]
-            - save_cache:
-                  key: v0.3-flax-{{ checksum "setup.py" }}
-                  paths:
-                      - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
-            - store_artifacts:
-                  path: ~/transformers/output.txt
-                  destination: test_output.txt
     run_tests_custom_tokenizers:
         working_directory: ~/transformers
         docker:
@@ -223,7 +198,7 @@ jobs:
                       - v0.3-build_doc-{{ checksum "setup.py" }}
                       - v0.3-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[tf,torch,sentencepiece,docs]
+            - run: pip install .[tf,torch,docs]
             - save_cache:
                   key: v0.3-build_doc-{{ checksum "setup.py" }}
                   paths:
@@ -244,7 +219,7 @@ jobs:
                   keys:
                       - v0.3-deploy_doc-{{ checksum "setup.py" }}
                       - v0.3-{{ checksum "setup.py" }}
-            - run: pip install .[tf,torch,sentencepiece,docs]
+            - run: pip install .[tf,torch,docs]
             - save_cache:
                   key: v0.3-deploy_doc-{{ checksum "setup.py" }}
                   paths:
@@ -264,7 +239,7 @@ jobs:
                       - v0.3-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
             - run: pip install isort
-            - run: pip install .[tf,torch,flax,quality]
+            - run: pip install .[tf,torch,quality]
             - save_cache:
                   key: v0.3-code_quality-{{ checksum "setup.py" }}
                   paths:
@@ -330,7 +305,6 @@ workflows:
             - run_tests_torch_and_tf
             - run_tests_torch
             - run_tests_tf
-            - run_tests_flax
             - build_doc
             - deploy_doc: *workflow_filters
     tpu_testing_jobs:

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,15 +16,15 @@ Fixes # (issue)
 
 
 ## Before submitting
-- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
-- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md#start-contributing-pull-requests),
+- [ ] This PR fixes a typo or improves the docs (you can dimiss the other checks if that's the case).
+- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md#start-contributing-pull-requests), 
       Pull Request section?
 - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
       to the it if that's the case.
 - [ ] Did you make sure to update the documentation with your changes? Here are the
       [documentation guidelines](https://github.com/huggingface/transformers/tree/master/docs), and
       [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/master/docs#writing-source-documentation).
-- [ ] Did you write any new necessary tests?
+- [ ] Did you write any new necessary tests? 
 
 
 ## Who can review?
@@ -37,7 +37,7 @@ members/contributors which may be interested in your PR.
  If you know how to use git blame, that is the easiest way, otherwise, here is a rough guide of **who to tag**.
  Please tag fewer than 3 people.
 
- albert, bert, XLM: @LysandreJik
+ albert, bert, XLM: @LysandreJik 
  GPT2: @LysandreJik, @patrickvonplaten
  tokenizers: @mfuntowicz
  Trainer: @sgugger

.github/workflows/github-torch-hub.yml

@@ -30,7 +30,8 @@ jobs:
       run: |
         pip install --upgrade pip
         pip install torch
-        pip install numpy filelock protobuf requests tqdm regex sentencepiece sacremoses tokenizers packaging
+        pip install numpy filelock protobuf requests tqdm regex sentencepiece sacremoses packaging
+        pip install tokenizers==0.9.0.rc2
 
     - name: Torch hub list
       run: |

.github/workflows/self-push.yml

@@ -59,6 +59,7 @@ jobs:
         TF_FORCE_GPU_ALLOW_GROWTH: "true"
         # TF_GPU_MEMORY_LIMIT: 4096
         OMP_NUM_THREADS: 1
+        USE_CUDA: yes
       run: |
         source .env/bin/activate
         python -m pytest -n 2 --dist=loadfile -s ./tests/
@@ -109,6 +110,7 @@ jobs:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
           # TF_GPU_MEMORY_LIMIT: 4096
           OMP_NUM_THREADS: 1
+          USE_CUDA: yes
         run: |
           source .env/bin/activate
           python -m pytest -n 2 --dist=loadfile -s ./tests/

.github/workflows/self-scheduled.yml

@@ -57,19 +57,21 @@ jobs:
         TF_FORCE_GPU_ALLOW_GROWTH: "true"
         OMP_NUM_THREADS: 1
         RUN_SLOW: yes
+        USE_CUDA: yes
       run: |
         source .env/bin/activate
-        python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0
+        python -m pytest -n 1 --dist=loadfile -s ./tests/
 
     - name: Run examples tests on GPU
       env:
         TF_FORCE_GPU_ALLOW_GROWTH: "true"
         OMP_NUM_THREADS: 1
         RUN_SLOW: yes
+        USE_CUDA: yes
       run: |
         source .env/bin/activate
         pip install -r examples/requirements.txt
-        python -m pytest -n 1 --dist=loadfile -s examples --durations=0
+        python -m pytest -n 1 --dist=loadfile -s examples
 
   run_all_tests_torch_and_tf_multiple_gpu:
     runs-on: [self-hosted, multi-gpu]
@@ -118,16 +120,18 @@ jobs:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
           OMP_NUM_THREADS: 1
           RUN_SLOW: yes
+          USE_CUDA: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0
+          python -m pytest -n 1 --dist=loadfile -s ./tests/
 
       - name: Run examples tests on GPU
         env:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
           OMP_NUM_THREADS: 1
           RUN_SLOW: yes
+          USE_CUDA: yes
         run: |
           source .env/bin/activate
           pip install -r examples/requirements.txt
-          python -m pytest -n 1 --dist=loadfile -s examples --durations=0
+          python -m pytest -n 1 --dist=loadfile -s examples

.gitignore

@@ -9,8 +9,7 @@ __pycache__/
 *.so
 
 # tests and logs
-tests/fixtures/*
-!tests/fixtures/sample_text_no_unicode.txt
+tests/fixtures
 logs/
 lightning_logs/
 lang_code_data/
@@ -157,6 +156,3 @@ debug.env
 
 #ctags
 tags
-
-# pre-commit
-.pre-commit*

README.md

@@ -81,7 +81,7 @@ This is another example of pipeline used for that can extract question answers f
 
 On top of the answer, the pretrained model used here returned its confidence score, along with the start position and its end position in the tokenized sentence. You can learn more about the tasks supported by the `pipeline` API in [this tutorial](https://huggingface.co/transformers/task_summary.html).
 
-To download and use any of the pretrained models on your given task, you just need to use those three lines of codes (PyTorch version):
+To download and use any of the pretrained models on your given task, you just need to use those three lines of codes (PyTorch verison):
 ```python
 >>> from transformers import AutoTokenizer, AutoModel
 
@@ -162,10 +162,10 @@ If you'd like to play with the examples, you must [install the library from sour
 1. **[BART](https://huggingface.co/transformers/model_doc/bart.html)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://arxiv.org/pdf/1910.13461.pdf) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer.
 1. **[BERT](https://huggingface.co/transformers/model_doc/bert.html)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.
 1. **[BERT For Sequence Generation](https://huggingface.co/transformers/model_doc/bertgeneration.html)** (from Google) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn.
-1. **[Blenderbot](https://huggingface.co/transformers/model_doc/blenderbot.html)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
+1. **[Blenderbot](https://huggingface.co/transformers/master/model_doc/blenderbot.html)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
 1. **[CamemBERT](https://huggingface.co/transformers/model_doc/camembert.html)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
 1. **[CTRL](https://huggingface.co/transformers/model_doc/ctrl.html)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
-1. **[DeBERTa](https://huggingface.co/transformers/model_doc/deberta.html)** (from Microsoft Research) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
+1. **[DeBERTa](https://huggingface.co/transformers/master/model_doc/deberta.html)** (from Microsoft Research) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
 1. **[DialoGPT](https://huggingface.co/transformers/model_doc/dialogpt.html)** (from Microsoft Research) released with the paper [DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation](https://arxiv.org/abs/1911.00536) by Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan.
 1. **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/master/examples/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT.
 1. **[DPR](https://huggingface.co/transformers/model_doc/dpr.html)** (from Facebook) released with the paper [Dense Passage Retrieval
@@ -182,15 +182,13 @@ Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
 1. **[MarianMT](https://huggingface.co/transformers/model_doc/marian.html)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team.
 1. **[MBart](https://huggingface.co/transformers/model_doc/mbart.html)** (from Facebook) released with the paper  [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
 1. **[Pegasus](https://huggingface.co/transformers/model_doc/pegasus.html)** (from Google) released with the paper [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization](https://arxiv.org/abs/1912.08777)> by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
-1. **[ProphetNet](https://huggingface.co/transformers/model_doc/prophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
 1. **[Reformer](https://huggingface.co/transformers/model_doc/reformer.html)** (from Google Research) released with the paper [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
 1. **[RoBERTa](https://huggingface.co/transformers/model_doc/roberta.html)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
 ultilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT.
-1. **[SqueezeBert](https://huggingface.co/transformers/model_doc/squeezebert.html)** released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://arxiv.org/abs/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
+1. **[SqueezeBert](https://huggingface.co/transformers/master/model_doc/squeezebert.html)** released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://arxiv.org/abs/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
 1. **[T5](https://huggingface.co/transformers/model_doc/t5.html)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
 1. **[Transformer-XL](https://huggingface.co/transformers/model_doc/transformerxl.html)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
 1. **[XLM](https://huggingface.co/transformers/model_doc/xlm.html)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau.
-1. **[XLM-ProphetNet](https://huggingface.co/transformers/model_doc/xlmprophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
 1. **[XLM-RoBERTa](https://huggingface.co/transformers/model_doc/xlmroberta.html)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov.
 1. **[XLNet](https://huggingface.co/transformers/model_doc/xlnet.html)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
 1. **[Other community models](https://huggingface.co/models)**, contributed by the [community](https://huggingface.co/users).

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'3.4.0'
+release = u'3.3.1'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/glossary.rst

@@ -218,52 +218,6 @@ positional embeddings.
 Absolute positional embeddings are selected in the range ``[0, config.max_position_embeddings - 1]``. Some models
 use other types of positional embeddings, such as sinusoidal position embeddings or relative position embeddings.
 
-.. _labels:
-
-Labels
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The labels are an optional argument which can be passed in order for the model to compute the loss itself. These labels
-should be the expected prediction of the model: it will use the standard loss in order to compute the loss between
-its predictions and the expected value (the label).
-
-These labels are different according to the model head, for example:
-
-- For sequence classification models (e.g., :class:`~transformers.BertForSequenceClassification`), the model expects
-  a tensor of dimension :obj:`(batch_size)` with each value of the batch corresponding to the expected label of the
-  entire sequence.
-- For token classification models (e.g., :class:`~transformers.BertForTokenClassification`), the model expects
-  a tensor of dimension :obj:`(batch_size, seq_length)` with each value corresponding to the expected label of each
-  individual token.
-- For masked language modeling (e.g., :class:`~transformers.BertForMaskedLM`), the model expects
-  a tensor of dimension :obj:`(batch_size, seq_length)` with each value corresponding to the expected label of each
-  individual token: the labels being the token ID for the masked token, and values to be ignored for the rest (usually
-  -100).
-- For sequence to sequence tasks,(e.g., :class:`~transformers.BartForConditionalGeneration`,
-  :class:`~transformers.MBartForConditionalGeneration`), the model expects a tensor of dimension
-  :obj:`(batch_size, tgt_seq_length)` with each value corresponding to the target sequences associated with each
-  input sequence. During training, both `BART` and `T5` will make the appropriate `decoder_input_ids` and decoder
-  attention masks internally. They usually do not need to be supplied. This does not apply to models leveraging the
-  Encoder-Decoder framework.
-  See the documentation of each model for more information on each specific model's labels.
-
-The base models (e.g., :class:`~transformers.BertModel`) do not accept labels, as these are the base transformer models,
-simply outputting features.
-
-.. _decoder-input-ids:
-
-Decoder input IDs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This input is specific to encoder-decoder models, and contains the input IDs that will be fed to the decoder.
-These inputs should be used for sequence to sequence tasks, such as translation or summarization, and are usually
-built in a way specific to each model.
-
-Most encoder-decoder models (BART, T5) create their :obj:`decoder_input_ids` on their own from the :obj:`labels`.
-In such models, passing the :obj:`labels` is the preferred way to handle training.
-
-Please check each model's docs to see how they handle these input IDs for sequence to sequence training.
-
 .. _feed-forward-chunking:
 
 Feed Forward Chunking

docs/source/index.rst

@@ -71,18 +71,19 @@ conversion utilities for the following models:
 4. :doc:`BERT For Sequence Generation <model_doc/bertgeneration>` (from Google) released with the paper `Leveraging
    Pre-trained Checkpoints for Sequence Generation Tasks <https://arxiv.org/abs/1907.12461>`__ by Sascha Rothe, Shashi
    Narayan, Aliaksei Severyn.
-5. :doc:`Blenderbot <model_doc/blenderbot>` (from Facebook) released with the paper `Recipes for building an
-   open-domain chatbot <https://arxiv.org/abs/2004.13637>`__ by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary
-   Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
+5. `Blenderbot <https://huggingface.co/transformers/master/model_doc/blenderbot.html>`__ (from Facebook) released with
+   the paper `Recipes for building an open-domain chatbot <https://arxiv.org/abs/2004.13637>`__ by Stephen Roller,
+   Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan
+   Boureau, Jason Weston.
 6. :doc:`CamemBERT <model_doc/camembert>` (from Inria/Facebook/Sorbonne) released with the paper `CamemBERT: a Tasty
    French Language Model <https://arxiv.org/abs/1911.03894>`__ by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz
    Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
 7. :doc:`CTRL <model_doc/ctrl>` (from Salesforce) released with the paper `CTRL: A Conditional Transformer Language
    Model for Controllable Generation <https://arxiv.org/abs/1909.05858>`__ by Nitish Shirish Keskar*, Bryan McCann*,
    Lav R. Varshney, Caiming Xiong and Richard Socher.
-8. :doc:`DeBERTa <model_doc/deberta>` (from Microsoft Research) released with the paper `DeBERTa: Decoding-enhanced
-   BERT with Disentangled Attention <https://arxiv.org/abs/2006.03654>`__ by Pengcheng He, Xiaodong Liu, Jianfeng Gao,
-   Weizhu Chen.
+8. `DeBERTa <https://huggingface.co/transformers/master/model_doc/deberta.html>`__ (from Microsoft Research) released
+   with the paper `DeBERTa: Decoding-enhanced BERT with Disentangled Attention <https://arxiv.org/abs/2006.03654>`__ by
+   Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
 9. :doc:`DialoGPT <model_doc/dialogpt>` (from Microsoft Research) released with the paper `DialoGPT: Large-Scale
    Generative Pre-training for Conversational Response Generation <https://arxiv.org/abs/1911.00536>`__ by Yizhe Zhang,
    Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan.
@@ -128,38 +129,32 @@ conversion utilities for the following models:
 22. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
     Gap-sentences for Abstractive Summarization <https://arxiv.org/abs/1912.08777>`__> by Jingqing Zhang, Yao Zhao,
     Mohammad Saleh and Peter J. Liu.
-23. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
-    Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi,
-    Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-24. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
+23. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
     Transformer <https://arxiv.org/abs/2001.04451>`__ by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
-25. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
+24. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
     Pretraining Approach <https://arxiv.org/abs/1907.11692>`__ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar
     Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. ultilingual BERT into `DistilmBERT
     <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German version of
     DistilBERT.
-26. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
-    about efficient neural networks? <https://arxiv.org/abs/2006.11316>`__ by Forrest N. Iandola, Albert E. Shaw, Ravi
-    Krishna, and Kurt W. Keutzer.
-27. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
+25. `SqueezeBert <https://huggingface.co/transformers/master/model_doc/squeezebert.html>`__ released with the paper
+    `SqueezeBERT: What can computer vision teach NLP about efficient neural networks?
+    <https://arxiv.org/abs/2006.11316>`__ by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
+26. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
     Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by Colin Raffel and Noam Shazeer and Adam
     Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
-28. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
+27. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
     Attentive Language Models Beyond a Fixed-Length Context <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*,
     Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
-29. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
+28. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
     Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis Conneau.
-30. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
-    Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan,
-    Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-31. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
+29. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
     Cross-lingual Representation Learning at Scale <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay
     Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke
     Zettlemoyer and Veselin Stoyanov.
-32. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `​XLNet: Generalized Autoregressive
+30. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `​XLNet: Generalized Autoregressive
     Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by Zhilin Yang*, Zihang Dai*, Yiming
     Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
-33. `Other community models <https://huggingface.co/models>`__, contributed by the `community
+31. `Other community models <https://huggingface.co/models>`__, contributed by the `community
     <https://huggingface.co/users>`__.
 
 .. toctree::
@@ -250,7 +245,6 @@ conversion utilities for the following models:
     model_doc/gpt
     model_doc/gpt2
     model_doc/pegasus
-    model_doc/prophetnet
     model_doc/rag
     model_doc/reformer
     model_doc/retribert
@@ -259,7 +253,6 @@ conversion utilities for the following models:
     model_doc/t5
     model_doc/transformerxl
     model_doc/xlm
-    model_doc/xlmprophetnet
     model_doc/xlmroberta
     model_doc/xlnet
 

docs/source/internal/trainer_utils.rst

@@ -19,9 +19,3 @@ Callbacks internals
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.trainer_callback.CallbackHandler
-
-Distributed Evaluation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.trainer_pt_utils.DistributedTensorGatherer
-    :members:

docs/source/model_doc/encoderdecoder.rst

@@ -27,4 +27,4 @@ EncoderDecoderModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.EncoderDecoderModel
-    :members: forward, from_encoder_decoder_pretrained
+    :members: forward

docs/source/model_doc/prophetnet.rst

@@ -1,83 +0,0 @@
-ProphetNet
------------------------------------------------------------------------------------------------------------------------
-
-**DISCLAIMER:** If you see something strange, file a `Github Issue
-<https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ and assign
-@patrickvonplaten
-
-Overview
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The ProphetNet model was proposed in `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou on 13 Jan, 2020.
-
-ProphetNet is an encoder-decoder model and can predict n-future tokens for "ngram" language modeling instead of just the next token.
-
-The abstract from the paper is the following:
-
-*In this paper, we present a new sequence-to-sequence pre-training model called ProphetNet, which introduces a novel self-supervised objective named future n-gram prediction and the proposed n-stream self-attention mechanism. Instead of the optimization of one-step ahead prediction in traditional sequence-to-sequence model, the ProphetNet is optimized by n-step ahead prediction which predicts the next n tokens simultaneously based on previous context tokens at each time step. The future n-gram prediction explicitly encourages the model to plan for the future tokens and prevent overfitting on strong local correlations. We pre-train ProphetNet using a base scale dataset (16GB) and a large scale dataset (160GB) respectively. Then we conduct experiments on CNN/DailyMail, Gigaword, and SQuAD 1.1 benchmarks for abstractive summarization and question generation tasks. Experimental results show that ProphetNet achieves new state-of-the-art results on all these datasets compared to the models using the same scale pre-training corpus.*
-
-The Authors' code can be found `here <https://github.com/microsoft/ProphetNet>`__.
-
-
-ProphetNetConfig
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ProphetNetConfig
-    :members:
-
-
-ProphetNetTokenizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ProphetNetTokenizer
-    :members:
-
-
-ProphetNet specific outputs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqLMOutput
-    :members:
-
-.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqModelOutput
-    :members:
-
-.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderModelOutput
-    :members:
-
-.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderLMOutput
-    :members:
-
-ProphetNetModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ProphetNetModel
-    :members: forward
-
-
-ProphetNetEncoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ProphetNetEncoder
-    :members: forward
-
-
-ProphetNetDecoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ProphetNetDecoder
-    :members: forward
-
-
-ProphetNetForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ProphetNetForConditionalGeneration
-    :members: forward
-
-
-ProphetNetForCausalLM
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ProphetNetForCausalLM
-    :members: forward

docs/source/model_doc/rag.rst

@@ -62,7 +62,8 @@ Rag specific outputs
 .. autoclass:: transformers.modeling_rag.RetrievAugLMOutput
     :members:
 
-RagRetriever
+
+RAGRetriever
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.RagRetriever

docs/source/model_doc/t5.rst

@@ -62,10 +62,10 @@ token. T5 can be trained / fine-tuned both in a supervised and unsupervised fash
 
 .. code-block::
 
-  input_ids = tokenizer('The <extra_id_0> walks in <extra_id_1> park', return_tensors='pt').input_ids
-  labels = tokenizer('<extra_id_0> cute dog <extra_id_1> the <extra_id_2>', return_tensors='pt').input_ids
+  input_ids = tokenizer.encode('The <extra_id_0> walks in <extra_id_1> park', return_tensors='pt')
+  labels = tokenizer.encode('<extra_id_0> cute dog <extra_id_1> the <extra_id_2> </s>', return_tensors='pt')
   # the forward function automatically creates the correct decoder_input_ids
-  loss = model(input_ids=input_ids, labels=labels, return_dict=True).loss
+  model(input_ids=input_ids, labels=labels)
 
 - Supervised training
 
@@ -75,10 +75,10 @@ token. T5 can be trained / fine-tuned both in a supervised and unsupervised fash
   
 .. code-block::
 
-  input_ids = tokenizer('translate English to German: The house is wonderful.', return_tensors='pt').input_ids
-  labels = tokenizer('Das Haus ist wunderbar.', return_tensors='pt').input_ids
+  input_ids = tokenizer.encode('translate English to German: The house is wonderful. </s>', return_tensors='pt')
+  labels = tokenizer.encode('Das Haus ist wunderbar. </s>', return_tensors='pt')
   # the forward function automatically creates the correct decoder_input_ids
-  loss = model(input_ids=input_ids, labels=labels, return_dict=True).loss
+  model(input_ids=input_ids, labels=labels)
 
 
 T5Config

docs/source/model_doc/xlmprophetnet.rst

@@ -1,63 +0,0 @@
-XLM-ProphetNet
------------------------------------------------------------------------------------------------------------------------
-
-**DISCLAIMER:** If you see something strange, file a `Github Issue
-<https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ and assign
-@patrickvonplaten
-
-
-Overview
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The XLM-ProphetNet model was proposed in `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou on 13 Jan, 2020.
-
-XLM-ProphetNet is an encoder-decoder model and can predict n-future tokens for "ngram" language modeling instead of just the next token. Its architecture is identical to ProhpetNet, but the model was trained on the multi-lingual "wiki100" Wikipedia dump.
-
-The abstract from the paper is the following:
-
-*In this paper, we present a new sequence-to-sequence pre-training model called ProphetNet, which introduces a novel self-supervised objective named future n-gram prediction and the proposed n-stream self-attention mechanism. Instead of the optimization of one-step ahead prediction in traditional sequence-to-sequence model, the ProphetNet is optimized by n-step ahead prediction which predicts the next n tokens simultaneously based on previous context tokens at each time step. The future n-gram prediction explicitly encourages the model to plan for the future tokens and prevent overfitting on strong local correlations. We pre-train ProphetNet using a base scale dataset (16GB) and a large scale dataset (160GB) respectively. Then we conduct experiments on CNN/DailyMail, Gigaword, and SQuAD 1.1 benchmarks for abstractive summarization and question generation tasks. Experimental results show that ProphetNet achieves new state-of-the-art results on all these datasets compared to the models using the same scale pre-training corpus.*
-
-The Authors' code can be found `here <https://github.com/microsoft/ProphetNet>`__.
-
-XLMProphetNetConfig
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLMProphetNetConfig
-    :members:
-
-
-XLMProphetNetTokenizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLMProphetNetTokenizer
-    :members:
-
-
-XLMProphetNetModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLMProphetNetModel
-
-
-XLMProphetNetEncoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLMProphetNetEncoder
-
-
-XLMProphetNetDecoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLMProphetNetDecoder
-
-
-XLMProphetNetForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLMProphetNetForConditionalGeneration
-
-
-XLMProphetNetForCausalLM
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.XLMProphetNetForCausalLM

docs/source/model_summary.rst

@@ -612,43 +612,6 @@ The `mbart-large-cc25 <https://huggingface.co/facebook/mbart-large-cc25>`_ check
 
 .. _multimodal-models:
 
-ProphetNet
------------------------------------------------------------------------------------------------------------------------
-
-.. raw:: html
-
-   <a href="https://huggingface.co/models?filter=prophetnet">
-       <img alt="Models" src="https://img.shields.io/badge/All_model_pages-prophetnet-blueviolet">
-   </a>
-   <a href="model_doc/prophetnet.html">
-       <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-prophetnet-blueviolet">
-   </a>
-
-`ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.
-
-ProphetNet introduces a novel *sequence-to-sequence* pre-training objective, called *future n-gram prediction*. In future n-gram prediction, the model predicts the next n tokens simultaneously based on previous context tokens at each time step instead instead of just the single next token. The future n-gram prediction explicitly encourages the model to plan for the future tokens and prevent overfitting on strong local correlations.
-The model architecture is based on the original Transformer, but replaces the "standard" self-attention mechanism in the decoder by a a main self-attention mechanism and a self and n-stream (predict) self-attention mechanism.
-
-The library provides a pre-trained version of this model for conditional generation and a fine-tuned version for summarization.
-
-XLM-ProphetNet
------------------------------------------------------------------------------------------------------------------------
-
-.. raw:: html
-
-   <a href="https://huggingface.co/models?filter=xprophetnet">
-       <img alt="Models" src="https://img.shields.io/badge/All_model_pages-xprophetnet-blueviolet">
-   </a>
-   <a href="model_doc/xlmprophetnet.html">
-       <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-xprophetnet-blueviolet">
-   </a>
-
-`ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.
-
-XLM-ProphetNet's model architecture and pre-training objective is same as ProphetNet, but XLM-ProphetNet was pre-trained on the cross-lingual dataset `XGLUE <https://arxiv.org/abs/2004.01401>`__.
-
-The library provides a pre-trained version of this model for multi-lingual conditional generation and fine-tuned versions for headline generation and question generation, respectively.
-
 Multimodal models
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/source/task_summary.rst

@@ -758,8 +758,8 @@ Here is an example of using the pipelines to do summarization. It leverages a Ba
     ... If convicted, Barrientos faces up to four years in prison.  Her next court appearance is scheduled for May 18.
     ... """
 
-Because the summarization pipeline depends on the ``PreTrainedModel.generate()`` method, we can override the default arguments
-of ``PreTrainedModel.generate()`` directly in the pipeline for ``max_length`` and ``min_length`` as shown below.
+Because the summarization pipeline depends on the ``PretrainedModel.generate()`` method, we can override the default arguments
+of ``PretrainedModel.generate()`` directly in the pipeline for ``max_length`` and ``min_length`` as shown below.
 This outputs the following summary:
 
 .. code-block::
@@ -772,7 +772,7 @@ Here is an example of doing summarization using a model and a tokenizer. The pro
 1. Instantiate a tokenizer and a model from the checkpoint name. Summarization is usually done using an encoder-decoder model, such as ``Bart`` or ``T5``.
 2. Define the article that should be summarized.
 3. Add the T5 specific prefix "summarize: ".
-4. Use the ``PreTrainedModel.generate()`` method to generate the summary.
+4. Use the ``PretrainedModel.generate()`` method to generate the summary.
 
 In this example we use Google`s T5 model. Even though it was pre-trained only on a multi-task mixed dataset (including CNN / Daily Mail), it yields very good results.
 
@@ -819,15 +819,15 @@ translation results.
     >>> print(translator("Hugging Face is a technology company based in New York and Paris", max_length=40))
     [{'translation_text': 'Hugging Face ist ein Technologieunternehmen mit Sitz in New York und Paris.'}]
 
-Because the translation pipeline depends on the ``PreTrainedModel.generate()`` method, we can override the default arguments
-of ``PreTrainedModel.generate()`` directly in the pipeline as is shown for ``max_length`` above.
+Because the translation pipeline depends on the ``PretrainedModel.generate()`` method, we can override the default arguments
+of ``PretrainedModel.generate()`` directly in the pipeline as is shown for ``max_length`` above.
 
 Here is an example of doing translation using a model and a tokenizer. The process is the following:
 
 1. Instantiate a tokenizer and a model from the checkpoint name. Summarization is usually done using an encoder-decoder model, such as ``Bart`` or ``T5``.
 2. Define the article that should be summarizaed.
 3. Add the T5 specific prefix "translate English to German: "
-4. Use the ``PreTrainedModel.generate()`` method to perform the translation.
+4. Use the ``PretrainedModel.generate()`` method to perform the translation.
 
 .. code-block::
 

docs/source/testing.rst

@@ -22,12 +22,12 @@ How transformers are tested
 
    * `self-hosted (push) <https://github.com/huggingface/transformers/blob/master/.github/workflows/self-push.yml>`__: runs fast tests on GPU only on commits on ``master``. It only runs if a commit on ``master`` has updated the code in one of the following folders: ``src``, ``tests``, ``.github`` (to prevent running on added model cards, notebooks, etc.)
      
-   * `self-hosted runner <https://github.com/huggingface/transformers/blob/master/.github/workflows/self-scheduled.yml>`__: runs normal and slow tests on GPU in ``tests`` and ``examples``:
+   * `self-hosted runner <https://github.com/huggingface/transformers/blob/master/.github/workflows/self-scheduled.yml>`__: runs slow tests on ``tests`` and ``examples``:
 
    .. code-block:: bash
 
-    RUN_SLOW=1 pytest tests/
-    RUN_SLOW=1 pytest examples/
+    RUN_SLOW=1 USE_CUDA=1 pytest tests/
+    RUN_SLOW=1 USE_CUDA=1 pytest examples/
 
    The results can be observed `here <https://github.com/huggingface/transformers/actions>`__.
 
@@ -393,53 +393,36 @@ On a GPU-enabled setup, to test in CPU-only mode add ``CUDA_VISIBLE_DEVICES=""``
                 
     CUDA_VISIBLE_DEVICES="" pytest tests/test_logging.py
 
-or if you have multiple gpus, you can specify which one is to be used by ``pytest``. For example, to use only the second gpu if you have gpus ``0`` and ``1``, you can run:
+or if you have multiple gpus, you can tell which one to use in this test session, e.g. to use only the second gpu if you have gpus ``0`` and ``1``, you can run:
 
 .. code-block:: bash
                 
     CUDA_VISIBLE_DEVICES="1" pytest tests/test_logging.py
 
 This is handy when you want to run different tasks on different GPUs.
+    
+And we have these decorators that require the condition described by the marker.
 
-Some tests must be run on CPU-only, others on either CPU or GPU or TPU, yet others on multiple-GPUs. The following skip decorators are used to set the requirements of tests CPU/GPU/TPU-wise:
-
-* ``require_torch`` - this test will run only under torch
-* ``require_torch_gpu`` - as ``require_torch`` plus requires at least 1 GPU
-* ``require_torch_multigpu`` - as ``require_torch`` plus requires at least 2 GPUs
-* ``require_torch_non_multigpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
-* ``require_torch_tpu`` - as ``require_torch`` plus requires at least 1 TPU
-
-For example, here is a test that must be run only when there are 2 or more GPUs available and pytorch is installed:
-
-.. code-block:: python
-
-    @require_torch_multigpu
-    def test_example_with_multigpu():
-
-If a test requires ``tensorflow`` use the ``require_tf`` decorator. For example:
-
-.. code-block:: python
-
-    @require_tf
-    def test_tf_thing_with_tensorflow():
-
-These decorators can be stacked. For example, if a test is slow and requires at least one GPU under pytorch, here is how to set it up:
-
-.. code-block:: python
-
-    @require_torch_gpu
-    @slow
-    def test_example_slow_on_gpu():
+``
+@require_torch
+@require_tf
+@require_multigpu
+@require_non_multigpu
+@require_torch_tpu
+@require_torch_and_cuda
+``
 
 Some decorators like ``@parametrized`` rewrite test names, therefore ``@require_*`` skip decorators have to be listed last for them to work correctly. Here is an example of the correct usage:
 
 .. code-block:: python
 
     @parameterized.expand(...)
-    @require_torch_multigpu
+    @require_multigpu
     def test_integration_foo():
+    
+There is no problem whatsoever with ``@pytest.mark.parametrize`` (but it only works with non-unittests) - can use it in any order.
 
-This order problem doesn't exist with ``@pytest.mark.parametrize``, you can put it first or last and it will still work. But it only works with non-unittests.
+This section will be expanded soon once our work in progress on those decorators is finished.
 
 Inside tests:
 

docs/source/training.rst

@@ -109,9 +109,9 @@ The following is equivalent to the previous example:
 .. code-block:: python
 
     from torch.nn import functional as F
-    labels = torch.tensor([1,0])
+    labels = torch.tensor([1,0]).unsqueeze(0)
     outputs = model(input_ids, attention_mask=attention_mask)
-    loss = F.cross_entropy(outputs.logits, labels)
+    loss = F.cross_entropy(labels, outputs.logitd)
     loss.backward()
     optimizer.step()
 

examples/conftest.py

@@ -2,7 +2,6 @@
 # by pytest before any tests are run
 
 import sys
-import warnings
 from os.path import abspath, dirname, join
 
 
@@ -10,7 +9,3 @@ from os.path import abspath, dirname, join
 # 'pip install -e .[dev]' when switching between checkouts and running tests.
 git_repo_path = abspath(join(dirname(dirname(__file__)), "src"))
 sys.path.insert(1, git_repo_path)
-
-# silence FutureWarning warnings in tests since often we can't act on them until
-# they become normal warnings - i.e. the tests still need to test the current functionality
-warnings.simplefilter(action="ignore", category=FutureWarning)

examples/lightning_base.py

@@ -291,8 +291,7 @@ class LoggingCallback(pl.Callback):
 
 
 def add_generic_args(parser, root_dir) -> None:
-    #  To allow all pl args uncomment the following line
-    #  parser = pl.Trainer.add_argparse_args(parser)
+    #  TODO(SS): allow all pl args? parser = pl.Trainer.add_argparse_args(parser)
     parser.add_argument(
         "--output_dir",
         default=None,

examples/rag/README.md

@@ -65,41 +65,26 @@ Does He Love You	Does He Love You	Red Sandy Spika dress of Reba McEntire	Greates
 We demonstrate how to evaluate retrieval against DPR evaluation data. You can download respective files from links listed [here](https://github.com/facebookresearch/DPR/blob/master/data/download_data.py#L39-L45).
 
 1. Download and unzip the gold data file. We use the `biencoder-nq-dev` from https://dl.fbaipublicfiles.com/dpr/data/retriever/biencoder-nq-dev.json.gz.
-    ```bash
-    wget https://dl.fbaipublicfiles.com/dpr/data/retriever/biencoder-nq-dev.json.gz && gzip -d biencoder-nq-dev.json.gz
-   ```
-
 2. Parse the unziped file using the `parse_dpr_relevance_data.py`
     ```bash
-    mkdir output # or wherever you want to save this
     python examples/rag/parse_dpr_relevance_data.py \
-        --src_path biencoder-nq-dev.json \
-        --evaluation_set output/biencoder-nq-dev.questions \
-        --gold_data_path output/biencoder-nq-dev.pages
+        --src_path path/to/unziped/biencoder-nq-dev.json \
+        --evaluation_set path/to/output/biencoder-nq-dev.questions \
+        --gold_data_path path/to/output/biencoder-nq-dev.pages
     ```
 3. Run evaluation:
-    ```bash    
-    python examples/rag/eval_rag.py \
-        --model_name_or_path facebook/rag-sequence-nq \
-        --model_type rag_sequence \
-        --evaluation_set output/biencoder-nq-dev.questions \
-        --gold_data_path output/biencoder-nq-dev.pages \
-        --predictions_path output/retrieval_preds.tsv  \
-        --eval_mode retrieval \
-        --k 1
-    ```
-   ```bash
-   # EXPLANATION
+    ```bash
     python examples/rag/eval_rag.py \
         --model_name_or_path facebook/rag-sequence-nq \ # model name or path of the model we're evaluating
         --model_type rag_sequence \ # RAG model type (rag_token or rag_sequence)
-        --evaluation_set output/biencoder-nq-dev.questions \ # an input dataset for evaluation
-        --gold_data_path poutput/biencoder-nq-dev.pages \ # a dataset containing ground truth answers for samples from the evaluation_set
-        --predictions_path output/retrieval_preds.tsv  \ # name of file where predictions will be stored
+        --evaluation_set path/to/output/biencoder-nq-dev.questions \ # an input dataset for evaluation
+        --gold_data_path path/to/output/biencoder-nq-dev.pages \ # a dataset containing ground truth answers for samples from the evaluation_set
+        --predictions_path path/to/retrieval_preds.tsv  \ # name of file where predictions will be stored
         --eval_mode retrieval \ # indicates whether we're performing retrieval evaluation or e2e evaluation
         --k 1 # parameter k for the precision@k metric
-   
     ```
+
+
 ## End-to-end evaluation
 
 We support two formats of the gold data file (controlled by the `gold_data_mode` parameter):
@@ -112,9 +97,7 @@ who is the owner of reading football club	['Xiu Li Dai', 'Dai Yongge', 'Dai Xiul
 Xiu Li Dai
 ```
 
-Predictions of the model for the samples from the `evaluation_set` will be saved under the path specified by the `predictions_path` parameter. 
-If this path already exists, the script will use saved predictions to calculate metrics. 
-Add `--recalculate` parameter to force the script to perform inference from scratch.
+Predictions of the model for the samples from the `evaluation_set` will be saved under the path specified by the `predictions_path` parameter. If this path already exists, the script will use saved predictions to calculate metrics. Add `--recalculate` parameter to force the script to perform inference from scratch.
 
 An example e2e evaluation run could look as follows:
 ```bash

examples/rag/__init__.py

@@ -1,5 +0,0 @@
-import os
-import sys
-
-
-sys.path.insert(1, os.path.dirname(os.path.realpath(__file__)))

examples/rag/distributed_retriever.py

@@ -27,18 +27,13 @@ class RagPyTorchDistributedRetriever(RagRetriever):
             It is used to decode the question and then use the generator_tokenizer.
         generator_tokenizer (:class:`~transformers.PretrainedTokenizer`):
             The tokenizer used for the generator part of the RagModel.
-        index (:class:`~transformers.retrieval_rag.Index`, optional, defaults to the one defined by the configuration):
-            If specified, use this index instead of the one built using the configuration
     """
 
     _init_retrieval = False
 
-    def __init__(self, config, question_encoder_tokenizer, generator_tokenizer, index=None):
+    def __init__(self, config, question_encoder_tokenizer, generator_tokenizer):
         super().__init__(
-            config,
-            question_encoder_tokenizer=question_encoder_tokenizer,
-            generator_tokenizer=generator_tokenizer,
-            index=index,
+            config, question_encoder_tokenizer=question_encoder_tokenizer, generator_tokenizer=generator_tokenizer
         )
 
         self.process_group = None

examples/rag/eval_rag.py

@@ -15,7 +15,7 @@ from transformers import logging as transformers_logging
 
 
 sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # isort:skip
-from utils import exact_match_score, f1_score  # noqa: E402 # isort:skip
+from examples.rag.utils import exact_match_score, f1_score  # noqa: E402 # isort:skip
 
 
 logger = logging.getLogger(__name__)
@@ -72,7 +72,7 @@ def get_precision_at_k(args, preds_path, gold_data_path):
     em = total = 0
     for hypo, reference in zip(hypos, references):
         hypo_provenance = set(hypo.split("\t")[:k])
-        ref_provenance = set(reference.split("\t"))
+        ref_provenance = set(reference.split("\t")[1 : (k + 1)])
         total += 1
         em += len(hypo_provenance & ref_provenance) / k
 

examples/rag/finetune.py

@@ -31,13 +31,16 @@ from transformers import (
 from transformers import logging as transformers_logging
 
 
-from callbacks import (  # noqa: E402 # isort:skipq
+sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip
+
+from examples.lightning_base import BaseTransformer, add_generic_args, generic_train  # noqa: E402 # isort:skip
+from examples.rag.callbacks import (  # noqa: E402 # isort:skip
     get_checkpoint_callback,
     get_early_stopping_callback,
     Seq2SeqLoggingCallback,
 )
-from distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
-from utils import (  # noqa: E402 # isort:skip
+from examples.rag.distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
+from examples.rag.utils import (  # noqa: E402 # isort:skip
     calculate_exact_match,
     flatten_list,
     get_git_info,
@@ -50,11 +53,6 @@ from utils import (  # noqa: E402 # isort:skip
     Seq2SeqDataset,
 )
 
-# need the parent dir module
-sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
-from lightning_base import BaseTransformer, add_generic_args, generic_train  # noqa
-
-
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
@@ -90,11 +88,6 @@ class GenerativeQAModule(BaseTransformer):
         config_class = RagConfig if self.is_rag_model else AutoConfig
         config = config_class.from_pretrained(hparams.model_name_or_path)
 
-        # set retriever parameters
-        config.index_name = args.index_name or config.index_name
-        config.passages_path = args.passages_path or config.passages_path
-        config.index_path = args.index_path or config.index_path
-
         # set extra_model_params for generator configs and load_model
         extra_model_params = ("encoder_layerdrop", "decoder_layerdrop", "attention_dropout", "dropout")
         if self.is_rag_model:
@@ -102,7 +95,7 @@ class GenerativeQAModule(BaseTransformer):
                 config.generator.prefix = args.prefix
             config.label_smoothing = hparams.label_smoothing
             hparams, config.generator = set_extra_model_params(extra_model_params, hparams, config.generator)
-            retriever = RagPyTorchDistributedRetriever.from_pretrained(hparams.model_name_or_path, config=config)
+            retriever = RagPyTorchDistributedRetriever.from_pretrained(hparams.model_name_or_path)
             model = self.model_class.from_pretrained(hparams.model_name_or_path, config=config, retriever=retriever)
             prefix = config.question_encoder.prefix
         else:
@@ -410,28 +403,6 @@ class GenerativeQAModule(BaseTransformer):
         )
         return parser
 
-    @staticmethod
-    def add_retriever_specific_args(parser):
-        parser.add_argument(
-            "--index_name",
-            type=str,
-            default=None,
-            help="Name of the index to use: 'hf' for a canonical dataset from the datasets library (default), 'custom' for a local index, or 'legacy' for the orignal one)",
-        )
-        parser.add_argument(
-            "--passages_path",
-            type=str,
-            default=None,
-            help="Path to the dataset of passages for custom index. More info about custom indexes in the RagRetriever documentation as well as in `examples/rag/use_own_knowledge_dataset.py`",
-        )
-        parser.add_argument(
-            "--index_path",
-            type=str,
-            default=None,
-            help="Path to the faiss index for custom index. More info about custom indexes in the RagRetriever documentation as well as in `examples/rag/use_own_knowledge_dataset.py`",
-        )
-        return parser
-
 
 def main(args, model=None) -> GenerativeQAModule:
     Path(args.output_dir).mkdir(exist_ok=True)
@@ -492,7 +463,6 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser = pl.Trainer.add_argparse_args(parser)
     parser = GenerativeQAModule.add_model_specific_args(parser, os.getcwd())
-    parser = GenerativeQAModule.add_retriever_specific_args(parser)
 
     args = parser.parse_args()
 

examples/rag/test_data/my_knowledge_dataset.csv

@@ -1,2 +0,0 @@
-Aaron	Aaron Aaron ( or ; "Ahärôn") is a prophet, high priest, and the brother of Moses in the Abrahamic religions. Knowledge of Aaron, along with his brother Moses, comes exclusively from religious texts, such as the Bible and Quran. The Hebrew Bible relates that, unlike Moses, who grew up in the Egyptian royal court, Aaron and his elder sister Miriam remained with their kinsmen in the eastern border-land of Egypt (Goshen). When Moses first confronted the Egyptian king about the Israelites, Aaron served as his brother's spokesman ("prophet") to the Pharaoh. Part of the Law (Torah) that Moses received from God at Sinai granted Aaron the priesthood for himself and his male descendants, and he became the first High Priest of the Israelites. Aaron died before the Israelites crossed the North Jordan river and he was buried on Mount Hor (Numbers 33:39; Deuteronomy 10:6 says he died and was buried at Moserah). Aaron is also mentioned in the New Testament of the Bible. According to the Book of Exodus, Aaron first functioned as Moses' assistant. Because Moses complained that he could not speak well, God appointed Aaron as Moses' "prophet" (Exodus 4:10-17; 7:1). At the command of Moses, he let his rod turn into a snake. Then he stretched out his rod in order to bring on the first three plagues. After that, Moses tended to act and speak for himself. During the journey in the wilderness, Aaron was not always prominent or active. At the battle with Amalek, he was chosen with Hur to support the hand of Moses that held the "rod of God". When the revelation was given to Moses at biblical Mount Sinai, he headed the elders of Israel who accompanied Moses on the way to the summit.
-"Pokémon"	Pokémon , also known as in Japan, is a media franchise managed by The Pokémon Company, a Japanese consortium between Nintendo, Game Freak, and Creatures. The franchise copyright is shared by all three companies, but Nintendo is the sole owner of the trademark. The franchise was created by Satoshi Tajiri in 1995, and is centered on fictional creatures called "Pokémon", which humans, known as Pokémon Trainers, catch and train to battle each other for sport. The English slogan for the franchise is "Gotta Catch 'Em All". Works within the franchise are set in the Pokémon universe. The franchise began as "Pokémon Red" and "Green" (released outside of Japan as "Pokémon Red" and "Blue"), a pair of video games for the original Game Boy that were developed by Game Freak and published by Nintendo in February 1996. "Pokémon" has since gone on to become the highest-grossing media franchise of all time, with over in revenue up until March 2017. The original video game series is the second best-selling video game franchise (behind Nintendo's "Mario" franchise) with more than 300million copies sold and over 800million mobile downloads. In addition, the "Pokémon" franchise includes the world's top-selling toy brand, the top-selling trading card game with over 25.7billion cards sold, an anime television series that has become the most successful video game adaptation with over 20 seasons and 1,000 episodes in 124 countries, as well as an anime film series, a , books, manga comics, music, and merchandise. The franchise is also represented in other Nintendo media, such as the "Super Smash Bros." series. In November 2005, 4Kids Entertainment, which had managed the non-game related licensing of "Pokémon", announced that it had agreed not to renew the "Pokémon" representation agreement. The Pokémon Company International oversees all "Pokémon" licensing outside Asia.

examples/rag/test_distributed_retriever.py

@@ -15,7 +15,6 @@ from transformers.configuration_bart import BartConfig
 from transformers.configuration_dpr import DPRConfig
 from transformers.configuration_rag import RagConfig
 from transformers.file_utils import is_datasets_available, is_faiss_available, is_psutil_available, is_torch_available
-from transformers.retrieval_rag import CustomHFIndex
 from transformers.tokenization_bart import BartTokenizer
 from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
 from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer
@@ -24,7 +23,7 @@ from transformers.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FI
 
 sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip
 
-from distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
+from examples.rag.distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
 
 
 def require_distributed_retrieval(test_case):
@@ -115,7 +114,7 @@ class RagRetrieverTest(TestCase):
     def tearDown(self):
         shutil.rmtree(self.tmpdirname)
 
-    def get_dummy_dataset(self):
+    def get_dummy_pytorch_distributed_retriever(self, init_retrieval, port=12345) -> RagPyTorchDistributedRetriever:
         dataset = Dataset.from_dict(
             {
                 "id": ["0", "1"],
@@ -125,12 +124,6 @@ class RagRetrieverTest(TestCase):
             }
         )
         dataset.add_faiss_index("embeddings", string_factory="Flat", metric_type=faiss.METRIC_INNER_PRODUCT)
-        return dataset
-
-    def get_dummy_pytorch_distributed_retriever(
-        self, init_retrieval: bool, port=12345
-    ) -> RagPyTorchDistributedRetriever:
-        dataset = self.get_dummy_dataset()
         config = RagConfig(
             retrieval_vector_size=self.retrieval_vector_size,
             question_encoder=DPRConfig().to_dict(),
@@ -147,37 +140,6 @@ class RagRetrieverTest(TestCase):
                 retriever.init_retrieval(port)
         return retriever
 
-    def get_dummy_custom_hf_index_retriever(self, init_retrieval: bool, from_disk: bool, port=12345):
-        dataset = self.get_dummy_dataset()
-        config = RagConfig(
-            retrieval_vector_size=self.retrieval_vector_size,
-            question_encoder=DPRConfig().to_dict(),
-            generator=BartConfig().to_dict(),
-            index_name="custom",
-        )
-        if from_disk:
-            config.passages_path = os.path.join(self.tmpdirname, "dataset")
-            config.index_path = os.path.join(self.tmpdirname, "index.faiss")
-            dataset.get_index("embeddings").save(os.path.join(self.tmpdirname, "index.faiss"))
-            dataset.drop_index("embeddings")
-            dataset.save_to_disk(os.path.join(self.tmpdirname, "dataset"))
-            del dataset
-            retriever = RagPyTorchDistributedRetriever(
-                config,
-                question_encoder_tokenizer=self.get_dpr_tokenizer(),
-                generator_tokenizer=self.get_bart_tokenizer(),
-            )
-        else:
-            retriever = RagPyTorchDistributedRetriever(
-                config,
-                question_encoder_tokenizer=self.get_dpr_tokenizer(),
-                generator_tokenizer=self.get_bart_tokenizer(),
-                index=CustomHFIndex(config.retrieval_vector_size, dataset),
-            )
-        if init_retrieval:
-            retriever.init_retrieval(port)
-        return retriever
-
     def test_pytorch_distributed_retriever_retrieve(self):
         n_docs = 1
         retriever = self.get_dummy_pytorch_distributed_retriever(init_retrieval=True)
@@ -192,33 +154,3 @@ class RagRetrieverTest(TestCase):
         self.assertEqual(doc_dicts[0]["id"][0], "1")  # max inner product is reached with second doc
         self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
         self.assertListEqual(doc_ids.tolist(), [[1], [0]])
-
-    def test_custom_hf_index_retriever_retrieve(self):
-        n_docs = 1
-        retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=False)
-        hidden_states = np.array(
-            [np.ones(self.retrieval_vector_size), -np.ones(self.retrieval_vector_size)], dtype=np.float32
-        )
-        retrieved_doc_embeds, doc_ids, doc_dicts = retriever.retrieve(hidden_states, n_docs=n_docs)
-        self.assertEqual(retrieved_doc_embeds.shape, (2, n_docs, self.retrieval_vector_size))
-        self.assertEqual(len(doc_dicts), 2)
-        self.assertEqual(sorted(doc_dicts[0]), ["embeddings", "id", "text", "title"])
-        self.assertEqual(len(doc_dicts[0]["id"]), n_docs)
-        self.assertEqual(doc_dicts[0]["id"][0], "1")  # max inner product is reached with second doc
-        self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
-        self.assertListEqual(doc_ids.tolist(), [[1], [0]])
-
-    def test_custom_pytorch_distributed_retriever_retrieve_from_disk(self):
-        n_docs = 1
-        retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=True)
-        hidden_states = np.array(
-            [np.ones(self.retrieval_vector_size), -np.ones(self.retrieval_vector_size)], dtype=np.float32
-        )
-        retrieved_doc_embeds, doc_ids, doc_dicts = retriever.retrieve(hidden_states, n_docs=n_docs)
-        self.assertEqual(retrieved_doc_embeds.shape, (2, n_docs, self.retrieval_vector_size))
-        self.assertEqual(len(doc_dicts), 2)
-        self.assertEqual(sorted(doc_dicts[0]), ["embeddings", "id", "text", "title"])
-        self.assertEqual(len(doc_dicts[0]["id"]), n_docs)
-        self.assertEqual(doc_dicts[0]["id"][0], "1")  # max inner product is reached with second doc
-        self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
-        self.assertListEqual(doc_ids.tolist(), [[1], [0]])

examples/rag/use_own_knowledge_dataset.py

@@ -1,199 +0,0 @@
-import logging
-import os
-from dataclasses import dataclass, field
-from functools import partial
-from pathlib import Path
-from tempfile import TemporaryDirectory
-from typing import List, Optional
-
-import torch
-from datasets import load_dataset
-
-import faiss
-from transformers import (
-    DPRContextEncoder,
-    DPRContextEncoderTokenizerFast,
-    HfArgumentParser,
-    RagRetriever,
-    RagSequenceForGeneration,
-    RagTokenizer,
-)
-
-
-logger = logging.getLogger(__name__)
-torch.set_grad_enabled(False)
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-
-def split_text(text: str, n=100, character=" ") -> List[str]:
-    """Split the text every ``n``-th occurence of ``character``"""
-    text = text.split(character)
-    return [character.join(text[i : i + n]).strip() for i in range(0, len(text), n)]
-
-
-def split_documents(documents: dict) -> dict:
-    """Split documents into passages"""
-    titles, texts = [], []
-    for title, text in zip(documents["title"], documents["text"]):
-        for passage in split_text(text):
-            titles.append(title)
-            texts.append(passage)
-    return {"title": titles, "text": texts}
-
-
-def embed(documents: dict, ctx_encoder: DPRContextEncoder, ctx_tokenizer: DPRContextEncoderTokenizerFast) -> dict:
-    """Compute the DPR embeddings of document passages"""
-    input_ids = ctx_tokenizer(
-        documents["title"], documents["text"], truncation=True, padding="longest", return_tensors="pt"
-    )["input_ids"]
-    embeddings = ctx_encoder(input_ids.to(device=device), return_dict=True).pooler_output
-    return {"embeddings": embeddings.detach().cpu().numpy()}
-
-
-def main(
-    rag_example_args: "RagExampleArguments",
-    processing_args: "ProcessingArguments",
-    index_hnsw_args: "IndexHnswArguments",
-):
-
-    ######################################
-    logger.info("Step 1 - Create the dataset")
-    ######################################
-
-    # The dataset needed for RAG must have three columns:
-    # - title (string): title of the document
-    # - text (string): text of a passage of the document
-    # - embeddings (array of dimension d): DPR representation of the passage
-
-    # Let's say you have documents in tab-separated csv files with columns "title" and "text"
-    assert os.path.isfile(rag_example_args.csv_path), "Please provide a valid path to a csv file"
-
-    # You can load a Dataset object this way
-    dataset = load_dataset(
-        "csv", data_files=[rag_example_args.csv_path], split="train", delimiter="\t", column_names=["title", "text"]
-    )
-
-    # More info about loading csv files in the documentation: https://huggingface.co/docs/datasets/loading_datasets.html?highlight=csv#csv-files
-
-    # Then split the documents into passages of 100 words
-    dataset = dataset.map(split_documents, batched=True, num_proc=processing_args.num_proc)
-
-    # And compute the embeddings
-    ctx_encoder = DPRContextEncoder.from_pretrained(rag_example_args.dpr_ctx_encoder_model_name).to(device=device)
-    ctx_tokenizer = DPRContextEncoderTokenizerFast.from_pretrained(rag_example_args.dpr_ctx_encoder_model_name)
-    dataset = dataset.map(
-        partial(embed, ctx_encoder=ctx_encoder, ctx_tokenizer=ctx_tokenizer),
-        batched=True,
-        batch_size=processing_args.batch_size,
-    )
-
-    # And finally save your dataset
-    passages_path = os.path.join(rag_example_args.output_dir, "my_knowledge_dataset")
-    dataset.save_to_disk(passages_path)
-    # from datasets import load_from_disk
-    # dataset = load_from_disk(passages_path)  # to reload the dataset
-
-    ######################################
-    logger.info("Step 2 - Index the dataset")
-    ######################################
-
-    # Let's use the Faiss implementation of HNSW for fast approximate nearest neighbor search
-    index = faiss.IndexHNSWFlat(index_hnsw_args.d, index_hnsw_args.m, faiss.METRIC_INNER_PRODUCT)
-    dataset.add_faiss_index("embeddings", custom_index=index)
-
-    # And save the index
-    index_path = os.path.join(rag_example_args.output_dir, "my_knowledge_dataset_hnsw_index.faiss")
-    dataset.get_index("embeddings").save(index_path)
-    # dataset.load_faiss_index("embeddings", index_path)  # to reload the index
-
-    ######################################
-    logger.info("Step 3 - Load RAG")
-    ######################################
-
-    # Easy way to load the model
-    retriever = RagRetriever.from_pretrained(
-        rag_example_args.rag_model_name, index_name="custom", indexed_dataset=dataset
-    )
-    model = RagSequenceForGeneration.from_pretrained(rag_example_args.rag_model_name, retriever=retriever)
-    tokenizer = RagTokenizer.from_pretrained(rag_example_args.rag_model_name)
-
-    # For distributed fine-tuning you'll need to provide the paths instead, as the dataset and the index are loaded separately.
-    # retriever = RagRetriever.from_pretrained(rag_model_name, index_name="custom", passages_path=passages_path, index_path=index_path)
-
-    ######################################
-    logger.info("Step 4 - Have fun")
-    ######################################
-
-    question = rag_example_args.question or "What does Moses' rod turn into ?"
-    input_ids = tokenizer.question_encoder(question, return_tensors="pt")["input_ids"]
-    generated = model.generate(input_ids)
-    generated_string = tokenizer.batch_decode(generated, skip_special_tokens=True)[0]
-    logger.info("Q: " + question)
-    logger.info("A: " + generated_string)
-
-
-@dataclass
-class RagExampleArguments:
-    csv_path: str = field(
-        default=str(Path(__file__).parent / "test_data" / "my_knowledge_dataset.csv"),
-        metadata={"help": "Path to a tab-separated csv file with columns 'title' and 'text'"},
-    )
-    question: Optional[str] = field(
-        default=None,
-        metadata={"help": "Question that is passed as input to RAG. Default is 'What does Moses' rod turn into ?'."},
-    )
-    rag_model_name: str = field(
-        default="facebook/rag-sequence-nq",
-        metadata={"help": "The RAG model to use. Either 'facebook/rag-sequence-nq' or 'facebook/rag-token-nq'"},
-    )
-    dpr_ctx_encoder_model_name: str = field(
-        default="facebook/dpr-ctx_encoder-multiset-base",
-        metadata={
-            "help": "The DPR context encoder model to use. Either 'facebook/dpr-ctx_encoder-single-nq-base' or 'facebook/dpr-ctx_encoder-multiset-base'"
-        },
-    )
-    output_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Path to a directory where the dataset passages and the index will be saved"},
-    )
-
-
-@dataclass
-class ProcessingArguments:
-    num_proc: Optional[int] = field(
-        default=None,
-        metadata={
-            "help": "The number of processes to use to split the documents into passages. Default is single process."
-        },
-    )
-    batch_size: int = field(
-        default=16,
-        metadata={
-            "help": "The batch size to use when computing the passages embeddings using the DPR context encoder."
-        },
-    )
-
-
-@dataclass
-class IndexHnswArguments:
-    d: int = field(
-        default=768,
-        metadata={"help": "The dimension of the embeddings to pass to the HNSW Faiss index."},
-    )
-    m: int = field(
-        default=128,
-        metadata={
-            "help": "The number of bi-directional links created for every new element during the HNSW index construction."
-        },
-    )
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.WARNING)
-    logger.setLevel(logging.INFO)
-
-    parser = HfArgumentParser((RagExampleArguments, ProcessingArguments, IndexHnswArguments))
-    rag_example_args, processing_args, index_hnsw_args = parser.parse_args_into_dataclasses()
-    with TemporaryDirectory() as tmp_dir:
-        rag_example_args.output_dir = rag_example_args.output_dir or tmp_dir
-        main(rag_example_args, processing_args, index_hnsw_args)

examples/requirements.txt

@@ -17,4 +17,3 @@ datasets
 fire
 pytest
 conllu
-sentencepiece != 0.1.92

examples/seq2seq/seq2seq_trainer.py

@@ -174,7 +174,7 @@ class Seq2SeqTrainer(Trainer):
             # Call forward again to get loss # TODO: avoidable?
             outputs = model(**inputs, use_cache=False)
             loss = self._compute_loss(outputs[1], labels_out)
-            loss = loss.mean().detach()
+            loss = loss.mean().item()
             if self.args.prediction_loss_only:
                 return (loss, None, None)
 

examples/seq2seq/test_bash_script.py

@@ -3,6 +3,7 @@
 import argparse
 import os
 import sys
+import tempfile
 from pathlib import Path
 from unittest.mock import patch
 
@@ -15,172 +16,172 @@ from distillation import BartSummarizationDistiller, distill_main
 from finetune import SummarizationModule, main
 from test_seq2seq_examples import CUDA_AVAILABLE, MBART_TINY
 from transformers import BartForConditionalGeneration, MarianMTModel
-from transformers.testing_utils import TestCasePlus, slow
+from transformers.testing_utils import slow
 from utils import load_json
 
 
 MODEL_NAME = MBART_TINY
+# TODO(SS): MODEL_NAME = "sshleifer/student_mbart_en_ro_1_1"
 MARIAN_MODEL = "sshleifer/student_marian_en_ro_6_1"
 
 
-class TestAll(TestCasePlus):
-    @slow
-    @pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
-    def test_model_download(self):
-        """This warms up the cache so that we can time the next test without including download time, which varies between machines."""
-        BartForConditionalGeneration.from_pretrained(MODEL_NAME)
-        MarianMTModel.from_pretrained(MARIAN_MODEL)
+@slow
+@pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
+def test_model_download():
+    """This warms up the cache so that we can time the next test without including download time, which varies between machines."""
+    BartForConditionalGeneration.from_pretrained(MODEL_NAME)
+    MarianMTModel.from_pretrained(MARIAN_MODEL)
 
-    @timeout_decorator.timeout(120)
-    @slow
-    @pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
-    def test_train_mbart_cc25_enro_script(self):
-        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
-        env_vars_to_replace = {
-            "--fp16_opt_level=O1": "",
-            "$MAX_LEN": 128,
-            "$BS": 4,
-            "$GAS": 1,
-            "$ENRO_DIR": data_dir,
-            "facebook/mbart-large-cc25": MODEL_NAME,
-            # Download is 120MB in previous test.
-            "val_check_interval=0.25": "val_check_interval=1.0",
-        }
 
-        # Clean up bash script
-        bash_script = Path("examples/seq2seq/train_mbart_cc25_enro.sh").open().read().split("finetune.py")[1].strip()
-        bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
-        for k, v in env_vars_to_replace.items():
-            bash_script = bash_script.replace(k, str(v))
-        output_dir = self.get_auto_remove_tmp_dir()
+@timeout_decorator.timeout(120)
+@slow
+@pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
+def test_train_mbart_cc25_enro_script():
+    data_dir = "examples/seq2seq/test_data/wmt_en_ro"
+    env_vars_to_replace = {
+        "--fp16_opt_level=O1": "",
+        "$MAX_LEN": 128,
+        "$BS": 4,
+        "$GAS": 1,
+        "$ENRO_DIR": data_dir,
+        "facebook/mbart-large-cc25": MODEL_NAME,
+        # Download is 120MB in previous test.
+        "val_check_interval=0.25": "val_check_interval=1.0",
+    }
 
-        bash_script = bash_script.replace("--fp16 ", "")
-        testargs = (
-            ["finetune.py"]
-            + bash_script.split()
-            + [
-                f"--output_dir={output_dir}",
-                "--gpus=1",
-                "--learning_rate=3e-1",
-                "--warmup_steps=0",
-                "--val_check_interval=1.0",
-                "--tokenizer_name=facebook/mbart-large-en-ro",
-            ]
-        )
-        with patch.object(sys, "argv", testargs):
-            parser = argparse.ArgumentParser()
-            parser = pl.Trainer.add_argparse_args(parser)
-            parser = SummarizationModule.add_model_specific_args(parser, os.getcwd())
-            args = parser.parse_args()
-            args.do_predict = False
-            # assert args.gpus == gpus THIS BREAKS for multigpu
-            model = main(args)
+    # Clean up bash script
+    bash_script = Path("examples/seq2seq/train_mbart_cc25_enro.sh").open().read().split("finetune.py")[1].strip()
+    bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
+    for k, v in env_vars_to_replace.items():
+        bash_script = bash_script.replace(k, str(v))
+    output_dir = tempfile.mkdtemp(prefix="output_mbart")
 
-        # Check metrics
-        metrics = load_json(model.metrics_save_path)
-        first_step_stats = metrics["val"][0]
-        last_step_stats = metrics["val"][-1]
-        assert (
-            len(metrics["val"]) == (args.max_epochs / args.val_check_interval) + 1
-        )  # +1 accounts for val_sanity_check
+    bash_script = bash_script.replace("--fp16 ", "")
+    testargs = (
+        ["finetune.py"]
+        + bash_script.split()
+        + [
+            f"--output_dir={output_dir}",
+            "--gpus=1",
+            "--learning_rate=3e-1",
+            "--warmup_steps=0",
+            "--val_check_interval=1.0",
+            "--tokenizer_name=facebook/mbart-large-en-ro",
+        ]
+    )
+    with patch.object(sys, "argv", testargs):
+        parser = argparse.ArgumentParser()
+        parser = pl.Trainer.add_argparse_args(parser)
+        parser = SummarizationModule.add_model_specific_args(parser, os.getcwd())
+        args = parser.parse_args()
+        args.do_predict = False
+        # assert args.gpus == gpus THIS BREAKS for multigpu
+        model = main(args)
 
-        assert last_step_stats["val_avg_gen_time"] >= 0.01
+    # Check metrics
+    metrics = load_json(model.metrics_save_path)
+    first_step_stats = metrics["val"][0]
+    last_step_stats = metrics["val"][-1]
+    assert len(metrics["val"]) == (args.max_epochs / args.val_check_interval) + 1  # +1 accounts for val_sanity_check
 
-        assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
-        assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
-        assert isinstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
+    assert last_step_stats["val_avg_gen_time"] >= 0.01
 
-        # check lightning ckpt can be loaded and has a reasonable statedict
-        contents = os.listdir(output_dir)
-        ckpt_path = [x for x in contents if x.endswith(".ckpt")][0]
-        full_path = os.path.join(args.output_dir, ckpt_path)
-        ckpt = torch.load(full_path, map_location="cpu")
-        expected_key = "model.model.decoder.layers.0.encoder_attn_layer_norm.weight"
-        assert expected_key in ckpt["state_dict"]
-        assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.float32
+    assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
+    assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
+    assert isinstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
 
-        # TODO: turn on args.do_predict when PL bug fixed.
-        if args.do_predict:
-            contents = {os.path.basename(p) for p in contents}
-            assert "test_generations.txt" in contents
-            assert "test_results.txt" in contents
-            # assert len(metrics["val"]) ==  desired_n_evals
-            assert len(metrics["test"]) == 1
+    # check lightning ckpt can be loaded and has a reasonable statedict
+    contents = os.listdir(output_dir)
+    ckpt_path = [x for x in contents if x.endswith(".ckpt")][0]
+    full_path = os.path.join(args.output_dir, ckpt_path)
+    ckpt = torch.load(full_path, map_location="cpu")
+    expected_key = "model.model.decoder.layers.0.encoder_attn_layer_norm.weight"
+    assert expected_key in ckpt["state_dict"]
+    assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.float32
 
-    @timeout_decorator.timeout(600)
-    @slow
-    @pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
-    def test_opus_mt_distill_script(self):
-        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
-        env_vars_to_replace = {
-            "--fp16_opt_level=O1": "",
-            "$MAX_LEN": 128,
-            "$BS": 16,
-            "$GAS": 1,
-            "$ENRO_DIR": data_dir,
-            "$m": "sshleifer/student_marian_en_ro_6_1",
-            "val_check_interval=0.25": "val_check_interval=1.0",
-        }
+    # TODO(SS): turn on args.do_predict when PL bug fixed.
+    if args.do_predict:
+        contents = {os.path.basename(p) for p in contents}
+        assert "test_generations.txt" in contents
+        assert "test_results.txt" in contents
+        # assert len(metrics["val"]) ==  desired_n_evals
+        assert len(metrics["test"]) == 1
 
-        # Clean up bash script
-        bash_script = (
-            Path("examples/seq2seq/distil_marian_no_teacher.sh").open().read().split("distillation.py")[1].strip()
-        )
-        bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
-        bash_script = bash_script.replace("--fp16 ", " ")
 
-        for k, v in env_vars_to_replace.items():
-            bash_script = bash_script.replace(k, str(v))
-        output_dir = self.get_auto_remove_tmp_dir()
-        bash_script = bash_script.replace("--fp16", "")
-        epochs = 6
-        testargs = (
-            ["distillation.py"]
-            + bash_script.split()
-            + [
-                f"--output_dir={output_dir}",
-                "--gpus=1",
-                "--learning_rate=1e-3",
-                f"--num_train_epochs={epochs}",
-                "--warmup_steps=10",
-                "--val_check_interval=1.0",
-            ]
-        )
-        with patch.object(sys, "argv", testargs):
-            parser = argparse.ArgumentParser()
-            parser = pl.Trainer.add_argparse_args(parser)
-            parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
-            args = parser.parse_args()
-            args.do_predict = False
-            # assert args.gpus == gpus THIS BREAKS for multigpu
+@timeout_decorator.timeout(600)
+@slow
+@pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
+def test_opus_mt_distill_script():
+    data_dir = "examples/seq2seq/test_data/wmt_en_ro"
+    env_vars_to_replace = {
+        "--fp16_opt_level=O1": "",
+        "$MAX_LEN": 128,
+        "$BS": 16,
+        "$GAS": 1,
+        "$ENRO_DIR": data_dir,
+        "$m": "sshleifer/student_marian_en_ro_6_1",
+        "val_check_interval=0.25": "val_check_interval=1.0",
+    }
 
-            model = distill_main(args)
+    # Clean up bash script
+    bash_script = (
+        Path("examples/seq2seq/distil_marian_no_teacher.sh").open().read().split("distillation.py")[1].strip()
+    )
+    bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
+    bash_script = bash_script.replace("--fp16 ", " ")
 
-        # Check metrics
-        metrics = load_json(model.metrics_save_path)
-        first_step_stats = metrics["val"][0]
-        last_step_stats = metrics["val"][-1]
-        assert len(metrics["val"]) >= (args.max_epochs / args.val_check_interval)  # +1 accounts for val_sanity_check
+    for k, v in env_vars_to_replace.items():
+        bash_script = bash_script.replace(k, str(v))
+    output_dir = tempfile.mkdtemp(prefix="marian_output")
+    bash_script = bash_script.replace("--fp16", "")
+    epochs = 6
+    testargs = (
+        ["distillation.py"]
+        + bash_script.split()
+        + [
+            f"--output_dir={output_dir}",
+            "--gpus=1",
+            "--learning_rate=1e-3",
+            f"--num_train_epochs={epochs}",
+            "--warmup_steps=10",
+            "--val_check_interval=1.0",
+        ]
+    )
+    with patch.object(sys, "argv", testargs):
+        parser = argparse.ArgumentParser()
+        parser = pl.Trainer.add_argparse_args(parser)
+        parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+        args = parser.parse_args()
+        args.do_predict = False
+        # assert args.gpus == gpus THIS BREAKS for multigpu
 
-        assert last_step_stats["val_avg_gen_time"] >= 0.01
+        model = distill_main(args)
 
-        assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
-        assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
-        assert isinstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
+    # Check metrics
+    metrics = load_json(model.metrics_save_path)
+    first_step_stats = metrics["val"][0]
+    last_step_stats = metrics["val"][-1]
+    assert len(metrics["val"]) >= (args.max_epochs / args.val_check_interval)  # +1 accounts for val_sanity_check
 
-        # check lightning ckpt can be loaded and has a reasonable statedict
-        contents = os.listdir(output_dir)
-        ckpt_path = [x for x in contents if x.endswith(".ckpt")][0]
-        full_path = os.path.join(args.output_dir, ckpt_path)
-        ckpt = torch.load(full_path, map_location="cpu")
-        expected_key = "model.model.decoder.layers.0.encoder_attn_layer_norm.weight"
-        assert expected_key in ckpt["state_dict"]
-        assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.float32
+    assert last_step_stats["val_avg_gen_time"] >= 0.01
 
-        # TODO: turn on args.do_predict when PL bug fixed.
-        if args.do_predict:
-            contents = {os.path.basename(p) for p in contents}
-            assert "test_generations.txt" in contents
-            assert "test_results.txt" in contents
-            # assert len(metrics["val"]) ==  desired_n_evals
-            assert len(metrics["test"]) == 1
+    assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
+    assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
+    assert isinstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
+
+    # check lightning ckpt can be loaded and has a reasonable statedict
+    contents = os.listdir(output_dir)
+    ckpt_path = [x for x in contents if x.endswith(".ckpt")][0]
+    full_path = os.path.join(args.output_dir, ckpt_path)
+    ckpt = torch.load(full_path, map_location="cpu")
+    expected_key = "model.model.decoder.layers.0.encoder_attn_layer_norm.weight"
+    assert expected_key in ckpt["state_dict"]
+    assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.float32
+
+    # TODO(SS): turn on args.do_predict when PL bug fixed.
+    if args.do_predict:
+        contents = {os.path.basename(p) for p in contents}
+        assert "test_generations.txt" in contents
+        assert "test_results.txt" in contents
+        # assert len(metrics["val"]) ==  desired_n_evals
+        assert len(metrics["test"]) == 1

examples/seq2seq/test_datasets.py

@@ -1,4 +1,5 @@
 import os
+import tempfile
 from pathlib import Path
 
 import numpy as np
@@ -6,12 +7,11 @@ import pytest
 from torch.utils.data import DataLoader
 
 from pack_dataset import pack_data_dir
-from parameterized import parameterized
 from save_len_file import save_len_file
 from test_seq2seq_examples import ARTICLES, BART_TINY, MARIAN_TINY, MBART_TINY, SUMMARIES, T5_TINY, make_test_data_dir
 from transformers import AutoTokenizer
 from transformers.modeling_bart import shift_tokens_right
-from transformers.testing_utils import TestCasePlus, slow
+from transformers.testing_utils import slow
 from utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeq2SeqDataset, Seq2SeqDataset
 
 
@@ -19,198 +19,202 @@ BERT_BASE_CASED = "bert-base-cased"
 PEGASUS_XSUM = "google/pegasus-xsum"
 
 
-class TestAll(TestCasePlus):
-    @parameterized.expand(
-        [
-            MBART_TINY,
-            MARIAN_TINY,
-            T5_TINY,
-            BART_TINY,
-            PEGASUS_XSUM,
-        ],
+@slow
+@pytest.mark.parametrize(
+    "tok_name",
+    [
+        MBART_TINY,
+        MARIAN_TINY,
+        T5_TINY,
+        BART_TINY,
+        PEGASUS_XSUM,
+    ],
+)
+def test_seq2seq_dataset_truncation(tok_name):
+    tokenizer = AutoTokenizer.from_pretrained(tok_name)
+    tmp_dir = make_test_data_dir()
+    max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
+    max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
+    max_src_len = 4
+    max_tgt_len = 8
+    assert max_len_target > max_src_len  # Will be truncated
+    assert max_len_source > max_src_len  # Will be truncated
+    src_lang, tgt_lang = "ro_RO", "de_DE"  # ignored for all but mbart, but never causes error.
+    train_dataset = Seq2SeqDataset(
+        tokenizer,
+        data_dir=tmp_dir,
+        type_path="train",
+        max_source_length=max_src_len,
+        max_target_length=max_tgt_len,  # ignored
+        src_lang=src_lang,
+        tgt_lang=tgt_lang,
     )
-    @slow
-    def test_seq2seq_dataset_truncation(self, tok_name):
-        tokenizer = AutoTokenizer.from_pretrained(tok_name)
-        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
-        max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
-        max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
-        max_src_len = 4
-        max_tgt_len = 8
-        assert max_len_target > max_src_len  # Will be truncated
-        assert max_len_source > max_src_len  # Will be truncated
-        src_lang, tgt_lang = "ro_RO", "de_DE"  # ignored for all but mbart, but never causes error.
+    dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
+    for batch in dataloader:
+        assert isinstance(batch, dict)
+        assert batch["attention_mask"].shape == batch["input_ids"].shape
+        # show that articles were trimmed.
+        assert batch["input_ids"].shape[1] == max_src_len
+        # show that targets are the same len
+        assert batch["labels"].shape[1] == max_tgt_len
+        if tok_name != MBART_TINY:
+            continue
+        # check language codes in correct place
+        batch["decoder_input_ids"] = shift_tokens_right(batch["labels"], tokenizer.pad_token_id)
+        assert batch["decoder_input_ids"][0, 0].item() == tokenizer.lang_code_to_id[tgt_lang]
+        assert batch["decoder_input_ids"][0, -1].item() == tokenizer.eos_token_id
+        assert batch["input_ids"][0, -2].item() == tokenizer.eos_token_id
+        assert batch["input_ids"][0, -1].item() == tokenizer.lang_code_to_id[src_lang]
+
+        break  # No need to test every batch
+
+
+@pytest.mark.parametrize("tok", [BART_TINY, BERT_BASE_CASED])
+def test_legacy_dataset_truncation(tok):
+    tokenizer = AutoTokenizer.from_pretrained(tok)
+    tmp_dir = make_test_data_dir()
+    max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
+    max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
+    trunc_target = 4
+    train_dataset = LegacySeq2SeqDataset(
+        tokenizer,
+        data_dir=tmp_dir,
+        type_path="train",
+        max_source_length=20,
+        max_target_length=trunc_target,
+    )
+    dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
+    for batch in dataloader:
+        assert batch["attention_mask"].shape == batch["input_ids"].shape
+        # show that articles were trimmed.
+        assert batch["input_ids"].shape[1] == max_len_source
+        assert 20 >= batch["input_ids"].shape[1]  # trimmed significantly
+        # show that targets were truncated
+        assert batch["labels"].shape[1] == trunc_target  # Truncated
+        assert max_len_target > trunc_target  # Truncated
+        break  # No need to test every batch
+
+
+def test_pack_dataset():
+    tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25")
+
+    tmp_dir = Path(make_test_data_dir())
+    orig_examples = tmp_dir.joinpath("train.source").open().readlines()
+    save_dir = Path(tempfile.mkdtemp(prefix="packed_"))
+    pack_data_dir(tokenizer, tmp_dir, 128, save_dir)
+    orig_paths = {x.name for x in tmp_dir.iterdir()}
+    new_paths = {x.name for x in save_dir.iterdir()}
+    packed_examples = save_dir.joinpath("train.source").open().readlines()
+    # orig: [' Sam ate lunch today.\n', 'Sams lunch ingredients.']
+    # desired_packed: [' Sam ate lunch today.\n Sams lunch ingredients.']
+    assert len(packed_examples) < len(orig_examples)
+    assert len(packed_examples) == 1
+    assert len(packed_examples[0]) == sum(len(x) for x in orig_examples)
+    assert orig_paths == new_paths
+
+
+@pytest.mark.skipif(not FAIRSEQ_AVAILABLE, reason="This test requires fairseq")
+def test_dynamic_batch_size():
+    if not FAIRSEQ_AVAILABLE:
+        return
+    ds, max_tokens, tokenizer = _get_dataset(max_len=64)
+    required_batch_size_multiple = 64
+    batch_sampler = ds.make_dynamic_sampler(max_tokens, required_batch_size_multiple=required_batch_size_multiple)
+    batch_sizes = [len(x) for x in batch_sampler]
+    assert len(set(batch_sizes)) > 1  # it's not dynamic batch size if every batch is the same length
+    assert sum(batch_sizes) == len(ds)  # no dropped or added examples
+    data_loader = DataLoader(ds, batch_sampler=batch_sampler, collate_fn=ds.collate_fn, num_workers=2)
+    failures = []
+    num_src_per_batch = []
+    for batch in data_loader:
+        src_shape = batch["input_ids"].shape
+        bs = src_shape[0]
+        assert bs % required_batch_size_multiple == 0 or bs < required_batch_size_multiple
+        num_src_tokens = np.product(batch["input_ids"].shape)
+        num_src_per_batch.append(num_src_tokens)
+        if num_src_tokens > (max_tokens * 1.1):
+            failures.append(num_src_tokens)
+    assert num_src_per_batch[0] == max(num_src_per_batch)
+    if failures:
+        raise AssertionError(f"too many tokens in {len(failures)} batches")
+
+
+def test_sortish_sampler_reduces_padding():
+    ds, _, tokenizer = _get_dataset(max_len=512)
+    bs = 2
+    sortish_sampler = ds.make_sortish_sampler(bs, shuffle=False)
+
+    naive_dl = DataLoader(ds, batch_size=bs, collate_fn=ds.collate_fn, num_workers=2)
+    sortish_dl = DataLoader(ds, batch_size=bs, collate_fn=ds.collate_fn, num_workers=2, sampler=sortish_sampler)
+
+    pad = tokenizer.pad_token_id
+
+    def count_pad_tokens(data_loader, k="input_ids"):
+        return [batch[k].eq(pad).sum().item() for batch in data_loader]
+
+    assert sum(count_pad_tokens(sortish_dl, k="labels")) < sum(count_pad_tokens(naive_dl, k="labels"))
+    assert sum(count_pad_tokens(sortish_dl)) < sum(count_pad_tokens(naive_dl))
+    assert len(sortish_dl) == len(naive_dl)
+
+
+def _get_dataset(n_obs=1000, max_len=128):
+    if os.getenv("USE_REAL_DATA", False):
+        data_dir = "examples/seq2seq/wmt_en_ro"
+        max_tokens = max_len * 2 * 64
+        if not Path(data_dir).joinpath("train.len").exists():
+            save_len_file(MARIAN_TINY, data_dir)
+    else:
+        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
+        max_tokens = max_len * 4
+        save_len_file(MARIAN_TINY, data_dir)
+
+    tokenizer = AutoTokenizer.from_pretrained(MARIAN_TINY)
+    ds = Seq2SeqDataset(
+        tokenizer,
+        data_dir=data_dir,
+        type_path="train",
+        max_source_length=max_len,
+        max_target_length=max_len,
+        n_obs=n_obs,
+    )
+    return ds, max_tokens, tokenizer
+
+
+def test_distributed_sortish_sampler_splits_indices_between_procs():
+    ds, max_tokens, tokenizer = _get_dataset()
+    ids1 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=0, add_extra_examples=False))
+    ids2 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=1, add_extra_examples=False))
+    assert ids1.intersection(ids2) == set()
+
+
+@pytest.mark.parametrize(
+    "tok_name",
+    [
+        MBART_TINY,
+        MARIAN_TINY,
+        T5_TINY,
+        BART_TINY,
+        PEGASUS_XSUM,
+    ],
+)
+def test_dataset_kwargs(tok_name):
+    tokenizer = AutoTokenizer.from_pretrained(tok_name)
+    if tok_name == MBART_TINY:
         train_dataset = Seq2SeqDataset(
             tokenizer,
-            data_dir=tmp_dir,
+            data_dir=make_test_data_dir(),
             type_path="train",
-            max_source_length=max_src_len,
-            max_target_length=max_tgt_len,  # ignored
-            src_lang=src_lang,
-            tgt_lang=tgt_lang,
+            max_source_length=4,
+            max_target_length=8,
+            src_lang="EN",
+            tgt_lang="FR",
         )
-        dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
-        for batch in dataloader:
-            assert isinstance(batch, dict)
-            assert batch["attention_mask"].shape == batch["input_ids"].shape
-            # show that articles were trimmed.
-            assert batch["input_ids"].shape[1] == max_src_len
-            # show that targets are the same len
-            assert batch["labels"].shape[1] == max_tgt_len
-            if tok_name != MBART_TINY:
-                continue
-            # check language codes in correct place
-            batch["decoder_input_ids"] = shift_tokens_right(batch["labels"], tokenizer.pad_token_id)
-            assert batch["decoder_input_ids"][0, 0].item() == tokenizer.lang_code_to_id[tgt_lang]
-            assert batch["decoder_input_ids"][0, -1].item() == tokenizer.eos_token_id
-            assert batch["input_ids"][0, -2].item() == tokenizer.eos_token_id
-            assert batch["input_ids"][0, -1].item() == tokenizer.lang_code_to_id[src_lang]
-
-            break  # No need to test every batch
-
-    @parameterized.expand([BART_TINY, BERT_BASE_CASED])
-    def test_legacy_dataset_truncation(self, tok):
-        tokenizer = AutoTokenizer.from_pretrained(tok)
-        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
-        max_len_source = max(len(tokenizer.encode(a)) for a in ARTICLES)
-        max_len_target = max(len(tokenizer.encode(a)) for a in SUMMARIES)
-        trunc_target = 4
-        train_dataset = LegacySeq2SeqDataset(
-            tokenizer,
-            data_dir=tmp_dir,
-            type_path="train",
-            max_source_length=20,
-            max_target_length=trunc_target,
+        kwargs = train_dataset.dataset_kwargs
+        assert "src_lang" in kwargs and "tgt_lang" in kwargs
+    else:
+        train_dataset = Seq2SeqDataset(
+            tokenizer, data_dir=make_test_data_dir(), type_path="train", max_source_length=4, max_target_length=8
         )
-        dataloader = DataLoader(train_dataset, batch_size=2, collate_fn=train_dataset.collate_fn)
-        for batch in dataloader:
-            assert batch["attention_mask"].shape == batch["input_ids"].shape
-            # show that articles were trimmed.
-            assert batch["input_ids"].shape[1] == max_len_source
-            assert 20 >= batch["input_ids"].shape[1]  # trimmed significantly
-            # show that targets were truncated
-            assert batch["labels"].shape[1] == trunc_target  # Truncated
-            assert max_len_target > trunc_target  # Truncated
-            break  # No need to test every batch
-
-    def test_pack_dataset(self):
-        tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25")
-
-        tmp_dir = Path(make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()))
-        orig_examples = tmp_dir.joinpath("train.source").open().readlines()
-        save_dir = Path(make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()))
-        pack_data_dir(tokenizer, tmp_dir, 128, save_dir)
-        orig_paths = {x.name for x in tmp_dir.iterdir()}
-        new_paths = {x.name for x in save_dir.iterdir()}
-        packed_examples = save_dir.joinpath("train.source").open().readlines()
-        # orig: [' Sam ate lunch today.\n', 'Sams lunch ingredients.']
-        # desired_packed: [' Sam ate lunch today.\n Sams lunch ingredients.']
-        assert len(packed_examples) < len(orig_examples)
-        assert len(packed_examples) == 1
-        assert len(packed_examples[0]) == sum(len(x) for x in orig_examples)
-        assert orig_paths == new_paths
-
-    @pytest.mark.skipif(not FAIRSEQ_AVAILABLE, reason="This test requires fairseq")
-    def test_dynamic_batch_size(self):
-        if not FAIRSEQ_AVAILABLE:
-            return
-        ds, max_tokens, tokenizer = self._get_dataset(max_len=64)
-        required_batch_size_multiple = 64
-        batch_sampler = ds.make_dynamic_sampler(max_tokens, required_batch_size_multiple=required_batch_size_multiple)
-        batch_sizes = [len(x) for x in batch_sampler]
-        assert len(set(batch_sizes)) > 1  # it's not dynamic batch size if every batch is the same length
-        assert sum(batch_sizes) == len(ds)  # no dropped or added examples
-        data_loader = DataLoader(ds, batch_sampler=batch_sampler, collate_fn=ds.collate_fn, num_workers=2)
-        failures = []
-        num_src_per_batch = []
-        for batch in data_loader:
-            src_shape = batch["input_ids"].shape
-            bs = src_shape[0]
-            assert bs % required_batch_size_multiple == 0 or bs < required_batch_size_multiple
-            num_src_tokens = np.product(batch["input_ids"].shape)
-            num_src_per_batch.append(num_src_tokens)
-            if num_src_tokens > (max_tokens * 1.1):
-                failures.append(num_src_tokens)
-        assert num_src_per_batch[0] == max(num_src_per_batch)
-        if failures:
-            raise AssertionError(f"too many tokens in {len(failures)} batches")
-
-    def test_sortish_sampler_reduces_padding(self):
-        ds, _, tokenizer = self._get_dataset(max_len=512)
-        bs = 2
-        sortish_sampler = ds.make_sortish_sampler(bs, shuffle=False)
-
-        naive_dl = DataLoader(ds, batch_size=bs, collate_fn=ds.collate_fn, num_workers=2)
-        sortish_dl = DataLoader(ds, batch_size=bs, collate_fn=ds.collate_fn, num_workers=2, sampler=sortish_sampler)
-
-        pad = tokenizer.pad_token_id
-
-        def count_pad_tokens(data_loader, k="input_ids"):
-            return [batch[k].eq(pad).sum().item() for batch in data_loader]
-
-        assert sum(count_pad_tokens(sortish_dl, k="labels")) < sum(count_pad_tokens(naive_dl, k="labels"))
-        assert sum(count_pad_tokens(sortish_dl)) < sum(count_pad_tokens(naive_dl))
-        assert len(sortish_dl) == len(naive_dl)
-
-    def _get_dataset(self, n_obs=1000, max_len=128):
-        if os.getenv("USE_REAL_DATA", False):
-            data_dir = "examples/seq2seq/wmt_en_ro"
-            max_tokens = max_len * 2 * 64
-            if not Path(data_dir).joinpath("train.len").exists():
-                save_len_file(MARIAN_TINY, data_dir)
-        else:
-            data_dir = "examples/seq2seq/test_data/wmt_en_ro"
-            max_tokens = max_len * 4
-            save_len_file(MARIAN_TINY, data_dir)
-
-        tokenizer = AutoTokenizer.from_pretrained(MARIAN_TINY)
-        ds = Seq2SeqDataset(
-            tokenizer,
-            data_dir=data_dir,
-            type_path="train",
-            max_source_length=max_len,
-            max_target_length=max_len,
-            n_obs=n_obs,
-        )
-        return ds, max_tokens, tokenizer
-
-    def test_distributed_sortish_sampler_splits_indices_between_procs(self):
-        ds, max_tokens, tokenizer = self._get_dataset()
-        ids1 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=0, add_extra_examples=False))
-        ids2 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=1, add_extra_examples=False))
-        assert ids1.intersection(ids2) == set()
-
-    @parameterized.expand(
-        [
-            MBART_TINY,
-            MARIAN_TINY,
-            T5_TINY,
-            BART_TINY,
-            PEGASUS_XSUM,
-        ],
-    )
-    def test_dataset_kwargs(self, tok_name):
-        tokenizer = AutoTokenizer.from_pretrained(tok_name)
-        if tok_name == MBART_TINY:
-            train_dataset = Seq2SeqDataset(
-                tokenizer,
-                data_dir=make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()),
-                type_path="train",
-                max_source_length=4,
-                max_target_length=8,
-                src_lang="EN",
-                tgt_lang="FR",
-            )
-            kwargs = train_dataset.dataset_kwargs
-            assert "src_lang" in kwargs and "tgt_lang" in kwargs
-        else:
-            train_dataset = Seq2SeqDataset(
-                tokenizer,
-                data_dir=make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()),
-                type_path="train",
-                max_source_length=4,
-                max_target_length=8,
-            )
-            kwargs = train_dataset.dataset_kwargs
-            assert "add_prefix_space" not in kwargs if tok_name != BART_TINY else "add_prefix_space" in kwargs
-            assert len(kwargs) == 1 if tok_name == BART_TINY else len(kwargs) == 0
+        kwargs = train_dataset.dataset_kwargs
+        assert "add_prefix_space" not in kwargs if tok_name != BART_TINY else "add_prefix_space" in kwargs
+        assert len(kwargs) == 1 if tok_name == BART_TINY else len(kwargs) == 0

examples/seq2seq/test_finetune_trainer.py

@@ -1,8 +1,9 @@
 import os
 import sys
+import tempfile
 from unittest.mock import patch
 
-from transformers.testing_utils import TestCasePlus, slow
+from transformers.testing_utils import slow
 from transformers.trainer_callback import TrainerState
 from transformers.trainer_utils import set_seed
 
@@ -14,71 +15,93 @@ set_seed(42)
 MARIAN_MODEL = "sshleifer/student_marian_en_ro_6_1"
 
 
-class TestFinetuneTrainer(TestCasePlus):
-    def test_finetune_trainer(self):
-        output_dir = self.run_trainer(1, "12", MBART_TINY, 1)
-        logs = TrainerState.load_from_json(os.path.join(output_dir, "trainer_state.json")).log_history
-        eval_metrics = [log for log in logs if "eval_loss" in log.keys()]
-        first_step_stats = eval_metrics[0]
-        assert "eval_bleu" in first_step_stats
+def test_finetune_trainer():
+    output_dir = run_trainer(1, "12", MBART_TINY, 1)
+    logs = TrainerState.load_from_json(os.path.join(output_dir, "trainer_state.json")).log_history
+    eval_metrics = [log for log in logs if "eval_loss" in log.keys()]
+    first_step_stats = eval_metrics[0]
+    assert "eval_bleu" in first_step_stats
 
-    @slow
-    def test_finetune_trainer_slow(self):
-        # There is a missing call to __init__process_group somewhere
-        output_dir = self.run_trainer(eval_steps=2, max_len="128", model_name=MARIAN_MODEL, num_train_epochs=3)
 
-        # Check metrics
-        logs = TrainerState.load_from_json(os.path.join(output_dir, "trainer_state.json")).log_history
-        eval_metrics = [log for log in logs if "eval_loss" in log.keys()]
-        first_step_stats = eval_metrics[0]
-        last_step_stats = eval_metrics[-1]
+@slow
+def test_finetune_trainer_slow():
+    # TODO(SS): This will fail on devices with more than 1 GPU.
+    # There is a missing call to __init__process_group somewhere
+    output_dir = run_trainer(eval_steps=2, max_len="128", model_name=MARIAN_MODEL, num_train_epochs=3)
 
-        assert first_step_stats["eval_bleu"] < last_step_stats["eval_bleu"]  # model learned nothing
-        assert isinstance(last_step_stats["eval_bleu"], float)
+    # Check metrics
+    logs = TrainerState.load_from_json(os.path.join(output_dir, "trainer_state.json")).log_history
+    eval_metrics = [log for log in logs if "eval_loss" in log.keys()]
+    first_step_stats = eval_metrics[0]
+    last_step_stats = eval_metrics[-1]
 
-        # test if do_predict saves generations and metrics
-        contents = os.listdir(output_dir)
-        contents = {os.path.basename(p) for p in contents}
-        assert "test_generations.txt" in contents
-        assert "test_results.json" in contents
+    assert first_step_stats["eval_bleu"] < last_step_stats["eval_bleu"]  # model learned nothing
+    assert isinstance(last_step_stats["eval_bleu"], float)
 
-    def run_trainer(self, eval_steps: int, max_len: str, model_name: str, num_train_epochs: int):
-        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
-        output_dir = self.get_auto_remove_tmp_dir()
-        argv = f"""
-            --model_name_or_path {model_name}
-            --data_dir {data_dir}
-            --output_dir {output_dir}
-            --overwrite_output_dir
-            --n_train 8
-            --n_val 8
-            --max_source_length {max_len}
-            --max_target_length {max_len}
-            --val_max_target_length {max_len}
-            --do_train
-            --do_eval
-            --do_predict
-            --num_train_epochs {str(num_train_epochs)}
-            --per_device_train_batch_size 4
-            --per_device_eval_batch_size 4
-            --learning_rate 3e-4
-            --warmup_steps 8
-            --evaluate_during_training
-            --predict_with_generate
-            --logging_steps 0
-            --save_steps {str(eval_steps)}
-            --eval_steps {str(eval_steps)}
-            --sortish_sampler
-            --label_smoothing 0.1
-            --adafactor
-            --task translation
-            --tgt_lang ro_RO
-            --src_lang en_XX
-        """.split()
-        # --eval_beams  2
+    # test if do_predict saves generations and metrics
+    contents = os.listdir(output_dir)
+    contents = {os.path.basename(p) for p in contents}
+    assert "test_generations.txt" in contents
+    assert "test_results.json" in contents
 
-        testargs = ["finetune_trainer.py"] + argv
-        with patch.object(sys, "argv", testargs):
-            main()
 
-        return output_dir
+def run_trainer(eval_steps: int, max_len: str, model_name: str, num_train_epochs: int):
+    data_dir = "examples/seq2seq/test_data/wmt_en_ro"
+    output_dir = tempfile.mkdtemp(prefix="test_output")
+    argv = [
+        "--model_name_or_path",
+        model_name,
+        "--data_dir",
+        data_dir,
+        "--output_dir",
+        output_dir,
+        "--overwrite_output_dir",
+        "--n_train",
+        "8",
+        "--n_val",
+        "8",
+        "--max_source_length",
+        max_len,
+        "--max_target_length",
+        max_len,
+        "--val_max_target_length",
+        max_len,
+        "--do_train",
+        "--do_eval",
+        "--do_predict",
+        "--num_train_epochs",
+        str(num_train_epochs),
+        "--per_device_train_batch_size",
+        "4",
+        "--per_device_eval_batch_size",
+        "4",
+        "--learning_rate",
+        "3e-4",
+        "--warmup_steps",
+        "8",
+        "--evaluate_during_training",
+        "--predict_with_generate",
+        "--logging_steps",
+        0,
+        "--save_steps",
+        str(eval_steps),
+        "--eval_steps",
+        str(eval_steps),
+        "--sortish_sampler",
+        "--label_smoothing",
+        "0.1",
+        # "--eval_beams",
+        # "2",
+        "--adafactor",
+        "--task",
+        "translation",
+        "--tgt_lang",
+        "ro_RO",
+        "--src_lang",
+        "en_XX",
+    ]
+    testargs = ["finetune_trainer.py"] + argv
+    with patch.object(sys, "argv", testargs):
+        main()
+
+    return output_dir

examples/seq2seq/test_seq2seq_examples.py

@@ -3,6 +3,7 @@ import logging
 import os
 import sys
 import tempfile
+import unittest
 from pathlib import Path
 from unittest.mock import patch
 
@@ -14,12 +15,11 @@ import lightning_base
 from convert_pl_checkpoint_to_hf import convert_pl_to_hf
 from distillation import distill_main
 from finetune import SummarizationModule, main
-from parameterized import parameterized
 from run_eval import generate_summaries_or_translations, run_generate
 from run_eval_search import run_search
 from transformers import AutoConfig, AutoModelForSeq2SeqLM
 from transformers.hf_api import HfApi
-from transformers.testing_utils import CaptureStderr, CaptureStdout, TestCasePlus, require_torch_gpu, slow
+from transformers.testing_utils import CaptureStderr, CaptureStdout, require_multigpu, require_torch_and_cuda, slow
 from utils import ROUGE_KEYS, label_smoothed_nll_loss, lmap, load_json
 
 
@@ -88,7 +88,6 @@ CHEAP_ARGS = {
     "student_encoder_layers": 1,
     "freeze_encoder": False,
     "auto_scale_batch_size": False,
-    "overwrite_output_dir": False,
 }
 
 
@@ -111,23 +110,24 @@ logger.addHandler(stream_handler)
 logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
 
 
-def make_test_data_dir(tmp_dir):
+def make_test_data_dir(**kwargs):
+    tmp_dir = Path(tempfile.mkdtemp(**kwargs))
     for split in ["train", "val", "test"]:
-        _dump_articles(os.path.join(tmp_dir, f"{split}.source"), ARTICLES)
-        _dump_articles(os.path.join(tmp_dir, f"{split}.target"), SUMMARIES)
+        _dump_articles((tmp_dir / f"{split}.source"), ARTICLES)
+        _dump_articles((tmp_dir / f"{split}.target"), SUMMARIES)
     return tmp_dir
 
 
-class TestSummarizationDistiller(TestCasePlus):
+class TestSummarizationDistiller(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
         return cls
 
     @slow
-    @require_torch_gpu
+    @require_torch_and_cuda
     def test_hub_configs(self):
-        """I put require_torch_gpu cause I only want this to run with self-scheduled."""
+        """I put require_torch_and_cuda cause I only want this to run with self-scheduled."""
 
         model_list = HfApi().model_list()
         org = "sshleifer"
@@ -143,6 +143,17 @@ class TestSummarizationDistiller(TestCasePlus):
                 failures.append(m)
         assert not failures, f"The following models could not be loaded through AutoConfig: {failures}"
 
+    @require_multigpu
+    @unittest.skip("Broken at the moment")
+    def test_multigpu(self):
+        updates = dict(
+            no_teacher=True,
+            freeze_encoder=True,
+            gpus=2,
+            sortish_sampler=True,
+        )
+        self._test_distiller_cli(updates, check_contents=False)
+
     def test_distill_no_teacher(self):
         updates = dict(student_encoder_layers=2, student_decoder_layers=1, no_teacher=True)
         self._test_distiller_cli(updates)
@@ -162,12 +173,12 @@ class TestSummarizationDistiller(TestCasePlus):
         self.assertEqual(1, len(ckpts))
         transformer_ckpts = list(Path(model.output_dir).glob("**/*.bin"))
         self.assertEqual(len(transformer_ckpts), 2)
-        examples = lmap(str.strip, Path(model.hparams.data_dir).joinpath("test.source").open().readlines())
-        out_path = tempfile.mktemp()  # XXX: not being cleaned up
+        examples = lmap(str.strip, model.hparams.data_dir.joinpath("test.source").open().readlines())
+        out_path = tempfile.mktemp()
         generate_summaries_or_translations(examples, out_path, str(model.output_dir / "best_tfmr"))
         self.assertTrue(Path(out_path).exists())
 
-        out_path_new = self.get_auto_remove_tmp_dir()
+        out_path_new = tempfile.mkdtemp()
         convert_pl_to_hf(ckpts[0], transformer_ckpts[0].parent, out_path_new)
         assert os.path.exists(os.path.join(out_path_new, "pytorch_model.bin"))
 
@@ -242,8 +253,8 @@ class TestSummarizationDistiller(TestCasePlus):
         )
         default_updates.update(updates)
         args_d: dict = CHEAP_ARGS.copy()
-        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
-        output_dir = self.get_auto_remove_tmp_dir()
+        tmp_dir = make_test_data_dir()
+        output_dir = tempfile.mkdtemp(prefix="output_")
 
         args_d.update(data_dir=tmp_dir, output_dir=output_dir, **default_updates)
         model = distill_main(argparse.Namespace(**args_d))
@@ -268,253 +279,256 @@ class TestSummarizationDistiller(TestCasePlus):
         return model
 
 
-class TestTheRest(TestCasePlus):
-    def run_eval_tester(self, model):
-        input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
-        output_file_name = input_file_name.parent / "utest_output.txt"
-        assert not output_file_name.exists()
-        articles = [" New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County."]
-        _dump_articles(input_file_name, articles)
+def run_eval_tester(model):
+    input_file_name = Path(tempfile.mkdtemp()) / "utest_input.source"
+    output_file_name = input_file_name.parent / "utest_output.txt"
+    assert not output_file_name.exists()
+    articles = [" New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County."]
+    _dump_articles(input_file_name, articles)
+    score_path = str(Path(tempfile.mkdtemp()) / "scores.json")
+    task = "translation_en_to_de" if model == T5_TINY else "summarization"
+    testargs = f"""
+        run_eval_search.py
+        {model}
+        {input_file_name}
+        {output_file_name}
+        --score_path {score_path}
+        --task {task}
+        --num_beams 2
+        --length_penalty 2.0
+        """.split()
 
-        score_path = str(Path(self.get_auto_remove_tmp_dir()) / "scores.json")
-        task = "translation_en_to_de" if model == T5_TINY else "summarization"
-        testargs = f"""
-            run_eval_search.py
-            {model}
-            {input_file_name}
-            {output_file_name}
-            --score_path {score_path}
-            --task {task}
-            --num_beams 2
-            --length_penalty 2.0
-            """.split()
+    with patch.object(sys, "argv", testargs):
+        run_generate()
+        assert Path(output_file_name).exists()
+        os.remove(Path(output_file_name))
 
-        with patch.object(sys, "argv", testargs):
-            run_generate()
-            assert Path(output_file_name).exists()
-            # os.remove(Path(output_file_name))
 
-    # test one model to quickly (no-@slow) catch simple problems and do an
-    # extensive testing of functionality with multiple models as @slow separately
-    def test_run_eval(self):
-        self.run_eval_tester(T5_TINY)
+# test one model to quickly (no-@slow) catch simple problems and do an
+# extensive testing of functionality with multiple models as @slow separately
+def test_run_eval():
+    run_eval_tester(T5_TINY)
 
-    # any extra models should go into the list here - can be slow
-    @parameterized.expand([BART_TINY, MBART_TINY])
-    @slow
-    def test_run_eval_slow(self, model):
-        self.run_eval_tester(model)
 
-    # testing with 2 models to validate: 1. translation (t5) 2. summarization (mbart)
-    @parameterized.expand([T5_TINY, MBART_TINY])
-    @slow
-    def test_run_eval_search(self, model):
-        input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
-        output_file_name = input_file_name.parent / "utest_output.txt"
-        assert not output_file_name.exists()
+# any extra models should go into the list here - can be slow
+@slow
+@pytest.mark.parametrize("model", [BART_TINY, MBART_TINY])
+def test_run_eval_slow(model):
+    run_eval_tester(model)
 
-        text = {
-            "en": ["Machine learning is great, isn't it?", "I like to eat bananas", "Tomorrow is another great day!"],
-            "de": [
-                "Maschinelles Lernen ist großartig, oder?",
-                "Ich esse gerne Bananen",
-                "Morgen ist wieder ein toller Tag!",
-            ],
-        }
 
-        tmp_dir = Path(self.get_auto_remove_tmp_dir())
-        score_path = str(tmp_dir / "scores.json")
-        reference_path = str(tmp_dir / "val.target")
-        _dump_articles(input_file_name, text["en"])
-        _dump_articles(reference_path, text["de"])
-        task = "translation_en_to_de" if model == T5_TINY else "summarization"
-        testargs = f"""
-            run_eval_search.py
-            {model}
-            {str(input_file_name)}
-            {str(output_file_name)}
-            --score_path {score_path}
-            --reference_path {reference_path}
-            --task {task}
-            """.split()
-        testargs.extend(["--search", "num_beams=1:2 length_penalty=0.9:1.0"])
+# testing with 2 models to validate: 1. translation (t5) 2. summarization (mbart)
+@slow
+@pytest.mark.parametrize("model", [T5_TINY, MBART_TINY])
+def test_run_eval_search(model):
+    input_file_name = Path(tempfile.mkdtemp()) / "utest_input.source"
+    output_file_name = input_file_name.parent / "utest_output.txt"
+    assert not output_file_name.exists()
 
-        with patch.object(sys, "argv", testargs):
-            with CaptureStdout() as cs:
-                run_search()
-            expected_strings = [" num_beams | length_penalty", model, "Best score args"]
-            un_expected_strings = ["Info"]
-            if "translation" in task:
-                expected_strings.append("bleu")
-            else:
-                expected_strings.extend(ROUGE_KEYS)
-            for w in expected_strings:
-                assert w in cs.out
-            for w in un_expected_strings:
-                assert w not in cs.out
-            assert Path(output_file_name).exists()
-            os.remove(Path(output_file_name))
+    text = {
+        "en": ["Machine learning is great, isn't it?", "I like to eat bananas", "Tomorrow is another great day!"],
+        "de": [
+            "Maschinelles Lernen ist großartig, oder?",
+            "Ich esse gerne Bananen",
+            "Morgen ist wieder ein toller Tag!",
+        ],
+    }
 
-    @parameterized.expand(
-        [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY],
-    )
-    def test_finetune(self, model):
-        args_d: dict = CHEAP_ARGS.copy()
-        task = "translation" if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else "summarization"
-        args_d["label_smoothing"] = 0.1 if task == "translation" else 0
+    tmp_dir = Path(tempfile.mkdtemp())
+    score_path = str(tmp_dir / "scores.json")
+    reference_path = str(tmp_dir / "val.target")
+    _dump_articles(input_file_name, text["en"])
+    _dump_articles(reference_path, text["de"])
+    task = "translation_en_to_de" if model == T5_TINY else "summarization"
+    testargs = f"""
+        run_eval_search.py
+        {model}
+        {str(input_file_name)}
+        {str(output_file_name)}
+        --score_path {score_path}
+        --reference_path {reference_path}
+        --task {task}
+        """.split()
+    testargs.extend(["--search", "num_beams=1:2 length_penalty=0.9:1.0"])
 
-        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
-        output_dir = self.get_auto_remove_tmp_dir()
-        args_d.update(
-            data_dir=tmp_dir,
-            model_name_or_path=model,
-            tokenizer_name=None,
-            train_batch_size=2,
-            eval_batch_size=2,
-            output_dir=output_dir,
-            do_predict=True,
-            task=task,
-            src_lang="en_XX",
-            tgt_lang="ro_RO",
-            freeze_encoder=True,
-            freeze_embeds=True,
-        )
-        assert "n_train" in args_d
-        args = argparse.Namespace(**args_d)
-        module = main(args)
-
-        input_embeds = module.model.get_input_embeddings()
-        assert not input_embeds.weight.requires_grad
-        if model == T5_TINY:
-            lm_head = module.model.lm_head
-            assert not lm_head.weight.requires_grad
-            assert (lm_head.weight == input_embeds.weight).all().item()
-        elif model == FSMT_TINY:
-            fsmt = module.model.model
-            embed_pos = fsmt.decoder.embed_positions
-            assert not embed_pos.weight.requires_grad
-            assert not fsmt.decoder.embed_tokens.weight.requires_grad
-            # check that embeds are not the same
-            assert fsmt.decoder.embed_tokens != fsmt.encoder.embed_tokens
+    with patch.object(sys, "argv", testargs):
+        with CaptureStdout() as cs:
+            run_search()
+        expected_strings = [" num_beams | length_penalty", model, "Best score args"]
+        un_expected_strings = ["Info"]
+        if "translation" in task:
+            expected_strings.append("bleu")
         else:
-            bart = module.model.model
-            embed_pos = bart.decoder.embed_positions
-            assert not embed_pos.weight.requires_grad
-            assert not bart.shared.weight.requires_grad
-            # check that embeds are the same
-            assert bart.decoder.embed_tokens == bart.encoder.embed_tokens
-            assert bart.decoder.embed_tokens == bart.shared
+            expected_strings.extend(ROUGE_KEYS)
+        for w in expected_strings:
+            assert w in cs.out
+        for w in un_expected_strings:
+            assert w not in cs.out
+        assert Path(output_file_name).exists()
+        os.remove(Path(output_file_name))
 
-        example_batch = load_json(module.output_dir / "text_batch.json")
-        assert isinstance(example_batch, dict)
-        assert len(example_batch) >= 4
 
-    def test_finetune_extra_model_args(self):
-        args_d: dict = CHEAP_ARGS.copy()
+@pytest.mark.parametrize(
+    "model",
+    [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY],
+)
+def test_finetune(model):
+    args_d: dict = CHEAP_ARGS.copy()
+    task = "translation" if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else "summarization"
+    args_d["label_smoothing"] = 0.1 if task == "translation" else 0
 
-        task = "summarization"
-        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
+    tmp_dir = make_test_data_dir()
+    output_dir = tempfile.mkdtemp(prefix="output_")
+    args_d.update(
+        data_dir=tmp_dir,
+        model_name_or_path=model,
+        tokenizer_name=None,
+        train_batch_size=2,
+        eval_batch_size=2,
+        output_dir=output_dir,
+        do_predict=True,
+        task=task,
+        src_lang="en_XX",
+        tgt_lang="ro_RO",
+        freeze_encoder=True,
+        freeze_embeds=True,
+    )
+    assert "n_train" in args_d
+    args = argparse.Namespace(**args_d)
+    module = main(args)
 
-        args_d.update(
-            data_dir=tmp_dir,
-            tokenizer_name=None,
-            train_batch_size=2,
-            eval_batch_size=2,
-            do_predict=False,
-            task=task,
-            src_lang="en_XX",
-            tgt_lang="ro_RO",
-            freeze_encoder=True,
-            freeze_embeds=True,
-        )
+    input_embeds = module.model.get_input_embeddings()
+    assert not input_embeds.weight.requires_grad
+    if model == T5_TINY:
+        lm_head = module.model.lm_head
+        assert not lm_head.weight.requires_grad
+        assert (lm_head.weight == input_embeds.weight).all().item()
+    elif model == FSMT_TINY:
+        fsmt = module.model.model
+        embed_pos = fsmt.decoder.embed_positions
+        assert not embed_pos.weight.requires_grad
+        assert not fsmt.decoder.embed_tokens.weight.requires_grad
+        # check that embeds are not the same
+        assert fsmt.decoder.embed_tokens != fsmt.encoder.embed_tokens
+    else:
+        bart = module.model.model
+        embed_pos = bart.decoder.embed_positions
+        assert not embed_pos.weight.requires_grad
+        assert not bart.shared.weight.requires_grad
+        # check that embeds are the same
+        assert bart.decoder.embed_tokens == bart.encoder.embed_tokens
+        assert bart.decoder.embed_tokens == bart.shared
 
-        # test models whose config includes the extra_model_args
-        model = BART_TINY
-        output_dir = self.get_auto_remove_tmp_dir()
-        args_d1 = args_d.copy()
-        args_d1.update(
-            model_name_or_path=model,
-            output_dir=output_dir,
-        )
-        extra_model_params = ("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
-        for p in extra_model_params:
-            args_d1[p] = 0.5
-        args = argparse.Namespace(**args_d1)
+    example_batch = load_json(module.output_dir / "text_batch.json")
+    assert isinstance(example_batch, dict)
+    assert len(example_batch) >= 4
+
+
+def test_finetune_extra_model_args():
+    args_d: dict = CHEAP_ARGS.copy()
+
+    task = "summarization"
+    tmp_dir = make_test_data_dir()
+
+    args_d.update(
+        data_dir=tmp_dir,
+        tokenizer_name=None,
+        train_batch_size=2,
+        eval_batch_size=2,
+        do_predict=False,
+        task=task,
+        src_lang="en_XX",
+        tgt_lang="ro_RO",
+        freeze_encoder=True,
+        freeze_embeds=True,
+    )
+
+    # test models whose config includes the extra_model_args
+    model = BART_TINY
+    output_dir = tempfile.mkdtemp(prefix="output_1_")
+    args_d1 = args_d.copy()
+    args_d1.update(
+        model_name_or_path=model,
+        output_dir=output_dir,
+    )
+    extra_model_params = ("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
+    for p in extra_model_params:
+        args_d1[p] = 0.5
+    args = argparse.Namespace(**args_d1)
+    model = main(args)
+    for p in extra_model_params:
+        assert getattr(model.config, p) == 0.5, f"failed to override the model config for param {p}"
+
+    # test models whose config doesn't include the extra_model_args
+    model = T5_TINY
+    output_dir = tempfile.mkdtemp(prefix="output_2_")
+    args_d2 = args_d.copy()
+    args_d2.update(
+        model_name_or_path=model,
+        output_dir=output_dir,
+    )
+    unsupported_param = "encoder_layerdrop"
+    args_d2[unsupported_param] = 0.5
+    args = argparse.Namespace(**args_d2)
+    with pytest.raises(Exception) as excinfo:
         model = main(args)
-        for p in extra_model_params:
-            assert getattr(model.config, p) == 0.5, f"failed to override the model config for param {p}"
+    assert str(excinfo.value) == f"model config doesn't have a `{unsupported_param}` attribute"
 
-        # test models whose config doesn't include the extra_model_args
-        model = T5_TINY
-        output_dir = self.get_auto_remove_tmp_dir()
-        args_d2 = args_d.copy()
-        args_d2.update(
-            model_name_or_path=model,
-            output_dir=output_dir,
-        )
-        unsupported_param = "encoder_layerdrop"
-        args_d2[unsupported_param] = 0.5
-        args = argparse.Namespace(**args_d2)
-        with pytest.raises(Exception) as excinfo:
-            model = main(args)
-        assert str(excinfo.value) == f"model config doesn't have a `{unsupported_param}` attribute"
 
-    def test_finetune_lr_schedulers(self):
-        args_d: dict = CHEAP_ARGS.copy()
+def test_finetune_lr_schedulers():
+    args_d: dict = CHEAP_ARGS.copy()
 
-        task = "summarization"
-        tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
+    task = "summarization"
+    tmp_dir = make_test_data_dir()
 
-        model = BART_TINY
-        output_dir = self.get_auto_remove_tmp_dir()
+    model = BART_TINY
+    output_dir = tempfile.mkdtemp(prefix="output_1_")
 
-        args_d.update(
-            data_dir=tmp_dir,
-            model_name_or_path=model,
-            output_dir=output_dir,
-            tokenizer_name=None,
-            train_batch_size=2,
-            eval_batch_size=2,
-            do_predict=False,
-            task=task,
-            src_lang="en_XX",
-            tgt_lang="ro_RO",
-            freeze_encoder=True,
-            freeze_embeds=True,
-        )
+    args_d.update(
+        data_dir=tmp_dir,
+        model_name_or_path=model,
+        output_dir=output_dir,
+        tokenizer_name=None,
+        train_batch_size=2,
+        eval_batch_size=2,
+        do_predict=False,
+        task=task,
+        src_lang="en_XX",
+        tgt_lang="ro_RO",
+        freeze_encoder=True,
+        freeze_embeds=True,
+    )
 
-        # emulate finetune.py
-        parser = argparse.ArgumentParser()
-        parser = pl.Trainer.add_argparse_args(parser)
-        parser = SummarizationModule.add_model_specific_args(parser, os.getcwd())
-        args = {"--help": True}
+    # emulate finetune.py
+    parser = argparse.ArgumentParser()
+    parser = pl.Trainer.add_argparse_args(parser)
+    parser = SummarizationModule.add_model_specific_args(parser, os.getcwd())
+    args = {"--help": True}
 
-        # --help test
-        with pytest.raises(SystemExit) as excinfo:
-            with CaptureStdout() as cs:
-                args = parser.parse_args(args)
-            assert False, "--help is expected to sys.exit"
-        assert excinfo.type == SystemExit
-        expected = lightning_base.arg_to_scheduler_metavar
-        assert expected in cs.out, "--help is expected to list the supported schedulers"
+    # --help test
+    with pytest.raises(SystemExit) as excinfo:
+        with CaptureStdout() as cs:
+            args = parser.parse_args(args)
+        assert False, "--help is expected to sys.exit"
+    assert excinfo.type == SystemExit
+    expected = lightning_base.arg_to_scheduler_metavar
+    assert expected in cs.out, "--help is expected to list the supported schedulers"
 
-        # --lr_scheduler=non_existing_scheduler test
-        unsupported_param = "non_existing_scheduler"
-        args = {f"--lr_scheduler={unsupported_param}"}
-        with pytest.raises(SystemExit) as excinfo:
-            with CaptureStderr() as cs:
-                args = parser.parse_args(args)
-            assert False, "invalid argument is expected to sys.exit"
-        assert excinfo.type == SystemExit
-        expected = f"invalid choice: '{unsupported_param}'"
-        assert expected in cs.err, f"should have bailed on invalid choice of scheduler {unsupported_param}"
+    # --lr_scheduler=non_existing_scheduler test
+    unsupported_param = "non_existing_scheduler"
+    args = {f"--lr_scheduler={unsupported_param}"}
+    with pytest.raises(SystemExit) as excinfo:
+        with CaptureStderr() as cs:
+            args = parser.parse_args(args)
+        assert False, "invalid argument is expected to sys.exit"
+    assert excinfo.type == SystemExit
+    expected = f"invalid choice: '{unsupported_param}'"
+    assert expected in cs.err, f"should have bailed on invalid choice of scheduler {unsupported_param}"
 
-        # --lr_scheduler=existing_scheduler test
-        supported_param = "cosine"
-        args_d1 = args_d.copy()
-        args_d1["lr_scheduler"] = supported_param
-        args = argparse.Namespace(**args_d1)
-        model = main(args)
-        assert (
-            getattr(model.hparams, "lr_scheduler") == supported_param
-        ), f"lr_scheduler={supported_param} shouldn't fail"
+    # --lr_scheduler=existing_scheduler test
+    supported_param = "cosine"
+    args_d1 = args_d.copy()
+    args_d1["lr_scheduler"] = supported_param
+    args = argparse.Namespace(**args_d1)
+    model = main(args)
+    assert getattr(model.hparams, "lr_scheduler") == supported_param, f"lr_scheduler={supported_param} shouldn't fail"

examples/seq2seq/utils.py

@@ -457,22 +457,14 @@ def load_json(path):
 
 
 def get_git_info():
-    try:
-        repo = git.Repo(search_parent_directories=True)
-        repo_infos = {
-            "repo_id": str(repo),
-            "repo_sha": str(repo.head.object.hexsha),
-            "repo_branch": str(repo.active_branch),
-            "hostname": str(socket.gethostname()),
-        }
-        return repo_infos
-    except TypeError:
-        return {
-            "repo_id": None,
-            "repo_sha": None,
-            "repo_branch": None,
-            "hostname": None,
-        }
+    repo = git.Repo(search_parent_directories=True)
+    repo_infos = {
+        "repo_id": str(repo),
+        "repo_sha": str(repo.head.object.hexsha),
+        "repo_branch": str(repo.active_branch),
+        "hostname": str(socket.gethostname()),
+    }
+    return repo_infos
 
 
 ROUGE_KEYS = ["rouge1", "rouge2", "rougeL", "rougeLsum"]

examples/test_xla_examples.py

@@ -59,7 +59,7 @@ class TorchXLAExamplesTests(unittest.TestCase):
             --model_name_or_path=bert-base-cased
             --per_device_train_batch_size=64
             --per_device_eval_batch_size=64
-            --evaluation_strategy steps
+            --evaluate_during_training
             --overwrite_cache
             """.split()
         with patch.object(sys, "argv", testargs):
@@ -80,15 +80,4 @@ class TorchXLAExamplesTests(unittest.TestCase):
                 self.assertGreaterEqual(value, 0.70)
 
             # Assert that the script takes less than 300 seconds to make sure it doesn't hang.
-            self.assertLess(end - start, 500)
-
-    def test_trainer_tpu(self):
-        import xla_spawn
-
-        testargs = """
-            transformers/tests/test_trainer_tpu.py
-            --num_cores=8
-            transformers/tests/test_trainer_tpu.py
-            """.split()
-        with patch.object(sys, "argv", testargs):
-            xla_spawn.main()
+            self.assertLess(end - start, 300)

examples/text-classification/README.md

@@ -43,7 +43,7 @@ python run_tf_text_classification.py \
   --do_eval \
   --do_predict \
   --logging_steps 10 \
-  --evaluation_strategy steps \
+  --evaluate_during_training \
   --save_steps 10 \
   --overwrite_output_dir \
   --max_seq_length 128

model_cards/allegro/herbert-base-cased/README.md

@@ -1,51 +0,0 @@
----
-language: pl
-tags:
-- herbert
-license: cc-by-sa-4.0
----
-
-# HerBERT 
-**[HerBERT](https://en.wikipedia.org/wiki/Zbigniew_Herbert)** is a BERT-based Language Model trained on Polish Corpora
-using MLM and SSO objectives with dynamic masking of whole words.
-Model training and experiments were conducted with [transformers](https://github.com/huggingface/transformers) in version 2.9.
-
-## Tokenizer
-The training dataset was tokenized into subwords using ``CharBPETokenizer`` a character level byte-pair encoding with
-a vocabulary size of 50k tokens. The tokenizer itself was trained with a [tokenizers](https://github.com/huggingface/tokenizers) library. 
-We kindly encourage you to use the **Fast** version of tokenizer, namely ``HerbertTokenizerFast``.
-
-## HerBERT usage
-
-
-Example code:
-```python
-from transformers import AutoTokenizer, AutoModel
-
-tokenizer = AutoTokenizer.from_pretrained("allegro/herbert-base-cased")
-model = AutoModel.from_pretrained("allegro/herbert-base-cased")
-
-output = model(
-    **tokenizer.batch_encode_plus(
-        [
-            (
-                "A potem szedł środkiem drogi w kurzawie, bo zamiatał nogami, ślepy dziad prowadzony przez tłustego kundla na sznurku.",
-                "A potem leciał od lasu chłopak z butelką, ale ten ujrzawszy księdza przy drodze okrążył go z dala i biegł na przełaj pól do karczmy."
-            )
-        ],
-    padding='longest',
-    add_special_tokens=True,
-    return_tensors='pt'
-    )
-)
-```
-
-
-## License
-CC BY-SA 4.0
-
-
-## Authors
-Model was trained by **Allegro Machine Learning Research** team.
-
-You can contact us at: <a href="mailto:klejbenchmark@allegro.pl">klejbenchmark@allegro.pl</a>

model_cards/allegro/herbert-large-cased/README.md

@@ -1,50 +0,0 @@
----
-language: pl
-tags:
-- herbert
-license: cc-by-sa-4.0
----
-# HerBERT 
-**[HerBERT](https://en.wikipedia.org/wiki/Zbigniew_Herbert)** is a BERT-based Language Model trained on Polish Corpora
-using MLM and SSO objectives with dynamic masking of whole words.
-Model training and experiments were conducted with [transformers](https://github.com/huggingface/transformers) in version 2.9.
-
-## Tokenizer
-The training dataset was tokenized into subwords using ``CharBPETokenizer`` a character level byte-pair encoding with
-a vocabulary size of 50k tokens. The tokenizer itself was trained with a [tokenizers](https://github.com/huggingface/tokenizers) library. 
-We kindly encourage you to use the **Fast** version of tokenizer, namely ``HerbertTokenizerFast``.
-
-## HerBERT usage
-
-
-Example code:
-```python
-from transformers import AutoTokenizer, AutoModel
-
-tokenizer = AutoTokenizer.from_pretrained("allegro/herbert-large-cased")
-model = AutoModel.from_pretrained("allegro/herbert-large-cased")
-
-output = model(
-    **tokenizer.batch_encode_plus(
-        [
-            (
-                "A potem szedł środkiem drogi w kurzawie, bo zamiatał nogami, ślepy dziad prowadzony przez tłustego kundla na sznurku.",
-                "A potem leciał od lasu chłopak z butelką, ale ten ujrzawszy księdza przy drodze okrążył go z dala i biegł na przełaj pól do karczmy."
-            )
-        ],
-    padding='longest',
-    add_special_tokens=True,
-    return_tensors='pt'
-    )
-)
-```
-
-
-## License
-CC BY-SA 4.0
-
-
-## Authors
-Model was trained by **Allegro Machine Learning Research** team.
-
-You can contact us at: <a href="mailto:klejbenchmark@allegro.pl">klejbenchmark@allegro.pl</a>

model_cards/cooelf/limitbert/README.md

@@ -1,59 +0,0 @@
-# LIMIT-BERT
-
-Code and model for the *EMNLP 2020 Findings* paper: 
-
-[LIMIT-BERT: Linguistic Informed Multi-task BERT](https://arxiv.org/abs/1910.14296)) 
-
-## Contents
-
-1. [Requirements](#Requirements)
-2. [Training](#Training)
-
-## Requirements
-
-* Python 3.6 or higher.
-* Cython 0.25.2 or any compatible version.
-* [PyTorch](http://pytorch.org/) 1.0.0+. 
-* [EVALB](http://nlp.cs.nyu.edu/evalb/). Before starting, run `make` inside the `EVALB/` directory to compile an `evalb` executable. This will be called from Python for evaluation. 
-* [pytorch-transformers](https://github.com/huggingface/pytorch-transformers) PyTorch 1.0.0+ or any compatible version.
-
-#### Pre-trained Models (PyTorch)
-The following pre-trained models are available for download from Google Drive:
-* [`LIMIT-BERT`](https://drive.google.com/open?id=1fm0cK2A91iLG3lCpwowCCQSALnWS2X4i): 
-  PyTorch version, same setting with BERT-Large-WWM，loading model with [pytorch-transformers](https://github.com/huggingface/pytorch-transformers).
-
-## How to use
-
-```
-from transformers import AutoTokenizer, AutoModel
-tokenizer = AutoTokenizer.from_pretrained("cooelf/limitbert")
-model = AutoModel.from_pretrained("cooelf/limitbert")
-```
-
-Please see our original repo for the training scripts.
-
-https://github.com/cooelf/LIMIT-BERT
-
-## Training
-
-To train LIMIT-BERT, simply run:
-```
-sh run_limitbert.sh
-```
-### Evaluation Instructions
-
-To test after setting model path:
-```
-sh test_bert.sh
-```
-
-## Citation
-
-```
-@article{zhou2019limit,
-  title={{LIMIT-BERT}: Linguistic informed multi-task {BERT}},
-  author={Zhou, Junru and Zhang, Zhuosheng and Zhao, Hai},
-  journal={arXiv preprint arXiv:1910.14296},
-  year={2019}
-}
-```

model_cards/dslim/bert-base-NER/README.md

@@ -1,114 +0,0 @@
----
-language: en
-datasets:
-- conll2003
----
-# bert-base-NER
-
-## Model description
-
-**bert-base-NER** is a fine-tuned BERT model that is ready to use for **Named Entity Recognition** and achieves **state-of-the-art performance** for the NER task. It has been trained to recognize four types of entities: location (LOC), organizations (ORG), person (PER) and Miscellaneous (MISC). 
-
-Specifically, this model is a *bert-base-cased* model that was fine-tuned on the English version of the standard [CoNLL-2003 Named Entity Recognition](https://www.aclweb.org/anthology/W03-0419.pdf) dataset. 
-## Intended uses & limitations
-
-#### How to use
-
-You can use this model with Transformers *pipeline* for NER.
-
-```python
-from transformers import AutoTokenizer, AutoModelForTokenClassification
-from transformers import pipeline
-
-tokenizer = AutoTokenizer.from_pretrained("dslim/bert-base-NER")
-model = AutoModelForTokenClassification.from_pretrained("dslim/bert-base-NER")
-
-nlp = pipeline("ner", model=model, tokenizer=tokenizer)
-example = "My name is Wolfgang and I live in Berlin"
-
-ner_results = nlp(example)
-print(ner_results)
-```
-
-#### Limitations and bias
-
-This model is limited by its training dataset of entity-annotated news articles from a specific span of time. This may not generalize well for all use cases in different domains. Furthermore, the model occassionally tags subword tokens as entities and post-processing of results may be necessary to handle those cases. 
-
-## Training data
-
-This model was fine-tuned on English version of the standard [CoNLL-2003 Named Entity Recognition](https://www.aclweb.org/anthology/W03-0419.pdf) dataset. 
-
-The training dataset distinguishes between the beginning and continuation of an entity so that if there are back-to-back entities of the same type, the model can output where the second entity begins. As in the dataset, each token will be classified as one of the following classes:
-Abbreviation|Description
--|-
-O|Outside of a named entity
-B-MIS |Beginning of a miscellaneous entity right after another miscellaneous entity
-I-MIS |Miscellaneous entity
-B-PER |Beginning of a person’s name right after another person’s name
-I-PER |Person’s name
-B-ORG |Beginning of an organisation right after another organisation
-I-ORG |Organisation
-B-LOC |Beginning of a location right after another location
-I-LOC |Location
-
-
-### CoNLL-2003 English Dataset Statistics
-This dataset was derived from the Reuters corpus which consists of Reuters news stories. You can read more about how this dataset was created in the CoNLL-2003 paper. 
-#### # of training examples per entity type
-Dataset|LOC|MISC|ORG|PER
--|-|-|-|-
-Train|7140|3438|6321|6600
-Dev|1837|922|1341|1842
-Test|1668|702|1661|1617
-#### # of articles/sentences/tokens per dataset
-Dataset |Articles |Sentences |Tokens
--|-|-|-
-Train |946 |14,987 |203,621
-Dev |216 |3,466 |51,362
-Test |231 |3,684 |46,435
-
-## Training procedure
-
-This model was trained on a single NVIDIA V100 GPU with recommended hyperparameters from the [original BERT paper](https://arxiv.org/pdf/1810.04805) which trained & evaluated the model on CoNLL-2003 NER task. 
-
-## Eval results
-metric|dev|test
--|-|-
-f1 |95.1 |91.3
-precision |95.0 |90.7
-recall |95.3 |91.9
-
-The test metrics are a little lower than the official Google BERT results which encoded document context & experimented with CRF. More on replicating the original results [here](https://github.com/google-research/bert/issues/223).
-
-### BibTeX entry and citation info
-
-```
-@article{DBLP:journals/corr/abs-1810-04805,
-  author    = {Jacob Devlin and
-               Ming{-}Wei Chang and
-               Kenton Lee and
-               Kristina Toutanova},
-  title     = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
-               Understanding},
-  journal   = {CoRR},
-  volume    = {abs/1810.04805},
-  year      = {2018},
-  url       = {http://arxiv.org/abs/1810.04805},
-  archivePrefix = {arXiv},
-  eprint    = {1810.04805},
-  timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},
-  biburl    = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib},
-  bibsource = {dblp computer science bibliography, https://dblp.org}
-}
-```
-```
-@inproceedings{tjong-kim-sang-de-meulder-2003-introduction,
-    title = "Introduction to the {C}o{NLL}-2003 Shared Task: Language-Independent Named Entity Recognition",
-    author = "Tjong Kim Sang, Erik F.  and
-      De Meulder, Fien",
-    booktitle = "Proceedings of the Seventh Conference on Natural Language Learning at {HLT}-{NAACL} 2003",
-    year = "2003",
-    url = "https://www.aclweb.org/anthology/W03-0419",
-    pages = "142--147",
-}
-```

model_cards/facebook/bart-large-cnn/README.md

@@ -3,5 +3,4 @@ tags:
 - summarization
 
 license: mit
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---

model_cards/facebook/bart-large-mnli/README.md

@@ -1,5 +1,3 @@
 ---
 license: mit
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
-pipeline_tag: zero-shot-classification
 ---

model_cards/facebook/bart-large/README.md

@@ -1,8 +1,3 @@
----
-license: mit
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
----
-
 The Bart model was proposed by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer on 29 Oct, 2019. According to the abstract,
 
 Bart uses a standard seq2seq/machine translation architecture with a bidirectional encoder (like BERT) and a left-to-right decoder (like GPT).

model_cards/facebook/rag-sequence-base/README.md

@@ -1,7 +1,3 @@
----
-license: apache-2.0
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
----
 ## RAG
 
 This is a non-finetuned version of the RAG-Sequence model of the the paper [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401.pdf) 

model_cards/facebook/rag-sequence-nq/README.md

@@ -3,7 +3,6 @@ language: en
 license: apache-2.0
 datasets:
 - wiki_dpr
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 ## RAG
 
@@ -24,9 +23,9 @@ The model can generate answers to any factoid question as follows:
 ```python
 from transformers import RagTokenizer, RagRetriever, RagSequenceForGeneration 
  
-tokenizer = RagTokenizer.from_pretrained("facebook/rag-sequence-nq") 
-retriever = RagRetriever.from_pretrained("facebook/rag-sequence-nq", index_name="exact", use_dummy_dataset=True) 
-model = RagSequenceForGeneration.from_pretrained("facebook/rag-sequence-nq", retriever=retriever) 
+tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-nq") 
+retriever = RagRetriever.from_pretrained("facebook/rag-token-nq", index_name="exact", use_dummy_dataset=True) 
+model = RagSequenceForGeneration.from_pretrained("facebook/rag-token-nq", retriever=retriever) 
  
 input_dict = tokenizer.prepare_seq2seq_batch("how many countries are in europe", return_tensors="pt") 
 

model_cards/facebook/rag-token-base/README.md

@@ -1,10 +1,3 @@
----
-language: en
-license: apache-2.0
-datasets:
-- wiki_dpr
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
----
 ## RAG
 
 This is a non-finetuned version of the RAG-Token model of the the paper [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401.pdf) 

model_cards/facebook/rag-token-nq/README.md

@@ -3,7 +3,6 @@ language: en
 license: apache-2.0
 datasets:
 - wiki_dpr
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 ## RAG
 

model_cards/facebook/wmt19-de-en/README.md

@@ -1,7 +1,9 @@
+
 ---
 language: 
 - de
 - en
+thumbnail:
 tags:
 - translation
 - wmt19
@@ -11,7 +13,6 @@ datasets:
 - wmt19
 metrics:
 - bleu
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 
 # FSMT

model_cards/facebook/wmt19-en-de/README.md

@@ -1,7 +1,9 @@
+
 ---
 language: 
 - en
 - de
+thumbnail:
 tags:
 - translation
 - wmt19
@@ -11,7 +13,6 @@ datasets:
 - wmt19
 metrics:
 - bleu
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 
 # FSMT

model_cards/facebook/wmt19-en-ru/README.md

@@ -1,7 +1,9 @@
+
 ---
 language: 
 - en
 - ru
+thumbnail:
 tags:
 - translation
 - wmt19
@@ -11,7 +13,6 @@ datasets:
 - wmt19
 metrics:
 - bleu
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 
 # FSMT

model_cards/facebook/wmt19-ru-en/README.md

@@ -1,7 +1,9 @@
+
 ---
 language: 
 - ru
 - en
+thumbnail:
 tags:
 - translation
 - wmt19
@@ -11,7 +13,6 @@ datasets:
 - wmt19
 metrics:
 - bleu
-thumbnail: https://huggingface.co/front/thumbnails/facebook.png
 ---
 
 # FSMT

model_cards/google/roberta2roberta_L-24_discofuse/README.md

@@ -19,8 +19,6 @@ Disclaimer: The model card has been written by the Hugging Face team.
 
 You can use this model for sentence fusion, *e.g.*
 
-IMPORTANT: The model was not trained on the `"` (double quotation mark) character -> so the before tokenizing the text, it is advised to replace all `"` (double quotation marks) with a single `` ` `` (single back tick).
-
 ```python
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 

model_cards/google/roberta2roberta_L-24_wikisplit/README.md

@@ -17,9 +17,6 @@ Disclaimer: The model card has been written by the Hugging Face team.
 
 You can use this model for sentence splitting, *e.g.*
 
-**IMPORTANT**: The model was not trained on the `"` (double quotation mark) character -> so the before tokenizing the text, 
-it is advised to replace all `"` (double quotation marks) with two single `'` (single quotation mark).
-
 ```python
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 
@@ -28,9 +25,10 @@ model = AutoModelForSeq2SeqLM.from_pretrained("google/roberta2roberta_L-24_wikis
 
 long_sentence = """Due to the hurricane, Lobsterfest has been canceled, making Bob very happy about it and he decides to open Bob 's Burgers for customers who were planning on going to Lobsterfest."""
 
-input_ids = tokenizer(tokenizer.bos_token + long_sentence + tokenizer.eos_token, return_tensors="pt").input_ids
+input_ids = tokenizer(long_sentence, return_tensors="pt").input_ids
 output_ids = model.generate(input_ids)[0]
 print(tokenizer.decode(output_ids, skip_special_tokens=True))
 # should output
-# Due to the hurricane, Lobsterfest has been canceled, making Bob very happy about it. He decides to open Bob's Burgers for customers who were planning on going to Lobsterfest. 
+# Due Due hurricane, Lobsterfest has been canceled, making Bob very happy about it. He decides to open B
+# ob's Burgers for customers who were planning on going to Lobsterfest.com.
 ```

model_cards/huawei-noah/TinyBERT_General_4L_312D/README.md

@@ -1,19 +0,0 @@
-TinyBERT: Distilling BERT for Natural Language Understanding
-======== 
-TinyBERT is 7.5x smaller and 9.4x faster on inference than BERT-base and achieves competitive performances in the tasks of natural language understanding. It performs a novel transformer distillation at both the pre-training and task-specific learning stages. In general distillation, we use the original BERT-base without fine-tuning as the teacher and a large-scale text corpus as the learning data. By performing the Transformer distillation on the text from general domain, we obtain a general TinyBERT which provides a good initialization for the task-specific distillation. We here provide the general TinyBERT for your tasks at hand.
-
-For more details about the techniques of TinyBERT, refer to our paper:
-[TinyBERT: Distilling BERT for Natural Language Understanding](https://arxiv.org/abs/1909.10351)
-
-
-Citation
-========
-If you find TinyBERT useful in your research, please cite the following paper:
-```
-@article{jiao2019tinybert,
-  title={Tinybert: Distilling bert for natural language understanding},
-  author={Jiao, Xiaoqi and Yin, Yichun and Shang, Lifeng and Jiang, Xin and Chen, Xiao and Li, Linlin and Wang, Fang and Liu, Qun},
-  journal={arXiv preprint arXiv:1909.10351},
-  year={2019}
-}
-```

model_cards/jordimas/julibert/README.md

@@ -1,23 +0,0 @@
----
-language: ca
----
-
-## Introduction
-
-
-Download the model here:
-
-* Catalan Roberta model: [julibert-2020-11-10.zip](https://www.softcatala.org/pub/softcatala/julibert/julibert-2020-11-10.zip)
-
-## What's this?
-
-Source code: https://github.com/Softcatala/julibert
-
-* Corpus: Oscar Catalan Corpus (3,8G)
-* Model type: Roberta
-* Vocabulary size: 50265
-* Steps: 500000
-
-
-
-

model_cards/microsoft/prophetnet-large-uncased-cnndm/README.md

@@ -1,29 +0,0 @@
-## prophetnet-large-uncased-cnndm
-Fine-tuned weights(converted from [original fairseq version repo](https://github.com/microsoft/ProphetNet)) for [ProphetNet](https://arxiv.org/abs/2001.04063) on summarization task CNN/DailyMail.  
-ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
-ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
-
-### Usage
-```
-from transformers import ProphetNetTokenizer, ProphetNetForConditionalGeneration, ProphetNetConfig
-
-model = ProphetNetForConditionalGeneration.from_pretrained('microsoft/prophetnet-large-uncased-cnndm')
-tokenizer = ProphetNetTokenizer.from_pretrained('microsoft/prophetnet-large-uncased-cnndm')
-
-ARTICLE_TO_SUMMARIZE = "USTC was founded in Beijing by the Chinese Academy of Sciences (CAS) in September 1958. The Director of CAS, Mr. Guo Moruo was appointed the first president of USTC. USTC's founding mission was to develop a high-level science and technology workforce, as deemed critical for development of China's economy, defense, and science and technology education. The establishment was hailed as \"A Major Event in the History of Chinese Education and Science.\" CAS has supported USTC by combining most of its institutes with the departments of the university. USTC is listed in the top 16 national key universities, becoming the youngest national key university.".lower()
-inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=100, return_tensors='pt')
-
-# Generate Summary
-summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=512, early_stopping=True)
-tokenizer.batch_decode(summary_ids.tolist())
-```
-Here, [X_SEP] is used as a special token to seperate sentences.
-### Citation
-```bibtex
-@article{yan2020prophetnet,
-  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
-  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
-  journal={arXiv preprint arXiv:2001.04063},
-  year={2020}
-}
-```

model_cards/microsoft/prophetnet-large-uncased/README.md

@@ -1,17 +0,0 @@
-## prophetnet-large-uncased
-Pretrained weights for [ProphetNet](https://arxiv.org/abs/2001.04063).  
-ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
-ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
-
-### Usage
-Please see [the official repository](https://github.com/microsoft/ProphetNet) for details.
-
-### Citation
-```bibtex
-@article{yan2020prophetnet,
-  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
-  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
-  journal={arXiv preprint arXiv:2001.04063},
-  year={2020}
-}
-```

model_cards/microsoft/xprophetnet-large-wiki100-cased-xglue-ntg/README.md

@@ -1,32 +0,0 @@
-## xprophetnet-large-wiki100-cased-xglue-ntg
-Cross-lingual version [ProphetNet](https://arxiv.org/abs/2001.04063), pretrained on [wiki100 xGLUE dataset](https://arxiv.org/abs/2004.01401) and finetuned on xGLUE cross-lingual News Titles Generation task.  
-ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
-ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
-
-xProphetNet is also served as the baseline model for xGLUE cross-lingual natural language generation tasks.  
-For xGLUE corss-lingual NLG tasks, xProphetNet is finetuned with English data, but inference with both English and other zero-shot language data.  
-### Usage
-A quick usage is like: 
-```
-from transformers import ProphetNetTokenizer, ProphetNetForConditionalGeneration, ProphetNetConfig
-
-model = ProphetNetForConditionalGeneration.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-ntg')
-tokenizer = ProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-ntg')
-
-EN_SENTENCE = "Microsoft Corporation intends to officially end free support for the Windows 7 operating system after January 14, 2020, according to the official portal of the organization. From that day, users of this system will not be able to receive security updates, which could make their computers vulnerable to cyber attacks."
-RU_SENTENCE = "орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7 после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми к кибератакам."
-ZH_SENTENCE = "根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。"
-inputs = tokenizer([EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE], padding=True, max_length=256, return_tensors='pt')
-
-summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=100, early_stopping=True)
-print([tokenizer.decode(g) for g in summary_ids])  
-```
-### Citation
-```bibtex
-@article{yan2020prophetnet,
-  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
-  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
-  journal={arXiv preprint arXiv:2001.04063},
-  year={2020}
-}
-```

model_cards/microsoft/xprophetnet-large-wiki100-cased-xglue-qg/README.md

@@ -1,31 +0,0 @@
-## xprophetnet-large-wiki100-cased-xglue-ntg
-Cross-lingual version [ProphetNet](https://arxiv.org/abs/2001.04063), pretrained on [wiki100 xGLUE dataset](https://arxiv.org/abs/2004.01401) and finetuned on xGLUE cross-lingual Question Generation task.  
-ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
-ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
-
-xProphetNet is also served as the baseline model for xGLUE cross-lingual natural language generation tasks.  
-For xGLUE corss-lingual NLG tasks, xProphetNet is finetuned with English data, but inference with both English and other zero-shot language data.  
-### Usage
-A quick usage is like: 
-```
-from transformers import ProphetNetTokenizer, ProphetNetForConditionalGeneration, ProphetNetConfig
-
-model = ProphetNetForConditionalGeneration.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-qg')
-tokenizer = ProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-qg')
-
-EN_SENTENCE = "Google left China in 2010"
-ZH_SENTENCE = "Google在2010年离开中国"
-inputs = tokenizer([EN_SENTENCE, ZH_SENTENCE], padding=True, max_length=256, return_tensors='pt')
-
-summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=100, early_stopping=True)
-print([tokenizer.decode(g) for g in summary_ids])  
-```
-### Citation
-```bibtex
-@article{yan2020prophetnet,
-  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
-  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
-  journal={arXiv preprint arXiv:2001.04063},
-  year={2020}
-}
-```

model_cards/microsoft/xprophetnet-large-wiki100-cased/README.md

@@ -1,19 +0,0 @@
-## xprophetnet-large-wiki100-cased
-Cross-lingual version [ProphetNet](https://arxiv.org/abs/2001.04063), pretrained on [wiki100 xGLUE dataset](https://arxiv.org/abs/2004.01401).  
-ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
-ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
-
-xProphetNet is also served as the baseline model for xGLUE cross-lingual natural language generation tasks.  
-For xGLUE corss-lingual NLG tasks, xProphetNet is finetuned with English data, but inference with both English and other zero-shot language data.  
-### Usage
-Please see [the official repository](https://github.com/microsoft/ProphetNet/tree/master/xProphetNet) for details.
-
-### Citation
-```bibtex
-@article{yan2020prophetnet,
-  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
-  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
-  journal={arXiv preprint arXiv:2001.04063},
-  year={2020}
-}
-```

model_cards/sagorsarker/bangla-bert-base/README.md

@@ -49,7 +49,6 @@ Our final vocab file availabe at [https://github.com/sagorbrur/bangla-bert](http
 
 ## Evaluation Results
 
-### LM Evaluation Results
 After training 1 millions steps here is the evaluation resutls. 
 
 ```
@@ -62,22 +61,9 @@ next_sentence_loss = 0.040997364
 perplexity = numpy.exp(2.2406516) = 9.393331287442784
 Loss for final step: 2.426227
 
+
 ```
 
-### Downstream Task Evaluation Results
-Huge Thanks to [Nick Doiron](https://twitter.com/mapmeld) for providing evalution results of classification task.
-He used [Bengali Classification Benchmark](https://github.com/rezacsedu/Classification_Benchmarks_Benglai_NLP) datasets for classification task.
-Comparing to Nick's [Bengali electra](https://huggingface.co/monsoon-nlp/bangla-electra) and multi-lingual BERT, Bangla BERT Base achieves state of the art result.
-Here is the [evaluation script](https://github.com/sagorbrur/bangla-bert/blob/master/notebook/bangla-bert-evaluation-classification-task.ipynb).
-
-
-| Model | Sentiment Analysis | Hate Speech Task | News Topic Task | Average |
-| ----- | -------------------| ---------------- | --------------- | ------- |
-| mBERT | 68.15 | 52.32 | 72.27 | 64.25 |
-| Bengali Electra | 69.19 | 44.84 | 82.33 | 65.45 |
-| Bangla BERT Base | 70.37 | 71.83 | 89.19 | 77.13 |
-
-
 **NB: If you use this model for any nlp task please share evaluation results with us. We will add it here.** 
 
 
@@ -87,8 +73,8 @@ You can use this model directly with a pipeline for masked language modeling:
 ```py
 from transformers import BertForMaskedLM, BertTokenizer, pipeline
 
-model = BertForMaskedLM.from_pretrained("sagorsarker/bangla-bert-base")
-tokenizer = BertTokenizer.from_pretrained("sagorsarker/bangla-bert-base")
+model = BertForMaskedLM.from_pretrained("bangla-bert-base")
+tokenizer = BertTokenizer.from_pretrained("bangla-bert-base")
 nlp = pipeline('fill-mask', model=model, tokenizer=tokenizer)
 for pred in nlp(f"আমি বাংলায় {nlp.tokenizer.mask_token} গাই।"):
   print(pred)

model_cards/sarahlintang/IndoBERT/README.md

@@ -1,43 +0,0 @@
----
-language: id
-datasets:
-- oscar
----
-# IndoBERT (Indonesian BERT Model)
-
-## Model description
-IndoBERT is a pre-trained language model based on BERT architecture for the Indonesian Language. 
-
-This model is base-uncased version which use bert-base config.
-
-## Intended uses & limitations
-
-#### How to use
-
-```python
-from transformers import AutoTokenizer, AutoModel
-tokenizer = AutoTokenizer.from_pretrained("sarahlintang/IndoBERT")
-model = AutoModel.from_pretrained("sarahlintang/IndoBERT")
-tokenizer.encode("hai aku mau makan.")
-[2, 8078, 1785, 2318, 1946, 18, 4]
-```
-
-
-## Training data
-
-This model was pre-trained on 16 GB of raw text ~2 B words from Oscar Corpus (https://oscar-corpus.com/). 
-
-This model is equal to bert-base model which has 32,000 vocabulary size. 
-
-## Training procedure
-
-The training of the model has been performed using Google’s original Tensorflow code on eight core Google Cloud TPU v2.
-We used a Google Cloud Storage bucket, for persistent storage of training data and models.
-
-## Eval results
-
-We evaluate this model on three Indonesian NLP downstream task:
-- some extractive summarization model
-- sentiment analysis
-- Part-of-Speech Tagger
-it was proven that this model outperforms multilingual BERT for all downstream tasks.

model_cards/sentence-transformers/LaBSE/README.md

@@ -1,37 +0,0 @@
-# LaBSE Pytorch Version 
-This is a pytorch port of the tensorflow version of [LaBSE](https://tfhub.dev/google/LaBSE/1).
-
-To get the sentence embeddings, you can  use the following code:
-```python
-from transformers import AutoTokenizer, AutoModel
-
-tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/LaBSE")
-model = AutoModel.from_pretrained("sentence-transformers/LaBSE")
-
-sentences = ["Hello World", "Hallo Welt"]
-
-encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=64, return_tensors='pt')
-
-with torch.no_grad():
-    model_output = model(**encoded_input, return_dict=True)
-
-embeddings = model_output.pooler_output
-embeddings = torch.nn.functional.normalize(embeddings)
-print(embeddings)
-```
-
-
-When you have [sentence-transformers](https://www.sbert.net/) installed, you can use the model like this:
-```python
-from sentence_transformers import SentenceTransformer
-sentences = ["Hello World", "Hallo Welt"]
-
-model = SentenceTransformer('LaBSE')
-embeddings = model.encode(sentences)
-print(embeddings)
-```
-
-## Reference:
-Fangxiaoyu Feng, Yinfei Yang, Daniel Cer, Narveen Ari, Wei Wang. [Language-agnostic BERT Sentence Embedding](https://arxiv.org/abs/2007.01852). July 2020
-
-License: [https://tfhub.dev/google/LaBSE/1](https://tfhub.dev/google/LaBSE/1)

scripts/fsmt/tests-to-run.sh

@@ -2,5 +2,5 @@
 
 # these scripts need to be run before any changes to FSMT-related code - it should cover all bases
 
-CUDA_VISIBLE_DEVICES="" RUN_SLOW=1 pytest --disable-warnings tests/test_tokenization_fsmt.py tests/test_configuration_auto.py tests/test_modeling_fsmt.py examples/seq2seq/test_fsmt_bleu_score.py
-RUN_SLOW=1 pytest --disable-warnings tests/test_tokenization_fsmt.py tests/test_configuration_auto.py tests/test_modeling_fsmt.py examples/seq2seq/test_fsmt_bleu_score.py
+USE_CUDA=0 RUN_SLOW=1 pytest --disable-warnings tests/test_tokenization_fsmt.py tests/test_configuration_auto.py tests/test_modeling_fsmt.py examples/seq2seq/test_fsmt_bleu_score.py
+USE_CUDA=1 RUN_SLOW=1 pytest --disable-warnings tests/test_tokenization_fsmt.py tests/test_configuration_auto.py tests/test_modeling_fsmt.py examples/seq2seq/test_fsmt_bleu_score.py

setup.py

@@ -87,23 +87,21 @@ extras["tf-cpu"] = [
     # "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx",
 ]
 extras["torch"] = ["torch>=1.0"]
-extras["flax"] = ["jaxlib==0.1.55", "jax>=0.2.0", "flax==0.2.2"]
 extras["onnxruntime"] = ["onnxruntime>=1.4.0", "onnxruntime-tools>=1.4.2"]
 
 extras["serving"] = ["pydantic", "uvicorn", "fastapi", "starlette"]
 extras["all"] = extras["serving"] + ["tensorflow", "torch"]
 
-extras["sentencepiece"] = ["sentencepiece!=0.1.92"]
 extras["retrieval"] = ["faiss-cpu", "datasets"]
 extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil"] + extras["retrieval"]
 # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
 extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
 extras["quality"] = ["black >= 20.8b1", "isort >= 5.5.4", "flake8 >= 3.8.3"]
-extras["dev"] = extras["testing"] + extras["quality"] + extras["ja"] + ["scikit-learn", "tensorflow", "torch", "sentencepiece!=0.1.92"]
+extras["dev"] = extras["testing"] + extras["quality"] + extras["ja"] + ["scikit-learn", "tensorflow", "torch"]
 
 setup(
     name="transformers",
-    version="3.4.0",
+    version="3.3.1",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Sylvain Gugger, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
@@ -116,7 +114,7 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.9.2",
+        "tokenizers == 0.9.0.rc2",
         # dataclasses for Python versions that don't have it
         "dataclasses;python_version<'3.7'",
         # utilities from PyPA to e.g. compare versions

src/transformers/__init__.py

@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-__version__ = "3.4.0"
+__version__ = "3.3.1"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -54,7 +54,6 @@ from .configuration_mmbt import MMBTConfig
 from .configuration_mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig
 from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
 from .configuration_pegasus import PegasusConfig
-from .configuration_prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig
 from .configuration_rag import RagConfig
 from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig
 from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
@@ -64,7 +63,6 @@ from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config
 from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
 from .configuration_utils import PretrainedConfig
 from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig
-from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig
 from .configuration_xlm_roberta import XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMRobertaConfig
 from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig
 from .data import (
@@ -94,7 +92,6 @@ from .file_utils import (
     MODEL_CARD_NAME,
     PYTORCH_PRETRAINED_BERT_CACHE,
     PYTORCH_TRANSFORMERS_CACHE,
-    SPIECE_UNDERLINE,
     TF2_WEIGHTS_NAME,
     TF_WEIGHTS_NAME,
     TRANSFORMERS_CACHE,
@@ -105,13 +102,10 @@ from .file_utils import (
     is_apex_available,
     is_datasets_available,
     is_faiss_available,
-    is_flax_available,
     is_psutil_available,
     is_py3nvml_available,
-    is_sentencepiece_available,
     is_sklearn_available,
     is_tf_available,
-    is_tokenizers_available,
     is_torch_available,
     is_torch_tpu_available,
 )
@@ -158,42 +152,48 @@ from .pipelines import (
 from .retrieval_rag import RagRetriever
 
 # Tokenizers
+from .tokenization_albert import AlbertTokenizer, AlbertTokenizerFast
 from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
-from .tokenization_bart import BartTokenizer
-from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer
+from .tokenization_bart import BartTokenizer, BartTokenizerFast
+from .tokenization_bert import BasicTokenizer, BertTokenizer, BertTokenizerFast, WordpieceTokenizer
+from .tokenization_bert_generation import BertGenerationTokenizer
 from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer
 from .tokenization_bertweet import BertweetTokenizer
 from .tokenization_blenderbot import BlenderbotSmallTokenizer, BlenderbotTokenizer
+from .tokenization_camembert import CamembertTokenizer, CamembertTokenizerFast
 from .tokenization_ctrl import CTRLTokenizer
 from .tokenization_deberta import DebertaTokenizer
-from .tokenization_distilbert import DistilBertTokenizer
+from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFast
 from .tokenization_dpr import (
     DPRContextEncoderTokenizer,
+    DPRContextEncoderTokenizerFast,
     DPRQuestionEncoderTokenizer,
-    DPRReaderOutput,
+    DPRQuestionEncoderTokenizerFast,
     DPRReaderTokenizer,
+    DPRReaderTokenizerFast,
 )
-from .tokenization_electra import ElectraTokenizer
+from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast
 from .tokenization_flaubert import FlaubertTokenizer
 from .tokenization_fsmt import FSMTTokenizer
-from .tokenization_funnel import FunnelTokenizer
-from .tokenization_gpt2 import GPT2Tokenizer
-from .tokenization_herbert import HerbertTokenizer
-from .tokenization_layoutlm import LayoutLMTokenizer
-from .tokenization_longformer import LongformerTokenizer
-from .tokenization_lxmert import LxmertTokenizer
-from .tokenization_mobilebert import MobileBertTokenizer
-from .tokenization_openai import OpenAIGPTTokenizer
+from .tokenization_funnel import FunnelTokenizer, FunnelTokenizerFast
+from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
+from .tokenization_layoutlm import LayoutLMTokenizer, LayoutLMTokenizerFast
+from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast
+from .tokenization_lxmert import LxmertTokenizer, LxmertTokenizerFast
+from .tokenization_mbart import MBartTokenizer, MBartTokenizerFast
+from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFast
+from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
+from .tokenization_pegasus import PegasusTokenizer, PegasusTokenizerFast
 from .tokenization_phobert import PhobertTokenizer
-from .tokenization_prophetnet import ProphetNetTokenizer
 from .tokenization_rag import RagTokenizer
-from .tokenization_retribert import RetriBertTokenizer
-from .tokenization_roberta import RobertaTokenizer
-from .tokenization_squeezebert import SqueezeBertTokenizer
+from .tokenization_reformer import ReformerTokenizer, ReformerTokenizerFast
+from .tokenization_retribert import RetriBertTokenizer, RetriBertTokenizerFast
+from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
+from .tokenization_squeezebert import SqueezeBertTokenizer, SqueezeBertTokenizerFast
+from .tokenization_t5 import T5Tokenizer, T5TokenizerFast
 from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer
 from .tokenization_utils import PreTrainedTokenizer
 from .tokenization_utils_base import (
-    AddedToken,
     BatchEncoding,
     CharSpan,
     PreTrainedTokenizerBase,
@@ -201,59 +201,10 @@ from .tokenization_utils_base import (
     TensorType,
     TokenSpan,
 )
+from .tokenization_utils_fast import PreTrainedTokenizerFast
 from .tokenization_xlm import XLMTokenizer
-
-
-if is_sentencepiece_available():
-    from .tokenization_albert import AlbertTokenizer
-    from .tokenization_bert_generation import BertGenerationTokenizer
-    from .tokenization_camembert import CamembertTokenizer
-    from .tokenization_marian import MarianTokenizer
-    from .tokenization_mbart import MBartTokenizer
-    from .tokenization_pegasus import PegasusTokenizer
-    from .tokenization_reformer import ReformerTokenizer
-    from .tokenization_t5 import T5Tokenizer
-    from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer
-    from .tokenization_xlm_roberta import XLMRobertaTokenizer
-    from .tokenization_xlnet import XLNetTokenizer
-else:
-    from .utils.dummy_sentencepiece_objects import *
-
-if is_tokenizers_available():
-    from .tokenization_albert_fast import AlbertTokenizerFast
-    from .tokenization_bart_fast import BartTokenizerFast
-    from .tokenization_bert_fast import BertTokenizerFast
-    from .tokenization_camembert_fast import CamembertTokenizerFast
-    from .tokenization_distilbert_fast import DistilBertTokenizerFast
-    from .tokenization_dpr_fast import (
-        DPRContextEncoderTokenizerFast,
-        DPRQuestionEncoderTokenizerFast,
-        DPRReaderTokenizerFast,
-    )
-    from .tokenization_electra_fast import ElectraTokenizerFast
-    from .tokenization_funnel_fast import FunnelTokenizerFast
-    from .tokenization_gpt2_fast import GPT2TokenizerFast
-    from .tokenization_herbert_fast import HerbertTokenizerFast
-    from .tokenization_layoutlm_fast import LayoutLMTokenizerFast
-    from .tokenization_longformer_fast import LongformerTokenizerFast
-    from .tokenization_lxmert_fast import LxmertTokenizerFast
-    from .tokenization_mbart_fast import MBartTokenizerFast
-    from .tokenization_mobilebert_fast import MobileBertTokenizerFast
-    from .tokenization_openai_fast import OpenAIGPTTokenizerFast
-    from .tokenization_pegasus_fast import PegasusTokenizerFast
-    from .tokenization_reformer_fast import ReformerTokenizerFast
-    from .tokenization_retribert_fast import RetriBertTokenizerFast
-    from .tokenization_roberta_fast import RobertaTokenizerFast
-    from .tokenization_squeezebert_fast import SqueezeBertTokenizerFast
-    from .tokenization_t5_fast import T5TokenizerFast
-    from .tokenization_utils_fast import PreTrainedTokenizerFast
-    from .tokenization_xlm_roberta_fast import XLMRobertaTokenizerFast
-    from .tokenization_xlnet_fast import XLNetTokenizerFast
-
-    if is_sentencepiece_available():
-        from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS, convert_slow_tokenizer
-else:
-    from .utils.dummy_tokenizers_objects import *
+from .tokenization_xlm_roberta import XLMRobertaTokenizer, XLMRobertaTokenizerFast
+from .tokenization_xlnet import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
 
 # Trainer
 from .trainer_callback import (
@@ -493,15 +444,6 @@ if is_torch_available():
         load_tf_weights_in_openai_gpt,
     )
     from .modeling_pegasus import PegasusForConditionalGeneration
-    from .modeling_prophetnet import (
-        PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
-        ProphetNetDecoder,
-        ProphetNetEncoder,
-        ProphetNetForCausalLM,
-        ProphetNetForConditionalGeneration,
-        ProphetNetModel,
-        ProphetNetPreTrainedModel,
-    )
     from .modeling_rag import RagModel, RagSequenceForGeneration, RagTokenForGeneration
     from .modeling_reformer import (
         REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
@@ -562,14 +504,6 @@ if is_torch_available():
         XLMPreTrainedModel,
         XLMWithLMHeadModel,
     )
-    from .modeling_xlm_prophetnet import (
-        XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
-        XLMProphetNetDecoder,
-        XLMProphetNetEncoder,
-        XLMProphetNetForCausalLM,
-        XLMProphetNetForConditionalGeneration,
-        XLMProphetNetModel,
-    )
     from .modeling_xlm_roberta import (
         XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
         XLMRobertaForCausalLM,
@@ -604,6 +538,7 @@ if is_torch_available():
         get_linear_schedule_with_warmup,
         get_polynomial_decay_schedule_with_warmup,
     )
+    from .tokenization_marian import MarianTokenizer
 
     # Trainer
     from .trainer import Trainer
@@ -838,15 +773,6 @@ else:
     from .utils.dummy_tf_objects import *
 
 
-if is_flax_available():
-    from .modeling_flax_bert import FlaxBertModel
-    from .modeling_flax_roberta import FlaxRobertaModel
-else:
-    # Import the same objects as dummies to get them in the namespace.
-    # They will raise an import error if the user tries to instantiate / use them.
-    from .utils.dummy_flax_objects import *
-
-
 if not is_tf_available() and not is_torch_available():
     logger.warning(
         "Neither PyTorch nor TensorFlow >= 2.0 have been found."

src/transformers/activations.py

@@ -66,3 +66,7 @@ def get_activation(activation_string):
         return ACT2FN[activation_string]
     else:
         raise KeyError("function {} not found in ACT2FN mapping {}".format(activation_string, list(ACT2FN.keys())))
+
+
+def mish(x):
+    return x * torch.tanh(torch.nn.functional.softplus(x))

src/transformers/configuration_auto.py

@@ -41,7 +41,6 @@ from .configuration_mbart import MBART_PRETRAINED_CONFIG_ARCHIVE_MAP, MBartConfi
 from .configuration_mobilebert import MobileBertConfig
 from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
 from .configuration_pegasus import PegasusConfig
-from .configuration_prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig
 from .configuration_rag import RagConfig
 from .configuration_reformer import ReformerConfig
 from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
@@ -51,7 +50,6 @@ from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config
 from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
 from .configuration_utils import PretrainedConfig
 from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig
-from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig
 from .configuration_xlm_roberta import XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMRobertaConfig
 from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig
 
@@ -86,8 +84,6 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
         DPR_PRETRAINED_CONFIG_ARCHIVE_MAP,
         DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
         SQUEEZEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
-        XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
-        PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
     ]
     for key, value, in pretrained_map.items()
 )
@@ -120,8 +116,6 @@ CONFIG_MAPPING = OrderedDict(
         ("gpt2", GPT2Config),
         ("transfo-xl", TransfoXLConfig),
         ("xlnet", XLNetConfig),
-        ("xlm-prophetnet", XLMProphetNetConfig),
-        ("prophetnet", ProphetNetConfig),
         ("xlm", XLMConfig),
         ("ctrl", CTRLConfig),
         ("electra", ElectraConfig),
@@ -170,8 +164,6 @@ MODEL_NAMES_MAPPING = OrderedDict(
         ("layoutlm", "LayoutLM"),
         ("dpr", "DPR"),
         ("rag", "RAG"),
-        ("xlm-prophetnet", "XLMProphetNet"),
-        ("prophetnet", "ProphetNet"),
     ]
 )
 
@@ -274,7 +266,7 @@ class AutoConfig:
                       our S3, e.g., ``dbmdz/bert-base-german-cased``.
                     - A path to a `directory` containing a configuration file saved using the
                       :meth:`~transformers.PretrainedConfig.save_pretrained` method, or the
-                      :meth:`~transformers.PreTrainedModel.save_pretrained` method, e.g., ``./my_model_directory/``.
+                      :meth:`~transformers.PretrainedModel.save_pretrained` method, e.g., ``./my_model_directory/``.
                     - A path or url to a saved configuration JSON `file`, e.g.,
                       ``./my_model_directory/configuration.json``.
             cache_dir (:obj:`str`, `optional`):

src/transformers/configuration_dpr.py

@@ -24,9 +24,6 @@ DPR_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "facebook/dpr-ctx_encoder-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-ctx_encoder-single-nq-base/config.json",
     "facebook/dpr-question_encoder-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-question_encoder-single-nq-base/config.json",
     "facebook/dpr-reader-single-nq-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-reader-single-nq-base/config.json",
-    "facebook/dpr-ctx_encoder-multiset-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-ctx_encoder-multiset-base/config.json",
-    "facebook/dpr-question_encoder-multiset-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-question_encoder-multiset-base/config.json",
-    "facebook/dpr-reader-multiset-base": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/dpr-reader-multiset-base/config.json",
 }
 
 

src/transformers/configuration_lxmert.py

@@ -15,11 +15,12 @@
 """ LXMERT model configuration """
 
 
+import logging
+
 from .configuration_utils import PretrainedConfig
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "unc-nlp/lxmert-base-uncased": "",

src/transformers/configuration_prophetnet.py

@@ -1,167 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Microsoft Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" ProphetNet model configuration """
-
-
-from .configuration_utils import PretrainedConfig
-from .utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "microsoft/prophetnet-large-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/microsoft/prophetnet-large-uncased/config.json",
-}
-
-
-class ProphetNetConfig(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a :class:`~transformers.ProphetNetModel`. It is used to
-    instantiate a ProphetNet model according to the specified arguments, defining the model architecture.
-
-    Configuration objects inherit from  :class:`~transformers.PretrainedConfig` and can be used
-    to control the model outputs. Read the documentation from  :class:`~transformers.PretrainedConfig`
-    for more information.
-
-    Args:
-        activation_dropout (:obj:`float`, `optional`, defaults to 0.1):
-            The dropout ratio for activations inside the fully connected layer.
-        activation_function (:obj:`str` or :obj:`function`, `optional`, defaults to :obj:`"gelu"`):
-            The non-linear activation function (function or string) in the encoder and pooler.
-            If string, :obj:`"gelu"`, :obj:`"relu"`, :obj:`"swish"` and :obj:`"gelu_new"` are supported.
-        vocab_size (:obj:`int`, `optional`, defaults to 30522):
-            Vocabulary size of the ProphetNET model. Defines the number of different tokens that can be represented by the
-            :obj:`inputs_ids` passed when calling :class:`~transformers.ProphetNetModel`.
-        hidden_size (:obj:`int`, `optional`, defaults to 1024):
-            Dimensionality of the layers and the pooler layer.
-        encoder_ffn_dim (:obj:`int`, `optional`, defaults to 4096):
-            Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
-        num_encoder_layers (:obj:`int`, `optional`, defaults to 12):
-            Number of encoder layers.
-        num_encoder_attention_heads (:obj:`int`, `optional`, defaults to 16):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        decoder_ffn_dim (:obj:`int`, `optional`, defaults to 4096):
-            Dimensionality of the ``intermediate`` (often named feed-forward) layer in decoder.
-        num_decoder_layers (:obj:`int`, `optional`, defaults to 12):
-            Number of decoder layers.
-        num_decoder_attention_heads (:obj:`int`, `optional`, defaults to 16):
-            Number of attention heads for each attention layer in the Transformer decoder.
-        attention_dropout (:obj:`float`, `optional`, defaults to 0.1):
-            The dropout ratio for the attention probabilities.
-        dropout (:obj:`float`, `optional`, defaults to 0.1):
-            The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
-        max_position_embeddings (:obj:`int`, `optional`, defaults to 512):
-            The maximum sequence length that this model might ever be used with.
-            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
-        init_std (:obj:`float`, `optional`, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        add_cross_attention (:obj:`bool`, `optional`, defaults to :obj:`True`):
-            Whether cross-attention layers should be added to the model.
-        is_encoder_decoder (:obj:`bool`, `optional`, defaults to :obj:`True`):
-            Whether this is an encoder/decoder model.
-        pad_token_id (:obj:`int`, `optional`, defaults to 1)
-            Padding token id.
-        bos_token_id (:obj:`int`, `optional`, defaults to 0)
-            Beginning of stream token id.
-        eos_token_id (:obj:`int`, `optional`, defaults to 2)
-            End of stream token id.
-        ngram (:obj:`int`, `optional`, defaults to 2)
-            Number of future tokens to predict.
-            Set to 1 to be same as traditional Language model to predict next first token.
-        num_buckets (:obj:`int`, `optional`, defaults to 32)
-            The number of buckets to use for each attention layer.
-            This is for relative position calculation. See the `T5 paper
-            <see https://arxiv.org/abs/1910.10683>`__ for more details.
-        relative_max_distance (:obj:`int`, `optional`, defaults to 128)
-            Relative distances greater than this number will be put into the last same bucket.
-            This is for relative position calculation. See the `T5 paper
-            <see https://arxiv.org/abs/1910.10683>`__ for more details.
-        disable_ngram_loss (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether be trained predicting only the next first token.
-        eps (:obj:`float`, `optional`, defaults to 0.0):
-            Controls the ``epsilon`` parameter value for label
-            smoothing in the loss calculation. If set to 0, no label smoothing is performed.
-    """
-    model_type = "prophetnet"
-
-    def __init__(
-        self,
-        activation_dropout=0.1,
-        activation_function="gelu",
-        vocab_size=30522,
-        hidden_size=1024,
-        encoder_ffn_dim=4096,
-        num_encoder_layers=12,
-        num_encoder_attention_heads=16,
-        decoder_ffn_dim=4096,
-        num_decoder_layers=12,
-        num_decoder_attention_heads=16,
-        attention_dropout=0.1,
-        dropout=0.1,
-        max_position_embeddings=512,
-        init_std=0.02,
-        is_encoder_decoder=True,
-        add_cross_attention=True,
-        pad_token_id=0,
-        bos_token_id=1,
-        eos_token_id=2,
-        decoder_start_token_id=0,
-        ngram=2,
-        num_buckets=32,
-        relative_max_distance=128,
-        disable_ngram_loss=False,
-        eps=0.0,
-        **kwargs
-    ):
-        super().__init__(
-            pad_token_id=pad_token_id,
-            bos_token_id=bos_token_id,
-            eos_token_id=eos_token_id,
-            is_encoder_decoder=is_encoder_decoder,
-            add_cross_attention=add_cross_attention,
-            decoder_start_token_id=decoder_start_token_id,
-            **kwargs,
-        )
-        self.vocab_size = vocab_size
-        self.hidden_size = hidden_size
-        self.encoder_ffn_dim = encoder_ffn_dim
-        self.num_encoder_layers = num_encoder_layers
-        self.num_encoder_attention_heads = num_encoder_attention_heads
-        self.decoder_ffn_dim = decoder_ffn_dim
-        self.num_decoder_layers = num_decoder_layers
-        self.num_decoder_attention_heads = num_decoder_attention_heads
-        self.max_position_embeddings = max_position_embeddings
-        self.init_std = init_std  # Normal(0, this parameter)
-        self.activation_function = activation_function
-
-        # parameters for prophetnet
-        self.ngram = ngram
-        self.num_buckets = num_buckets
-        self.relative_max_distance = relative_max_distance
-        self.disable_ngram_loss = disable_ngram_loss
-        self.eps = eps
-
-        # 3 Types of Dropout
-        self.attention_dropout = attention_dropout
-        self.activation_dropout = activation_dropout
-        self.dropout = dropout
-
-    @property
-    def num_attention_heads(self) -> int:
-        return self.num_encoder_attention_heads
-
-    @property
-    def num_hidden_layers(self) -> int:
-        return self.num_encoder_layers + self.num_decoder_layers

src/transformers/configuration_rag.py

@@ -41,7 +41,7 @@ RAG_CONFIG_DOC = r"""
             Retrieval batch size, defined as the number of queries issues concurrently to the faiss index excapsulated
             :class:`~transformers.RagRetriever`.
         dataset (:obj:`str`, `optional`, defaults to :obj:`"wiki_dpr"`):
-            A dataset identifier of the indexed dataset in HuggingFace Datasets (list all available datasets and
+            A dataset identifier of the indexed dataset on HuggingFace AWS bucket (list all available datasets and
             ids using :obj:`datasets.list_datasets()`).
         dataset_split (:obj:`str`, `optional`, defaults to :obj:`"train"`)
             Which split of the :obj:`dataset` to load.

src/transformers/configuration_utils.py

@@ -43,9 +43,6 @@ class PretrainedConfig(object):
           recreate the correct object in :class:`~transformers.AutoConfig`.
 
     Args:
-        name_or_path (:obj:`str`, `optional`, defaults to :obj:`""`):
-            Store the string that was passed to :func:`~transformers.PreTrainedModel.from_pretrained` or :func:`~transformers.TFPreTrainedModel.from_pretrained`
-            as ``pretrained_model_name_or_path`` if the configuration was created with such a method.
         output_hidden_states (:obj:`bool`, `optional`, defaults to :obj:`False`):
             Whether or not the model should return all hidden-states.
         output_attentions (:obj:`bool`, `optional`, defaults to :obj:`False`):
@@ -209,9 +206,6 @@ class PretrainedConfig(object):
         # TPU arguments
         self.xla_device = kwargs.pop("xla_device", None)
 
-        # Name or path to the pretrained checkpoint
-        self._name_or_path = str(kwargs.pop("name_or_path", ""))
-
         # Additional attributes without default values
         for key, value in kwargs.items():
             try:
@@ -220,14 +214,6 @@ class PretrainedConfig(object):
                 logger.error("Can't set {} with value {} for {}".format(key, value, self))
                 raise err
 
-    @property
-    def name_or_path(self) -> str:
-        return self._name_or_path
-
-    @name_or_path.setter
-    def name_or_path(self, value):
-        self._name_or_path = str(value)  # Make sure that name_or_path is a string (for JSON encoding)
-
     @property
     def use_return_dict(self) -> bool:
         """

src/transformers/configuration_xlm_prophetnet.py

@@ -1,35 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Microsoft Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" XLM-ProphetNet model configuration """
-
-
-from .configuration_prophetnet import ProphetNetConfig
-from .utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "microsoft/xprophetnet-large-wiki100-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/microsoft/xprophetnet-large-wiki100-cased/config.json",
-}
-
-
-class XLMProphetNetConfig(ProphetNetConfig):
-    """
-    This class overrides :class:`~transformers.ProphetNetConfig`. Please check the
-    superclass for the appropriate documentation alongside usage examples.
-    """
-
-    model_type = "xlm-prophetnet"

src/transformers/convert_dpr_original_checkpoint_to_pytorch.py

@@ -44,8 +44,7 @@ class DPRContextEncoderState(DPRState):
         print("Loading DPR biencoder from {}".format(self.src_file))
         saved_state = load_states_from_checkpoint(self.src_file)
         encoder, prefix = model.ctx_encoder, "ctx_model."
-        # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
-        state_dict = {"bert_model.embeddings.position_ids": model.ctx_encoder.bert_model.embeddings.position_ids}
+        state_dict = {}
         for key, value in saved_state.model_dict.items():
             if key.startswith(prefix):
                 key = key[len(prefix) :]
@@ -62,8 +61,7 @@ class DPRQuestionEncoderState(DPRState):
         print("Loading DPR biencoder from {}".format(self.src_file))
         saved_state = load_states_from_checkpoint(self.src_file)
         encoder, prefix = model.question_encoder, "question_model."
-        # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
-        state_dict = {"bert_model.embeddings.position_ids": model.question_encoder.bert_model.embeddings.position_ids}
+        state_dict = {}
         for key, value in saved_state.model_dict.items():
             if key.startswith(prefix):
                 key = key[len(prefix) :]
@@ -79,10 +77,7 @@ class DPRReaderState(DPRState):
         model = DPRReader(DPRConfig(**BertConfig.get_config_dict("bert-base-uncased")[0]))
         print("Loading DPR reader from {}".format(self.src_file))
         saved_state = load_states_from_checkpoint(self.src_file)
-        # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
-        state_dict = {
-            "encoder.bert_model.embeddings.position_ids": model.span_predictor.encoder.bert_model.embeddings.position_ids
-        }
+        state_dict = {}
         for key, value in saved_state.model_dict.items():
             if key.startswith("encoder.") and not key.startswith("encoder.encode_proj"):
                 key = "encoder.bert_model." + key[len("encoder.") :]

src/transformers/convert_pegasus_tf_to_pytorch.py

@@ -54,6 +54,8 @@ def rename_state_dict_key(k):
 
 # See appendix C of paper for all hyperparams
 
+# TODO(SS): one constant
+
 
 def convert_pegasus(tf_weights: dict, cfg_updates: dict) -> PegasusForConditionalGeneration:
     cfg_kwargs = DEFAULTS.copy()

src/transformers/convert_prophetnet_original_pytorch_checkpoint_to_pytorch.py

@@ -1,162 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Convert ProphetNet checkpoint."""
-
-
-import argparse
-
-import torch
-
-from transformers import logging
-from transformers.modeling_prophetnet import ProphetNetForConditionalGeneration
-from transformers.modeling_xlm_prophetnet import XLMProphetNetForConditionalGeneration
-
-# transformers_old should correspond to branch `save_old_prophetnet_model_structure` here
-# original prophetnet_checkpoints are saved under `patrickvonplaten/..._old` respectively
-from transformers_old.modeling_prophetnet import (
-    ProphetNetForConditionalGeneration as ProphetNetForConditionalGenerationOld,
-)
-from transformers_old.modeling_xlm_prophetnet import (
-    XLMProphetNetForConditionalGeneration as XLMProphetNetForConditionalGenerationOld,
-)
-
-
-logger = logging.get_logger(__name__)
-logging.set_verbosity_info()
-
-
-def convert_prophetnet_checkpoint_to_pytorch(prophetnet_checkpoint_path: str, pytorch_dump_folder_path: str):
-    """
-    Copy/paste/tweak prohpetnet's weights to our prophetnet structure.
-    """
-    if "xprophetnet" in prophetnet_checkpoint_path:
-        prophet_old = XLMProphetNetForConditionalGenerationOld.from_pretrained(prophetnet_checkpoint_path)
-        prophet, loading_info = XLMProphetNetForConditionalGeneration.from_pretrained(
-            prophetnet_checkpoint_path, output_loading_info=True
-        )
-    else:
-        prophet_old = ProphetNetForConditionalGenerationOld.from_pretrained(prophetnet_checkpoint_path)
-        prophet, loading_info = ProphetNetForConditionalGeneration.from_pretrained(
-            prophetnet_checkpoint_path, output_loading_info=True
-        )
-
-    special_keys = ["key_proj", "value_proj", "query_proj"]
-
-    mapping = {
-        "self_attn": "ngram_self_attn",
-        "cross_attn": "encoder_attn",
-        "cross_attn_layer_norm": "encoder_attn_layer_norm",
-        "feed_forward_layer_norm": "final_layer_norm",
-        "feed_forward": "",
-        "intermediate": "fc1",
-        "output": "fc2",
-        "key_proj": "k_proj",
-        "query_proj": "q_proj",
-        "value_proj": "v_proj",
-        "word_embeddings": "embed_tokens",
-        "embeddings_layer_norm": "emb_layer_norm",
-        "relative_pos_embeddings": "relative_linear",
-        "ngram_embeddings": "ngram_input_embed",
-        "position_embeddings": "embed_positions",
-    }
-
-    for key in loading_info["missing_keys"]:
-        attributes = key.split(".")
-
-        if attributes[0] == "lm_head":
-            model = prophet
-            old_model = prophet_old
-        else:
-            model = prophet.prophetnet
-            old_model = prophet_old.model
-
-        is_key_init = False
-        for attribute in attributes:
-            if attribute in mapping:
-                old_attribute = mapping[attribute]
-                if not hasattr(old_model, old_attribute) and len(old_attribute) > 0:
-                    old_attribute = attribute
-            elif hasattr(old_model, attribute):
-                old_attribute = attribute
-
-            if attribute == "weight":
-                assert old_model.weight.shape == model.weight.shape, "Shapes have to match!"
-                model.weight = old_model.weight
-                logger.info(f"{attribute} is initialized.")
-                is_key_init = True
-                break
-            elif attribute == "bias":
-                assert old_model.bias.shape == model.bias.shape, "Shapes have to match!"
-                model.bias = old_model.bias
-                logger.info(f"{attribute} is initialized")
-                is_key_init = True
-                break
-            elif attribute in special_keys and hasattr(old_model, "in_proj_weight"):
-                embed_dim = old_model.in_proj_weight.shape[0] // 3
-                param = getattr(model, attribute)
-                param.weight.shape == old_model.in_proj_weight[:embed_dim, :].shape, "Shapes have to match"
-                param.bias.shape == old_model.in_proj_bias[:embed_dim].shape, "Shapes have to match"
-                if attribute == "query_proj":
-                    model.query_proj.weight = torch.nn.Parameter(old_model.in_proj_weight[:embed_dim, :])
-                    model.query_proj.bias = torch.nn.Parameter(old_model.in_proj_bias[:embed_dim])
-
-                elif attribute == "key_proj":
-                    model.key_proj.weight = torch.nn.Parameter(old_model.in_proj_weight[embed_dim : 2 * embed_dim, :])
-                    model.key_proj.bias = torch.nn.Parameter(old_model.in_proj_bias[embed_dim : 2 * embed_dim])
-                elif attribute == "value_proj":
-                    model.value_proj.weight = torch.nn.Parameter(old_model.in_proj_weight[2 * embed_dim :, :])
-                    model.value_proj.bias = torch.nn.Parameter(old_model.in_proj_bias[2 * embed_dim :])
-                is_key_init = True
-                break
-            elif attribute == "position_embeddings":
-                assert (
-                    model.position_embeddings.weight.shape[-1] == old_model.embed_positions.weight.shape[-1]
-                ), "Hidden size has to match"
-                assert model.position_embeddings.weight.shape[0] == 512, "We want 512 position_embeddings."
-                model.position_embeddings.weight = torch.nn.Parameter(old_model.embed_positions.weight[:512, :])
-                is_key_init = True
-                break
-
-            if attribute.isdigit():
-                model = model[int(attribute)]
-                old_model = old_model[int(old_attribute)]
-            else:
-                model = getattr(model, attribute)
-
-                if old_attribute == "":
-                    old_model = old_model
-                else:
-                    if not hasattr(old_model, old_attribute):
-                        raise ValueError(f"{old_model} does not have {old_attribute}")
-                    old_model = getattr(old_model, old_attribute)
-
-        if not is_key_init:
-            raise ValueError(f"{key} was not correctly initialized!")
-
-    print(f"Saving model to {pytorch_dump_folder_path}")
-    prophet.save_pretrained(pytorch_dump_folder_path)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    # Required parameters
-    parser.add_argument(
-        "--prophetnet_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
-    )
-    parser.add_argument(
-        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
-    )
-    args = parser.parse_args()
-    convert_prophetnet_checkpoint_to_pytorch(args.prophetnet_checkpoint_path, args.pytorch_dump_folder_path)

src/transformers/convert_slow_tokenizer.py

@@ -20,14 +20,13 @@
 
 from typing import Dict, List, Tuple
 
+from sentencepiece import SentencePieceProcessor
 from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
 from tokenizers.models import BPE, Unigram, WordPiece
 
 # from transformers.tokenization_openai import OpenAIGPTTokenizer
 from transformers.utils import sentencepiece_model_pb2 as model
 
-from .file_utils import requires_sentencepiece
-
 
 class SentencePieceExtractor:
     """
@@ -36,9 +35,7 @@ class SentencePieceExtractor:
     """
 
     def __init__(self, model: str):
-        requires_sentencepiece(self)
-        from sentencepiece import SentencePieceProcessor
-
+        # Get SentencePiece
         self.sp = SentencePieceProcessor()
         self.sp.Load(model)
 
@@ -230,37 +227,6 @@ class GPT2Converter(Converter):
         return tokenizer
 
 
-class HerbertConverter(Converter):
-    def converted(self) -> Tokenizer:
-        tokenizer_info_str = "#version:"
-        token_suffix = "</w>"
-
-        vocab = self.original_tokenizer.encoder
-        merges = list(self.original_tokenizer.bpe_ranks.keys())
-        if tokenizer_info_str in merges[0][0]:
-            merges = merges[1:]
-
-        tokenizer = Tokenizer(
-            BPE(
-                vocab,
-                merges,
-                dropout=None,
-                unk_token=self.original_tokenizer.unk_token,
-                end_of_word_suffix=token_suffix,
-            )
-        )
-
-        tokenizer.normalizer = normalizers.BertNormalizer(lowercase=False, strip_accents=False)
-        tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
-        tokenizer.decoder = decoders.BPEDecoder(suffix=token_suffix)
-        tokenizer.post_processor = processors.BertProcessing(
-            sep=(self.original_tokenizer.sep_token, self.original_tokenizer.sep_token_id),
-            cls=(self.original_tokenizer.cls_token, self.original_tokenizer.cls_token_id),
-        )
-
-        return tokenizer
-
-
 class RobertaConverter(Converter):
     def converted(self) -> Tokenizer:
         ot = self.original_tokenizer
@@ -571,10 +537,11 @@ class T5Converter(SpmConverter):
         )
 
 
-SLOW_TO_FAST_CONVERTERS = {
+CONVERTERS = {
     "AlbertTokenizer": AlbertConverter,
-    "BartTokenizer": RobertaConverter,
     "BertTokenizer": BertConverter,
+    "BertGenerationTokenizer": BertGenerationConverter,
+    "BartTokenizer": RobertaConverter,
     "CamembertTokenizer": CamembertConverter,
     "DistilBertTokenizer": BertConverter,
     "DPRReaderTokenizer": BertConverter,
@@ -583,18 +550,12 @@ SLOW_TO_FAST_CONVERTERS = {
     "ElectraTokenizer": BertConverter,
     "FunnelTokenizer": FunnelConverter,
     "GPT2Tokenizer": GPT2Converter,
-    "HerbertTokenizer": HerbertConverter,
-    "LayoutLMTokenizer": BertConverter,
-    "LongformerTokenizer": RobertaConverter,
     "LxmertTokenizer": BertConverter,
     "MBartTokenizer": MBartConverter,
-    "MobileBertTokenizer": BertConverter,
     "OpenAIGPTTokenizer": OpenAIGPTConverter,
     "PegasusTokenizer": PegasusConverter,
     "ReformerTokenizer": ReformerConverter,
-    "RetriBertTokenizer": BertConverter,
     "RobertaTokenizer": RobertaConverter,
-    "SqueezeBertTokenizer": BertConverter,
     "T5Tokenizer": T5Converter,
     "XLMRobertaTokenizer": XLMRobertaConverter,
     "XLNetTokenizer": XLNetConverter,
@@ -602,26 +563,5 @@ SLOW_TO_FAST_CONVERTERS = {
 
 
 def convert_slow_tokenizer(transformer_tokenizer) -> Tokenizer:
-    """Utilities to convert a slow tokenizer instance in a fast tokenizer instance.
-
-    Args:
-        transformer_tokenizer (:class:`~transformers.tokenization_utils_base.PreTrainedTokenizer`):
-            Instance of a slow tokenizer to convert in the backend tokenizer for
-            :class:`~transformers.tokenization_utils_base.PreTrainedTokenizerFast`.
-
-    Return:
-        A instance of :class:`~tokenizers.Tokenizer` to be used as the backend tokenizer of a
-        :class:`~transformers.tokenization_utils_base.PreTrainedTokenizerFast`
-    """
-
-    tokenizer_class_name = transformer_tokenizer.__class__.__name__
-
-    if tokenizer_class_name not in SLOW_TO_FAST_CONVERTERS:
-        raise ValueError(
-            f"An instance of tokenizer class {tokenizer_class_name} cannot be converted in a Fast tokenizer instance. "
-            f"No converter was found. Currently available slow->fast convertors: {list(SLOW_TO_FAST_CONVERTERS.keys())}"
-        )
-
-    converter_class = SLOW_TO_FAST_CONVERTERS[tokenizer_class_name]
-
+    converter_class = CONVERTERS[transformer_tokenizer.__class__.__name__]
     return converter_class(transformer_tokenizer).converted()

src/transformers/convert_slow_tokenizers_checkpoints_to_fast.py

@@ -1,129 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" Convert slow tokenizers checkpoints in fast (serialization format of the `tokenizers` library) """
-
-import argparse
-import os
-
-import transformers
-from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
-from transformers.utils import logging
-
-
-logging.set_verbosity_info()
-
-logger = logging.get_logger(__name__)
-
-
-TOKENIZER_CLASSES = {name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}
-
-
-def convert_slow_checkpoint_to_fast(tokenizer_name, checkpoint_name, dump_path, force_download):
-    if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
-        raise ValueError("Unrecognized tokenizer name, should be one of {}.".format(list(TOKENIZER_CLASSES.keys())))
-
-    if tokenizer_name is None:
-        tokenizer_names = TOKENIZER_CLASSES
-    else:
-        tokenizer_names = {tokenizer_name: getattr(transformers, tokenizer_name + "Fast")}
-
-    logger.info(f"Loading tokenizer classes: {tokenizer_names}")
-
-    for tokenizer_name in tokenizer_names:
-        tokenizer_class = TOKENIZER_CLASSES[tokenizer_name]
-
-        add_prefix = True
-        if checkpoint_name is None:
-            checkpoint_names = list(tokenizer_class.max_model_input_sizes.keys())
-        else:
-            checkpoint_names = [checkpoint_name]
-
-        logger.info(f"For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}")
-
-        for checkpoint in checkpoint_names:
-            logger.info(f"Loading {tokenizer_class.__class__.__name__} {checkpoint}")
-
-            # Load tokenizer
-            tokenizer = tokenizer_class.from_pretrained(checkpoint, force_download=force_download)
-
-            # Save fast tokenizer
-            logger.info(
-                "Save fast tokenizer to {} with prefix {} add_prefix {}".format(dump_path, checkpoint, add_prefix)
-            )
-
-            # For organization names we create sub-directories
-            if "/" in checkpoint:
-                checkpoint_directory, checkpoint_prefix_name = checkpoint.split("/")
-                dump_path_full = os.path.join(dump_path, checkpoint_directory)
-            elif add_prefix:
-                checkpoint_prefix_name = checkpoint
-                dump_path_full = dump_path
-            else:
-                checkpoint_prefix_name = None
-                dump_path_full = dump_path
-
-            logger.info(
-                "=> {} with prefix {}, add_prefix {}".format(dump_path_full, checkpoint_prefix_name, add_prefix)
-            )
-
-            file_path = list(tokenizer.pretrained_vocab_files_map.values())[0][checkpoint]
-            next_char = file_path.split(checkpoint)[-1][0]
-            if next_char == "/":
-                dump_path_full = os.path.join(dump_path_full, checkpoint_prefix_name)
-                checkpoint_prefix_name = None
-
-            logger.info(
-                "=> {} with prefix {}, add_prefix {}".format(dump_path_full, checkpoint_prefix_name, add_prefix)
-            )
-
-            file_names = tokenizer.save_pretrained(
-                dump_path_full, legacy_format=False, filename_prefix=checkpoint_prefix_name
-            )
-            logger.info("=> File names {}".format(file_names))
-
-            for file_name in file_names:
-                if not file_name.endswith("tokenizer.json"):
-                    os.remove(file_name)
-                    logger.info("=> removing {}".format(file_name))
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    # Required parameters
-    parser.add_argument(
-        "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
-    )
-    parser.add_argument(
-        "--tokenizer_name",
-        default=None,
-        type=str,
-        help="Optional tokenizer type selected in the list of {}. If not given, will download and convert all the checkpoints from AWS.".format(
-            list(TOKENIZER_CLASSES.keys())
-        ),
-    )
-    parser.add_argument(
-        "--checkpoint_name",
-        default=None,
-        type=str,
-        help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
-    )
-    parser.add_argument(
-        "--force_download",
-        action="store_true",
-        help="Re-dowload checkpoints.",
-    )
-    args = parser.parse_args()
-
-    convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

src/transformers/data/data_collator.py

@@ -4,7 +4,9 @@ from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union
 import torch
 from torch.nn.utils.rnn import pad_sequence
 
-from ..tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTrainedTokenizerBase
+from ..tokenization_utils import PreTrainedTokenizer
+from ..tokenization_utils_base import BatchEncoding, PaddingStrategy
+from ..tokenization_utils_fast import PreTrainedTokenizerFast
 
 
 InputDataClass = NewType("InputDataClass", Any)
@@ -92,7 +94,7 @@ class DataCollatorWithPadding:
             >= 7.5 (Volta).
     """
 
-    tokenizer: PreTrainedTokenizerBase
+    tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
     padding: Union[bool, str, PaddingStrategy] = True
     max_length: Optional[int] = None
     pad_to_multiple_of: Optional[int] = None
@@ -122,7 +124,7 @@ class DataCollatorForLanguageModeling:
     - preprocesses batches for masked language modeling
     """
 
-    tokenizer: PreTrainedTokenizerBase
+    tokenizer: PreTrainedTokenizer
     mlm: bool = True
     mlm_probability: float = 0.15
 
@@ -272,7 +274,7 @@ class DataCollatorForPermutationLanguageModeling:
     - preprocesses batches for permutation language modeling with procedures specific to XLNet
     """
 
-    tokenizer: PreTrainedTokenizerBase
+    tokenizer: PreTrainedTokenizer
     plm_probability: float = 1 / 6
     max_span_length: int = 5  # maximum length of a span of masked tokens
 
@@ -404,7 +406,7 @@ class DataCollatorForNextSentencePrediction:
     - preprocesses batches for masked language modeling
     """
 
-    tokenizer: PreTrainedTokenizerBase
+    tokenizer: PreTrainedTokenizer
     mlm: bool = True
     block_size: int = 512
     short_seq_probability: float = 0.1
@@ -447,6 +449,8 @@ class DataCollatorForNextSentencePrediction:
             "labels": mlm_labels if self.mlm else None,
             "next_sentence_label": torch.tensor(nsp_labels),
         }
+        if self.mlm:
+            result["masked_lm_labels"] = mlm_labels
         return result
 
     def _tensorize_batch(self, examples: List[torch.Tensor]) -> torch.Tensor:

src/transformers/data/datasets/glue.py

@@ -9,7 +9,10 @@ from torch.utils.data.dataset import Dataset
 
 from filelock import FileLock
 
-from ...tokenization_utils_base import PreTrainedTokenizerBase
+from ...tokenization_bart import BartTokenizer, BartTokenizerFast
+from ...tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
+from ...tokenization_utils import PreTrainedTokenizer
+from ...tokenization_xlm_roberta import XLMRobertaTokenizer
 from ...utils import logging
 from ..processors.glue import glue_convert_examples_to_features, glue_output_modes, glue_processors
 from ..processors.utils import InputFeatures
@@ -66,7 +69,7 @@ class GlueDataset(Dataset):
     def __init__(
         self,
         args: GlueDataTrainingArguments,
-        tokenizer: PreTrainedTokenizerBase,
+        tokenizer: PreTrainedTokenizer,
         limit_length: Optional[int] = None,
         mode: Union[str, Split] = Split.train,
         cache_dir: Optional[str] = None,
@@ -90,12 +93,12 @@ class GlueDataset(Dataset):
             ),
         )
         label_list = self.processor.get_labels()
-        if args.task_name in ["mnli", "mnli-mm"] and tokenizer.__class__.__name__ in (
-            "RobertaTokenizer",
-            "RobertaTokenizerFast",
-            "XLMRobertaTokenizer",
-            "BartTokenizer",
-            "BartTokenizerFast",
+        if args.task_name in ["mnli", "mnli-mm"] and tokenizer.__class__ in (
+            RobertaTokenizer,
+            RobertaTokenizerFast,
+            XLMRobertaTokenizer,
+            BartTokenizer,
+            BartTokenizerFast,
         ):
             # HACK(label indices are swapped in RoBERTa pretrained model)
             label_list[1], label_list[2] = label_list[2], label_list[1]

src/transformers/file_utils.py

@@ -34,13 +34,10 @@ from .utils import logging
 
 logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
 
-ENV_VARS_TRUE_VALUES = {"1", "ON", "YES"}
-ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"})
-
 try:
     USE_TF = os.environ.get("USE_TF", "AUTO").upper()
     USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper()
-    if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
+    if USE_TORCH in ("1", "ON", "YES", "AUTO") and USE_TF not in ("1", "ON", "YES"):
         import torch
 
         _torch_available = True  # pylint: disable=invalid-name
@@ -55,7 +52,7 @@ try:
     USE_TF = os.environ.get("USE_TF", "AUTO").upper()
     USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper()
 
-    if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
+    if USE_TF in ("1", "ON", "YES", "AUTO") and USE_TORCH not in ("1", "ON", "YES"):
         import tensorflow as tf
 
         assert hasattr(tf, "__version__") and int(tf.__version__[0]) >= 2
@@ -68,22 +65,6 @@ except (ImportError, AssertionError):
     _tf_available = False  # pylint: disable=invalid-name
 
 
-try:
-    USE_JAX = os.environ.get("USE_FLAX", "AUTO").upper()
-
-    if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
-        import flax
-        import jax
-
-        logger.info("JAX version {}, Flax: available".format(jax.__version__))
-        logger.info("Flax available: {}".format(flax))
-        _flax_available = True
-    else:
-        _flax_available = False
-except ImportError:
-    _flax_available = False  # pylint: disable=invalid-name
-
-
 try:
     import datasets  # noqa: F401
 
@@ -161,38 +142,6 @@ try:
 except (AttributeError, ImportError):
     _has_sklearn = False
 
-try:
-    # Test copied from tqdm.autonotebook: https://github.com/tqdm/tqdm/blob/master/tqdm/autonotebook.py
-    get_ipython = sys.modules["IPython"].get_ipython
-    if "IPKernelApp" not in get_ipython().config:
-        raise ImportError("console")
-    if "VSCODE_PID" in os.environ:
-        raise ImportError("vscode")
-
-    import IPython  # noqa: F401
-
-    _in_notebook = True
-except (AttributeError, ImportError, KeyError):
-    _in_notebook = False
-
-
-try:
-    import sentencepiece  # noqa: F401
-
-    _sentencepiece_available = True
-
-except ImportError:
-    _sentencepiece_available = False
-
-
-try:
-    import tokenizers  # noqa: F401
-
-    _tokenizers_available = True
-
-except ImportError:
-    _tokenizers_available = False
-
 
 default_cache_path = os.path.join(torch_cache_home, "transformers")
 
@@ -207,8 +156,6 @@ TF_WEIGHTS_NAME = "model.ckpt"
 CONFIG_NAME = "config.json"
 MODEL_CARD_NAME = "modelcard.json"
 
-SENTENCEPIECE_UNDERLINE = "▁"
-SPIECE_UNDERLINE = SENTENCEPIECE_UNDERLINE  # Kept for backward compatibility
 
 MULTIPLE_CHOICE_DUMMY_INPUTS = [
     [[0, 1, 0, 1], [1, 0, 0, 1]]
@@ -232,10 +179,6 @@ def is_tf_available():
     return _tf_available
 
 
-def is_flax_available():
-    return _flax_available
-
-
 def is_torch_tpu_available():
     return _torch_tpu_available
 
@@ -260,22 +203,6 @@ def is_faiss_available():
     return _faiss_available
 
 
-def is_sklearn_available():
-    return _has_sklearn
-
-
-def is_sentencepiece_available():
-    return _sentencepiece_available
-
-
-def is_tokenizers_available():
-    return _tokenizers_available
-
-
-def is_in_notebook():
-    return _in_notebook
-
-
 def torch_only_method(fn):
     def wrapper(*args, **kwargs):
         if not _torch_available:
@@ -289,6 +216,10 @@ def torch_only_method(fn):
     return wrapper
 
 
+def is_sklearn_available():
+    return _has_sklearn
+
+
 DATASETS_IMPORT_ERROR = """
 {0} requires the 🤗 Datasets library but it was not found in your enviromnent. You can install it with:
 ```
@@ -306,25 +237,6 @@ that python file if that's the case.
 """
 
 
-TOKENIZERS_IMPORT_ERROR = """
-{0} requires the 🤗 Tokenizers library but it was not found in your enviromnent. You can install it with:
-```
-pip install tokenizers
-```
-In a notebook or a colab, you can install it by executing a cell with
-```
-!pip install tokenizers
-```
-"""
-
-
-SENTENCEPIECE_IMPORT_ERROR = """
-{0} requires the SentencePiece library but it was not found in your enviromnent. Checkout the instructions on the
-installation page of its repo: https://github.com/google/sentencepiece#installation and follow the ones
-that match your enviromnent.
-"""
-
-
 FAISS_IMPORT_ERROR = """
 {0} requires the faiss library but it was not found in your enviromnent. Checkout the instructions on the
 installation page of its repo: https://github.com/facebookresearch/faiss/blob/master/INSTALL.md and follow the ones
@@ -356,12 +268,6 @@ installation page: https://www.tensorflow.org/install and follow the ones that m
 """
 
 
-FLAX_IMPORT_ERROR = """
-{0} requires the FLAX library but it was not found in your enviromnent. Checkout the instructions on the
-installation page: https://github.com/google/flax and follow the ones that match your enviromnent.
-"""
-
-
 def requires_datasets(obj):
     name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__
     if not is_datasets_available():
@@ -392,24 +298,6 @@ def requires_tf(obj):
         raise ImportError(TENSORFLOW_IMPORT_ERROR.format(name))
 
 
-def requires_flax(obj):
-    name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__
-    if not is_flax_available():
-        raise ImportError(FLAX_IMPORT_ERROR.format(name))
-
-
-def requires_tokenizers(obj):
-    name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__
-    if not is_tokenizers_available():
-        raise ImportError(TOKENIZERS_IMPORT_ERROR.format(name))
-
-
-def requires_sentencepiece(obj):
-    name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__
-    if not is_sentencepiece_available():
-        raise ImportError(SENTENCEPIECE_IMPORT_ERROR.format(name))
-
-
 def add_start_docstrings(*docstr):
     def docstring_decorator(fn):
         fn.__doc__ = "".join(docstr) + (fn.__doc__ if fn.__doc__ is not None else "")

src/transformers/generation_utils.py

@@ -111,7 +111,6 @@ class GenerationMixin:
     def generate(
         self,
         input_ids: Optional[torch.LongTensor] = None,
-        decoder_input_ids: Optional[torch.LongTensor] = None,
         max_length: Optional[int] = None,
         min_length: Optional[int] = None,
         do_sample: Optional[bool] = None,
@@ -152,9 +151,6 @@ class GenerationMixin:
             input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
                 The sequence used as a prompt for the generation. If :obj:`None` the method initializes
                 it as an empty :obj:`torch.LongTensor` of shape :obj:`(1,)`.
-            decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
-                initial input_ids for the decoder of encoder-decoder type models. If :obj:`None` then only
-                decoder_start_token_id is passed as the first token to the decoder.
             max_length (:obj:`int`, `optional`, defaults to 20):
                 The maximum length of the sequence to be generated.
             min_length (:obj:`int`, `optional`, defaults to 10):
@@ -421,19 +417,14 @@ class GenerationMixin:
             )  # shape: (batch_size * num_return_sequences * num_beams, cur_len)
 
         if self.config.is_encoder_decoder:
-            device = next(self.parameters()).device
-            if decoder_input_ids is not None:
-                # give initial decoder input ids
-                input_ids = decoder_input_ids.repeat(effective_batch_size * num_beams, 1).to(device)
-            else:
-                # create empty decoder input_ids
-                input_ids = torch.full(
-                    (effective_batch_size * num_beams, 1),
-                    decoder_start_token_id,
-                    dtype=torch.long,
-                    device=device,
-                )
-            cur_len = input_ids.shape[-1]
+            # create empty decoder input_ids
+            input_ids = torch.full(
+                (effective_batch_size * num_beams, 1),
+                decoder_start_token_id,
+                dtype=torch.long,
+                device=next(self.parameters()).device,
+            )
+            cur_len = 1
 
             assert (
                 batch_size == encoder_outputs.last_hidden_state.shape[0]

src/transformers/hf_argparser.py

@@ -65,8 +65,7 @@ class HfArgumentParser(ArgumentParser):
                 if field.default is not dataclasses.MISSING:
                     kwargs["default"] = field.default
             elif field.type is bool or field.type is Optional[bool]:
-                if field.type is bool or (field.default is not None and field.default is not dataclasses.MISSING):
-                    kwargs["action"] = "store_false" if field.default is True else "store_true"
+                kwargs["action"] = "store_false" if field.default is True else "store_true"
                 if field.default is True:
                     field_name = f"--no-{field.name}"
                     kwargs["dest"] = field.name

src/transformers/integrations.py

@@ -9,10 +9,8 @@ try:
     # Comet needs to be imported before any ML frameworks
     import comet_ml  # noqa: F401
 
-    # XXX: there should be comet_ml.ensure_configured(), like `wandb`, for now emulate it
-    comet_ml.Experiment(project_name="ensure_configured")
     _has_comet = True
-except (ImportError, ValueError):
+except (ImportError):
     _has_comet = False
 
 try:
@@ -271,7 +269,7 @@ class WandbCallback(TrainerCallback):
                 'Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"'
             )
             combined_dict = {**args.to_sanitized_dict()}
-            if getattr(model, "config", None) is not None:
+            if hasattr(model, "config"):
                 combined_dict = {**model.config.to_dict(), **combined_dict}
             wandb.init(project=os.getenv("WANDB_PROJECT", "huggingface"), config=combined_dict, name=args.run_name)
             # keep track of model topology and gradients, unsupported on TPU

src/transformers/modeling_auto.py

@@ -43,7 +43,6 @@ from .configuration_auto import (
     MobileBertConfig,
     OpenAIGPTConfig,
     PegasusConfig,
-    ProphetNetConfig,
     ReformerConfig,
     RetriBertConfig,
     RobertaConfig,
@@ -51,7 +50,6 @@ from .configuration_auto import (
     T5Config,
     TransfoXLConfig,
     XLMConfig,
-    XLMProphetNetConfig,
     XLMRobertaConfig,
     XLNetConfig,
     replace_list_option_in_docstrings,
@@ -157,7 +155,6 @@ from .modeling_mobilebert import (
 )
 from .modeling_openai import OpenAIGPTForSequenceClassification, OpenAIGPTLMHeadModel, OpenAIGPTModel
 from .modeling_pegasus import PegasusForConditionalGeneration
-from .modeling_prophetnet import ProphetNetForCausalLM, ProphetNetForConditionalGeneration, ProphetNetModel
 from .modeling_rag import (  # noqa: F401 - need to import all RagModels to be in globals() function
     RagModel,
     RagSequenceForGeneration,
@@ -197,11 +194,6 @@ from .modeling_xlm import (
     XLMModel,
     XLMWithLMHeadModel,
 )
-from .modeling_xlm_prophetnet import (
-    XLMProphetNetForCausalLM,
-    XLMProphetNetForConditionalGeneration,
-    XLMProphetNetModel,
-)
 from .modeling_xlm_roberta import (
     XLMRobertaForCausalLM,
     XLMRobertaForMaskedLM,
@@ -255,8 +247,6 @@ MODEL_MAPPING = OrderedDict(
         (BertGenerationConfig, BertGenerationEncoder),
         (DebertaConfig, DebertaModel),
         (DPRConfig, DPRQuestionEncoder),
-        (XLMProphetNetConfig, XLMProphetNetModel),
-        (ProphetNetConfig, ProphetNetModel),
     ]
 )
 
@@ -335,8 +325,6 @@ MODEL_FOR_CAUSAL_LM_MAPPING = OrderedDict(
         (CTRLConfig, CTRLLMHeadModel),
         (ReformerConfig, ReformerModelWithLMHead),
         (BertGenerationConfig, BertGenerationDecoder),
-        (ProphetNetConfig, XLMProphetNetForCausalLM),
-        (ProphetNetConfig, ProphetNetForCausalLM),
     ]
 )
 
@@ -371,8 +359,6 @@ MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
         (BartConfig, BartForConditionalGeneration),
         (FSMTConfig, FSMTForConditionalGeneration),
         (EncoderDecoderConfig, EncoderDecoderModel),
-        (XLMProphetNetConfig, XLMProphetNetForConditionalGeneration),
-        (ProphetNetConfig, ProphetNetForConditionalGeneration),
     ]
 )
 

src/transformers/modeling_deberta.py

@@ -491,7 +491,7 @@ class DisentangledSelfAttention(torch.nn.Module):
         self.in_proj = torch.nn.Linear(config.hidden_size, self.all_head_size * 3, bias=False)
         self.q_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
         self.v_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
-        self.pos_att_type = config.pos_att_type if config.pos_att_type is not None else []
+        self.pos_att_type = config.pos_att_type
 
         self.relative_attention = getattr(config, "relative_attention", False)
         self.talking_head = getattr(config, "talking_head", False)

src/transformers/modeling_dpr.py

@@ -35,15 +35,12 @@ _CONFIG_FOR_DOC = "DPRConfig"
 
 DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST = [
     "facebook/dpr-ctx_encoder-single-nq-base",
-    "facebook/dpr-ctx_encoder-multiset-base",
 ]
 DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST = [
     "facebook/dpr-question_encoder-single-nq-base",
-    "facebook/dpr-question_encoder-multiset-base",
 ]
 DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST = [
     "facebook/dpr-reader-single-nq-base",
-    "facebook/dpr-reader-multiset-base",
 ]
 
 

src/transformers/modeling_encoder_decoder.py

@@ -327,7 +327,7 @@ class EncoderDecoderModel(PreTrainedModel):
 
                 kwargs_decoder["config"] = decoder_config
 
-            if kwargs_decoder["config"].is_decoder is False or kwargs_decoder["config"].add_cross_attention is False:
+            if kwargs_decoder["config"].is_decoder is False or decoder_config.add_cross_attention is False:
                 logger.warning(
                     f"Decoder model {decoder_pretrained_model_name_or_path} is not initialized as a decoder. In order to initialize {decoder_pretrained_model_name_or_path} as a decoder, make sure that the attributes `is_decoder` and `add_cross_attention` of `decoder_config` passed to `.from_encoder_decoder_pretrained(...)` are set to `True` or do not pass a `decoder_config` to `.from_encoder_decoder_pretrained(...)`"
                 )
@@ -434,6 +434,8 @@ class EncoderDecoderModel(PreTrainedModel):
             encoder_attentions=encoder_outputs.attentions,
         )
 
+        return decoder_outputs + encoder_outputs
+
     def prepare_inputs_for_generation(self, input_ids, past, attention_mask, encoder_outputs, **kwargs):
         decoder_inputs = self.decoder.prepare_inputs_for_generation(input_ids)
         decoder_attention_mask = decoder_inputs["attention_mask"] if "attention_mask" in decoder_inputs else None

src/transformers/modeling_flax_auto.py

@@ -1,167 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Google Flax Team Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" Auto Model class. """
-
-
-from collections import OrderedDict
-
-from .configuration_auto import AutoConfig, BertConfig, RobertaConfig
-from .configuration_utils import PretrainedConfig
-from .modeling_flax_bert import FlaxBertModel
-from .modeling_flax_roberta import FlaxRobertaModel
-from .utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-
-ALL_PRETRAINED_MODEL_ARCHIVE_MAP = dict(
-    (key, value)
-    for pretrained_map in [
-        FlaxBertModel.pretrained_model_archive_map,
-        FlaxRobertaModel.pretrained_model_archive_map,
-    ]
-    for key, value, in pretrained_map.items()
-)
-
-MODEL_MAPPING = OrderedDict(
-    [
-        (RobertaConfig, FlaxRobertaModel),
-        (BertConfig, FlaxBertModel),
-    ]
-)
-
-
-class FlaxAutoModel(object):
-    r"""
-    :class:`~transformers.FlaxAutoModel` is a generic model class
-    that will be instantiated as one of the base model classes of the library
-    when created with the `FlaxAutoModel.from_pretrained(pretrained_model_name_or_path)`
-    or the `FlaxAutoModel.from_config(config)` class methods.
-
-    This class cannot be instantiated using `__init__()` (throws an error).
-    """
-
-    def __init__(self):
-        raise EnvironmentError(
-            "FlaxAutoModel is designed to be instantiated "
-            "using the `FlaxAutoModel.from_pretrained(pretrained_model_name_or_path)` or "
-            "`FlaxAutoModel.from_config(config)` methods."
-        )
-
-    @classmethod
-    def from_config(cls, config):
-        r"""Instantiates one of the base model classes of the library
-        from a configuration.
-
-        Args:
-            config (:class:`~transformers.PretrainedConfig`):
-                The model class to instantiate is selected based on the configuration class:
-
-                - isInstance of `roberta` configuration class: :class:`~transformers.FlaxRobertaModel` (RoBERTa model)
-                - isInstance of `bert` configuration class: :class:`~transformers.FlaxBertModel` (Bert model)
-        Examples:
-
-            config = BertConfig.from_pretrained('bert-base-uncased')    # Download configuration from S3 and cache.
-            model = FlaxAutoModel.from_config(config)  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
-        """
-        for config_class, model_class in MODEL_MAPPING.items():
-            if isinstance(config, config_class):
-                return model_class(config)
-        raise ValueError(
-            f"Unrecognized configuration class {config.__class__} "
-            f"for this kind of FlaxAutoModel: {cls.__name__}.\n"
-            f"Model type should be one of {', '.join(c.__name__ for c in MODEL_MAPPING.keys())}."
-        )
-
-    @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
-        r"""Instantiates one of the base model classes of the library
-        from a pre-trained model configuration.
-
-        The `from_pretrained()` method takes care of returning the correct model class instance
-        based on the `model_type` property of the config object, or when it's missing,
-        falling back to using pattern matching on the `pretrained_model_name_or_path` string.
-
-        The base model class to instantiate is selected as the first pattern matching
-        in the `pretrained_model_name_or_path` string (in the following order):
-            - contains `roberta`: :class:`~transformers.FlaxRobertaModel` (RoBERTa model)
-            - contains `bert`: :class:`~transformers.FlaxBertModel` (Bert model)
-
-            The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated)
-            To train the model, you should first set it back in training mode with `model.train()`
-
-        Args:
-            pretrained_model_name_or_path: either:
-
-                - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``.
-                - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``.
-                - a path to a `directory` containing model weights saved using :func:`~transformers.FlaxPreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``.
-                - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
-
-            model_args: (`optional`) Sequence of positional arguments:
-                All remaining positional arguments will be passed to the underlying model's ``__init__`` method
-
-            config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`:
-                Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when:
-
-                - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or
-                - the model was saved using :func:`~transformers.FlaxPreTrainedModel.save_pretrained` and is reloaded by suppling the save directory.
-                - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory.
-
-            state_dict: (`optional`) dict:
-                an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file.
-                This option can be used if you want to create a model from a pretrained configuration but load your own weights.
-                In this case though, you should check if using :func:`~transformers.FlaxPreTrainedModel.save_pretrained` and :func:`~transformers.FlaxPreTrainedModel.from_pretrained` is not a simpler option.
-
-            cache_dir: (`optional`) string:
-                Path to a directory in which a downloaded pre-trained model
-                configuration should be cached if the standard cache should not be used.
-
-            force_download: (`optional`) boolean, default False:
-                Force to (re-)download the model weights and configuration files and override the cached versions if they exists.
-
-            resume_download: (`optional`) boolean, default False:
-                Do not delete incompletely recieved file. Attempt to resume the download if such a file exists.
-
-            proxies: (`optional`) dict, default None:
-                A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.
-                The proxies are used on each request.
-
-            output_loading_info: (`optional`) boolean:
-                Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
-
-            kwargs: (`optional`) Remaining dictionary of keyword arguments:
-                These arguments will be passed to the configuration and the model.
-
-        Examples::
-
-            model = FlaxAutoModel.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
-            model = FlaxAutoModel.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
-            assert model.config.output_attention == True
-
-        """
-        config = kwargs.pop("config", None)
-        if not isinstance(config, PretrainedConfig):
-            config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
-
-        for config_class, model_class in MODEL_MAPPING.items():
-            if isinstance(config, config_class):
-                return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
-        raise ValueError(
-            f"Unrecognized configuration class {config.__class__} "
-            f"for this kind of FlaxAutoModel: {cls.__name__}.\n"
-            f"Model type should be one of {', '.join(c.__name__ for c in MODEL_MAPPING.keys())}"
-        )

src/transformers/modeling_flax_bert.py

@@ -1,438 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Google Flax Team Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Callable, Dict
-
-import numpy as np
-
-import flax.linen as nn
-import jax
-import jax.numpy as jnp
-from flax.linen import compact
-
-from .configuration_bert import BertConfig
-from .file_utils import add_start_docstrings
-from .modeling_flax_utils import FlaxPreTrainedModel, gelu
-from .utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-_CONFIG_FOR_DOC = "BertConfig"
-_TOKENIZER_FOR_DOC = "BertTokenizer"
-
-
-BERT_START_DOCSTRING = r"""
-
-    This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic
-    methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
-    pruning heads etc.)
-
-    This model is also a PyTorch `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__ subclass.
-    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
-    usage and behavior.
-
-    Parameters:
-        config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model.
-            Initializing with a config file does not load the weights associated with the model, only the configuration.
-            Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
-"""
-
-BERT_INPUTS_DOCSTRING = r"""
-    Args:
-        input_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`):
-            Indices of input sequence tokens in the vocabulary.
-
-            Indices can be obtained using :class:`~transformers.BertTokenizer`.
-            See :meth:`transformers.PreTrainedTokenizer.encode` and
-            :meth:`transformers.PreTrainedTokenizer.__call__` for details.
-
-            `What are input IDs? <../glossary.html#input-ids>`__
-        attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`):
-            Mask to avoid performing attention on padding token indices.
-            Mask values selected in ``[0, 1]``:
-
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
-
-            `What are attention masks? <../glossary.html#attention-mask>`__
-        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
-            Segment token indices to indicate first and second portions of the inputs.
-            Indices are selected in ``[0, 1]``:
-
-            - 0 corresponds to a `sentence A` token,
-            - 1 corresponds to a `sentence B` token.
-
-            `What are token type IDs? <../glossary.html#token-type-ids>`_
-        position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
-            Indices of positions of each input sequence tokens in the position embeddings.
-            Selected in the range ``[0, config.max_position_embeddings - 1]``.
-
-            `What are position IDs? <../glossary.html#position-ids>`_
-        head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`):
-            Mask to nullify selected heads of the self-attention modules.
-            Mask values selected in ``[0, 1]``:
-
-            - 1 indicates the head is **not masked**,
-            - 0 indicates the head is **masked**.
-
-        inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`):
-            Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation.
-            This is useful if you want more control over how to convert :obj:`input_ids` indices into associated
-            vectors than the model's internal embedding lookup matrix.
-        output_attentions (:obj:`bool`, `optional`):
-            Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
-            tensors for more detail.
-        output_hidden_states (:obj:`bool`, `optional`):
-            Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for
-            more detail.
-        return_dict (:obj:`bool`, `optional`):
-            Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple.
-"""
-
-
-class FlaxBertLayerNorm(nn.Module):
-    """Layer normalization (https://arxiv.org/abs/1607.06450).
-    Operates on the last axis of the input data.
-    """
-
-    epsilon: float = 1e-6
-    dtype: jnp.dtype = jnp.float32
-    bias: bool = True
-    scale: bool = True
-    bias_init: jnp.ndarray = nn.initializers.zeros
-    scale_init: jnp.ndarray = nn.initializers.ones
-
-    @compact
-    def __call__(self, x):
-        """Applies layer normalization on the input.
-        It normalizes the activations of the layer for each given example in a
-        batch independently, rather than across a batch like Batch Normalization.
-        i.e. applies a transformation that maintains the mean activation within
-        each example close to 0 and the activation standard deviation close to 1.
-        Args:
-          x: the inputs
-          epsilon: A small float added to variance to avoid dividing by zero.
-          dtype: the dtype of the computation (default: float32).
-          bias:  If True, bias (beta) is added.
-          scale: If True, multiply by scale (gamma). When the next layer is linear
-            (also e.g. nn.relu), this can be disabled since the scaling will be done
-            by the next layer.
-          bias_init: Initializer for bias, by default, zero.
-          scale_init: Initializer for scale, by default, one.
-        Returns:
-          Normalized inputs (the same shape as inputs).
-        """
-        features = x.shape[-1]
-        mean = jnp.mean(x, axis=-1, keepdims=True)
-        mean2 = jnp.mean(jax.lax.square(x), axis=-1, keepdims=True)
-        var = mean2 - jax.lax.square(mean)
-        mul = jax.lax.rsqrt(var + self.epsilon)
-        if self.scale:
-            mul = mul * jnp.asarray(self.param("gamma", self.scale_init, (features,)), self.dtype)
-        y = (x - mean) * mul
-        if self.bias:
-            y = y + jnp.asarray(self.param("beta", self.bias_init, (features,)), self.dtype)
-        return y
-
-
-class FlaxBertEmbedding(nn.Module):
-    """
-    Specify a new class for doing the embedding stuff
-    as Flax's one use 'embedding' for the parameter name
-    and PyTorch use 'weight'
-    """
-
-    vocab_size: int
-    hidden_size: int
-    emb_init: Callable[..., np.ndarray] = nn.initializers.normal(stddev=0.1)
-
-    @compact
-    def __call__(self, inputs):
-        embedding = self.param("weight", self.emb_init, (self.vocab_size, self.hidden_size))
-        return jnp.take(embedding, inputs, axis=0)
-
-
-class FlaxBertEmbeddings(nn.Module):
-    """Construct the embeddings from word, position and token_type embeddings."""
-
-    vocab_size: int
-    hidden_size: int
-    type_vocab_size: int
-    max_length: int
-
-    @compact
-    def __call__(self, input_ids, token_type_ids, position_ids, attention_mask):
-
-        # Embed
-        w_emb = FlaxBertEmbedding(self.vocab_size, self.hidden_size, name="word_embeddings")(
-            jnp.atleast_2d(input_ids.astype("i4"))
-        )
-        p_emb = FlaxBertEmbedding(self.max_length, self.hidden_size, name="position_embeddings")(
-            jnp.atleast_2d(position_ids.astype("i4"))
-        )
-        t_emb = FlaxBertEmbedding(self.type_vocab_size, self.hidden_size, name="token_type_embeddings")(
-            jnp.atleast_2d(token_type_ids.astype("i4"))
-        )
-
-        # Sum all embeddings
-        summed_emb = w_emb + jnp.broadcast_to(p_emb, w_emb.shape) + t_emb
-
-        # Layer Norm
-        layer_norm = FlaxBertLayerNorm(name="layer_norm")(summed_emb)
-
-        return layer_norm
-
-
-class FlaxBertAttention(nn.Module):
-    num_heads: int
-    head_size: int
-
-    @compact
-    def __call__(self, hidden_state, attention_mask):
-        self_att = nn.attention.SelfAttention(num_heads=self.num_heads, qkv_features=self.head_size, name="self")(
-            hidden_state, attention_mask
-        )
-
-        layer_norm = FlaxBertLayerNorm(name="layer_norm")(self_att + hidden_state)
-        return layer_norm
-
-
-class FlaxBertIntermediate(nn.Module):
-    output_size: int
-
-    @compact
-    def __call__(self, hidden_state):
-        # TODO: Add ACT2FN reference to change activation function
-        dense = nn.Dense(features=self.output_size, name="dense")(hidden_state)
-        return gelu(dense)
-
-
-class FlaxBertOutput(nn.Module):
-    @compact
-    def __call__(self, intermediate_output, attention_output):
-        hidden_state = nn.Dense(attention_output.shape[-1], name="dense")(intermediate_output)
-        hidden_state = FlaxBertLayerNorm(name="layer_norm")(hidden_state + attention_output)
-        return hidden_state
-
-
-class FlaxBertLayer(nn.Module):
-    num_heads: int
-    head_size: int
-    intermediate_size: int
-
-    @compact
-    def __call__(self, hidden_state, attention_mask):
-        attention = FlaxBertAttention(self.num_heads, self.head_size, name="attention")(hidden_state, attention_mask)
-        intermediate = FlaxBertIntermediate(self.intermediate_size, name="intermediate")(attention)
-        output = FlaxBertOutput(name="output")(intermediate, attention)
-
-        return output
-
-
-class FlaxBertLayerCollection(nn.Module):
-    """
-    Stores N BertLayer(s)
-    """
-
-    num_layers: int
-    num_heads: int
-    head_size: int
-    intermediate_size: int
-
-    @compact
-    def __call__(self, inputs, attention_mask):
-        assert self.num_layers > 0, f"num_layers should be >= 1, got ({self.num_layers})"
-
-        # Initialize input / output
-        input_i = inputs
-
-        # Forward over all encoders
-        for i in range(self.num_layers):
-            layer = FlaxBertLayer(self.num_heads, self.head_size, self.intermediate_size, name=f"{i}")
-            input_i = layer(input_i, attention_mask)
-        return input_i
-
-
-class FlaxBertEncoder(nn.Module):
-    num_layers: int
-    num_heads: int
-    head_size: int
-    intermediate_size: int
-
-    @compact
-    def __call__(self, hidden_state, attention_mask):
-        layer = FlaxBertLayerCollection(
-            self.num_layers, self.num_heads, self.head_size, self.intermediate_size, name="layer"
-        )(hidden_state, attention_mask)
-        return layer
-
-
-class FlaxBertPooler(nn.Module):
-    @compact
-    def __call__(self, hidden_state):
-        cls_token = hidden_state[:, 0]
-        out = nn.Dense(hidden_state.shape[-1], name="dense")(cls_token)
-        return jax.lax.tanh(out)
-
-
-class FlaxBertModule(nn.Module):
-    vocab_size: int
-    hidden_size: int
-    type_vocab_size: int
-    max_length: int
-    num_encoder_layers: int
-    num_heads: int
-    head_size: int
-    intermediate_size: int
-
-    @compact
-    def __call__(self, input_ids, token_type_ids, position_ids, attention_mask):
-
-        # Embedding
-        embeddings = FlaxBertEmbeddings(
-            self.vocab_size, self.hidden_size, self.type_vocab_size, self.max_length, name="embeddings"
-        )(input_ids, token_type_ids, position_ids, attention_mask)
-
-        # N stacked encoding layers
-        encoder = FlaxBertEncoder(
-            self.num_encoder_layers, self.num_heads, self.head_size, self.intermediate_size, name="encoder"
-        )(embeddings, attention_mask)
-
-        pooled = FlaxBertPooler(name="pooler")(encoder)
-        return encoder, pooled
-
-
-@add_start_docstrings(
-    "The bare Bert Model transformer outputting raw hidden-states without any specific head on top.",
-    BERT_START_DOCSTRING,
-)
-class FlaxBertModel(FlaxPreTrainedModel):
-    """
-    The model can behave as an encoder (with only self-attention) as well
-    as a decoder, in which case a layer of cross-attention is added between
-    the self-attention layers, following the architecture described in `Attention is all you need
-    <https://arxiv.org/abs/1706.03762>`__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones,
-    Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
-    """
-
-    model_class = FlaxBertModule
-    config_class = BertConfig
-    base_model_prefix = "bert"
-
-    @staticmethod
-    def convert_from_pytorch(pt_state: Dict, config: BertConfig) -> Dict:
-        jax_state = dict(pt_state)
-
-        # Need to change some parameters name to match Flax names so that we don't have to fork any layer
-        for key, tensor in pt_state.items():
-            # Key parts
-            key_parts = set(key.split("."))
-
-            # Every dense layer has "kernel" parameters instead of "weight"
-            if "dense.weight" in key:
-                del jax_state[key]
-                key = key.replace("weight", "kernel")
-                jax_state[key] = tensor
-
-            # SelfAttention needs also to replace "weight" by "kernel"
-            if {"query", "key", "value"} & key_parts:
-
-                # Flax SelfAttention decomposes the heads (num_head, size // num_heads)
-                if "bias" in key:
-                    jax_state[key] = tensor.reshape((config.num_attention_heads, -1))
-                elif "weight":
-                    del jax_state[key]
-                    key = key.replace("weight", "kernel")
-                    tensor = tensor.reshape((config.num_attention_heads, -1, config.hidden_size)).transpose((2, 0, 1))
-                    jax_state[key] = tensor
-
-            # SelfAttention output is not a separate layer, remove one nesting
-            if "attention.output.dense" in key:
-                del jax_state[key]
-                key = key.replace("attention.output.dense", "attention.self.out")
-                jax_state[key] = tensor
-
-            # SelfAttention output is not a separate layer, remove nesting on layer norm
-            if "attention.output.LayerNorm" in key:
-                del jax_state[key]
-                key = key.replace("attention.output.LayerNorm", "attention.LayerNorm")
-                jax_state[key] = tensor
-
-            # There are some transposed parameters w.r.t their PyTorch counterpart
-            if "intermediate.dense.kernel" in key or "output.dense.kernel" in key:
-                jax_state[key] = tensor.T
-
-            # Self Attention output projection needs to be transposed
-            if "out.kernel" in key:
-                jax_state[key] = tensor.reshape((config.hidden_size, config.num_attention_heads, -1)).transpose(
-                    1, 2, 0
-                )
-
-            # Pooler needs to transpose its kernel
-            if "pooler.dense.kernel" in key:
-                jax_state[key] = tensor.T
-
-            # Handle LayerNorm conversion
-            if "LayerNorm" in key:
-                del jax_state[key]
-
-                # Replace LayerNorm by layer_norm
-                new_key = key.replace("LayerNorm", "layer_norm")
-
-                if "weight" in key:
-                    new_key = new_key.replace("weight", "gamma")
-                elif "bias" in key:
-                    new_key = new_key.replace("bias", "beta")
-
-                jax_state[new_key] = tensor
-
-        return jax_state
-
-    def __init__(self, config: BertConfig, state: dict, seed: int = 0, **kwargs):
-        model = FlaxBertModule(
-            vocab_size=config.vocab_size,
-            hidden_size=config.hidden_size,
-            type_vocab_size=config.type_vocab_size,
-            max_length=config.max_position_embeddings,
-            num_encoder_layers=config.num_hidden_layers,
-            num_heads=config.num_attention_heads,
-            head_size=config.hidden_size,
-            intermediate_size=config.intermediate_size,
-        )
-
-        super().__init__(config, model, state, seed)
-
-    @property
-    def module(self) -> nn.Module:
-        return self._module
-
-    def __call__(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
-        if token_type_ids is None:
-            token_type_ids = jnp.ones_like(input_ids)
-
-        if position_ids is None:
-            position_ids = jnp.arange(jnp.atleast_2d(input_ids).shape[-1])
-
-        if attention_mask is None:
-            attention_mask = jnp.ones_like(input_ids)
-
-        return self.model.apply(
-            {"params": self.params},
-            jnp.array(input_ids, dtype="i4"),
-            jnp.array(token_type_ids, dtype="i4"),
-            jnp.array(position_ids, dtype="i4"),
-            jnp.array(attention_mask, dtype="i4"),
-        )

src/transformers/modeling_flax_roberta.py

@@ -1,450 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Google Flax Team Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from typing import Callable, Dict
-
-import numpy as np
-
-import flax.linen as nn
-import jax
-import jax.numpy as jnp
-from flax.linen import compact
-
-from .configuration_roberta import RobertaConfig
-from .file_utils import add_start_docstrings
-from .modeling_flax_utils import FlaxPreTrainedModel, gelu
-from .utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-_CONFIG_FOR_DOC = "RobertaConfig"
-_TOKENIZER_FOR_DOC = "RobertaTokenizer"
-
-
-ROBERTA_START_DOCSTRING = r"""
-
-    This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic
-    methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
-    pruning heads etc.)
-
-    This model is also a PyTorch `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__ subclass.
-    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
-    usage and behavior.
-
-    Parameters:
-        config (:class:`~transformers.RobertaConfig`): Model configuration class with all the parameters of the
-            model. Initializing with a config file does not load the weights associated with the model, only the configuration.
-            Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
-"""
-
-ROBERTA_INPUTS_DOCSTRING = r"""
-    Args:
-        input_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`):
-            Indices of input sequence tokens in the vocabulary.
-
-            Indices can be obtained using :class:`~transformers.RobertaTokenizer`.
-            See :meth:`transformers.PreTrainedTokenizer.encode` and
-            :meth:`transformers.PreTrainedTokenizer.__call__` for details.
-
-            `What are input IDs? <../glossary.html#input-ids>`__
-        attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`):
-            Mask to avoid performing attention on padding token indices.
-            Mask values selected in ``[0, 1]``:
-
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **maked**.
-
-            `What are attention masks? <../glossary.html#attention-mask>`__
-        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
-            Segment token indices to indicate first and second portions of the inputs.
-            Indices are selected in ``[0, 1]``:
-
-            - 0 corresponds to a `sentence A` token,
-            - 1 corresponds to a `sentence B` token.
-
-            `What are token type IDs? <../glossary.html#token-type-ids>`_
-        position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
-            Indices of positions of each input sequence tokens in the position embeddings.
-            Selected in the range ``[0, config.max_position_embeddings - 1]``.
-
-            `What are position IDs? <../glossary.html#position-ids>`_
-        head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`):
-            Mask to nullify selected heads of the self-attention modules.
-            Mask values selected in ``[0, 1]``:
-
-            - 1 indicates the head is **not masked**,
-            - 0 indicates the head is **masked**.
-
-        inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`):
-            Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation.
-            This is useful if you want more control over how to convert :obj:`input_ids` indices into associated
-            vectors than the model's internal embedding lookup matrix.
-        output_attentions (:obj:`bool`, `optional`):
-            Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
-            tensors for more detail.
-        output_hidden_states (:obj:`bool`, `optional`):
-            Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for
-            more detail.
-        return_dict (:obj:`bool`, `optional`):
-            Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple.
-"""
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertLayerNorm with Bert->Roberta
-class FlaxRobertaLayerNorm(nn.Module):
-    """Layer normalization (https://arxiv.org/abs/1607.06450).
-    Operates on the last axis of the input data.
-    """
-
-    epsilon: float = 1e-6
-    dtype: jnp.dtype = jnp.float32
-    bias: bool = True
-    scale: bool = True
-    bias_init: jnp.ndarray = nn.initializers.zeros
-    scale_init: jnp.ndarray = nn.initializers.ones
-
-    @compact
-    def __call__(self, x):
-        """Applies layer normalization on the input.
-        It normalizes the activations of the layer for each given example in a
-        batch independently, rather than across a batch like Batch Normalization.
-        i.e. applies a transformation that maintains the mean activation within
-        each example close to 0 and the activation standard deviation close to 1.
-        Args:
-          x: the inputs
-          epsilon: A small float added to variance to avoid dividing by zero.
-          dtype: the dtype of the computation (default: float32).
-          bias:  If True, bias (beta) is added.
-          scale: If True, multiply by scale (gamma). When the next layer is linear
-            (also e.g. nn.relu), this can be disabled since the scaling will be done
-            by the next layer.
-          bias_init: Initializer for bias, by default, zero.
-          scale_init: Initializer for scale, by default, one.
-        Returns:
-          Normalized inputs (the same shape as inputs).
-        """
-        features = x.shape[-1]
-        mean = jnp.mean(x, axis=-1, keepdims=True)
-        mean2 = jnp.mean(jax.lax.square(x), axis=-1, keepdims=True)
-        var = mean2 - jax.lax.square(mean)
-        mul = jax.lax.rsqrt(var + self.epsilon)
-        if self.scale:
-            mul = mul * jnp.asarray(self.param("gamma", self.scale_init, (features,)), self.dtype)
-        y = (x - mean) * mul
-        if self.bias:
-            y = y + jnp.asarray(self.param("beta", self.bias_init, (features,)), self.dtype)
-        return y
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertEmbedding with Bert->Roberta
-class FlaxRobertaEmbedding(nn.Module):
-    """
-    Specify a new class for doing the embedding stuff
-    as Flax's one use 'embedding' for the parameter name
-    and PyTorch use 'weight'
-    """
-
-    vocab_size: int
-    hidden_size: int
-    emb_init: Callable[..., np.ndarray] = nn.initializers.normal(stddev=0.1)
-
-    @compact
-    def __call__(self, inputs):
-        embedding = self.param("weight", self.emb_init, (self.vocab_size, self.hidden_size))
-        return jnp.take(embedding, inputs, axis=0)
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertEmbeddings with Bert->Roberta
-class FlaxRobertaEmbeddings(nn.Module):
-    """Construct the embeddings from word, position and token_type embeddings."""
-
-    vocab_size: int
-    hidden_size: int
-    type_vocab_size: int
-    max_length: int
-
-    @compact
-    def __call__(self, input_ids, token_type_ids, position_ids, attention_mask):
-
-        # Embed
-        w_emb = FlaxRobertaEmbedding(self.vocab_size, self.hidden_size, name="word_embeddings")(
-            jnp.atleast_2d(input_ids.astype("i4"))
-        )
-        p_emb = FlaxRobertaEmbedding(self.max_length, self.hidden_size, name="position_embeddings")(
-            jnp.atleast_2d(position_ids.astype("i4"))
-        )
-        t_emb = FlaxRobertaEmbedding(self.type_vocab_size, self.hidden_size, name="token_type_embeddings")(
-            jnp.atleast_2d(token_type_ids.astype("i4"))
-        )
-
-        # Sum all embeddings
-        summed_emb = w_emb + jnp.broadcast_to(p_emb, w_emb.shape) + t_emb
-
-        # Layer Norm
-        layer_norm = FlaxRobertaLayerNorm(name="layer_norm")(summed_emb)
-
-        return layer_norm
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertAttention with Bert->Roberta
-class FlaxRobertaAttention(nn.Module):
-    num_heads: int
-    head_size: int
-
-    @compact
-    def __call__(self, hidden_state, attention_mask):
-        self_att = nn.attention.SelfAttention(num_heads=self.num_heads, qkv_features=self.head_size, name="self")(
-            hidden_state, attention_mask
-        )
-
-        layer_norm = FlaxRobertaLayerNorm(name="layer_norm")(self_att + hidden_state)
-        return layer_norm
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertIntermediate with Bert->Roberta
-class FlaxRobertaIntermediate(nn.Module):
-    output_size: int
-
-    @compact
-    def __call__(self, hidden_state):
-        # TODO: Add ACT2FN reference to change activation function
-        dense = nn.Dense(features=self.output_size, name="dense")(hidden_state)
-        return gelu(dense)
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertOutput with Bert->Roberta
-class FlaxRobertaOutput(nn.Module):
-    @compact
-    def __call__(self, intermediate_output, attention_output):
-        hidden_state = nn.Dense(attention_output.shape[-1], name="dense")(intermediate_output)
-        hidden_state = FlaxRobertaLayerNorm(name="layer_norm")(hidden_state + attention_output)
-        return hidden_state
-
-
-class FlaxRobertaLayer(nn.Module):
-    num_heads: int
-    head_size: int
-    intermediate_size: int
-
-    @compact
-    def __call__(self, hidden_state, attention_mask):
-        attention = FlaxRobertaAttention(self.num_heads, self.head_size, name="attention")(
-            hidden_state, attention_mask
-        )
-        intermediate = FlaxRobertaIntermediate(self.intermediate_size, name="intermediate")(attention)
-        output = FlaxRobertaOutput(name="output")(intermediate, attention)
-
-        return output
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertLayerCollection with Bert->Roberta
-class FlaxRobertaLayerCollection(nn.Module):
-    """
-    Stores N RobertaLayer(s)
-    """
-
-    num_layers: int
-    num_heads: int
-    head_size: int
-    intermediate_size: int
-
-    @compact
-    def __call__(self, inputs, attention_mask):
-        assert self.num_layers > 0, f"num_layers should be >= 1, got ({self.num_layers})"
-
-        # Initialize input / output
-        input_i = inputs
-
-        # Forward over all encoders
-        for i in range(self.num_layers):
-            layer = FlaxRobertaLayer(self.num_heads, self.head_size, self.intermediate_size, name=f"{i}")
-            input_i = layer(input_i, attention_mask)
-        return input_i
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertEncoder with Bert->Roberta
-class FlaxRobertaEncoder(nn.Module):
-    num_layers: int
-    num_heads: int
-    head_size: int
-    intermediate_size: int
-
-    @compact
-    def __call__(self, hidden_state, attention_mask):
-        layer = FlaxRobertaLayerCollection(
-            self.num_layers, self.num_heads, self.head_size, self.intermediate_size, name="layer"
-        )(hidden_state, attention_mask)
-        return layer
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertPooler with Bert->Roberta
-class FlaxRobertaPooler(nn.Module):
-    @compact
-    def __call__(self, hidden_state):
-        cls_token = hidden_state[:, 0]
-        out = nn.Dense(hidden_state.shape[-1], name="dense")(cls_token)
-        return jax.lax.tanh(out)
-
-
-# Copied from transformers.modeling_flax_bert.FlaxBertModule with Bert->Roberta
-class FlaxRobertaModule(nn.Module):
-    vocab_size: int
-    hidden_size: int
-    type_vocab_size: int
-    max_length: int
-    num_encoder_layers: int
-    num_heads: int
-    head_size: int
-    intermediate_size: int
-
-    @compact
-    def __call__(self, input_ids, token_type_ids, position_ids, attention_mask):
-
-        # Embedding
-        embeddings = FlaxRobertaEmbeddings(
-            self.vocab_size, self.hidden_size, self.type_vocab_size, self.max_length, name="embeddings"
-        )(input_ids, token_type_ids, position_ids, attention_mask)
-
-        # N stacked encoding layers
-        encoder = FlaxRobertaEncoder(
-            self.num_encoder_layers, self.num_heads, self.head_size, self.intermediate_size, name="encoder"
-        )(embeddings, attention_mask)
-
-        pooled = FlaxRobertaPooler(name="pooler")(encoder)
-        return encoder, pooled
-
-
-@add_start_docstrings(
-    "The bare RoBERTa Model transformer outputting raw hidden-states without any specific head on top.",
-    ROBERTA_START_DOCSTRING,
-)
-class FlaxRobertaModel(FlaxPreTrainedModel):
-    """
-    The model can behave as an encoder (with only self-attention) as well
-    as a decoder, in which case a layer of cross-attention is added between
-    the self-attention layers, following the architecture described in `Attention is all you need`_ by Ashish Vaswani,
-    Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
-    """
-
-    model_class = FlaxRobertaModule
-    config_class = RobertaConfig
-    base_model_prefix = "roberta"
-
-    @staticmethod
-    def convert_from_pytorch(pt_state: Dict, config: RobertaConfig) -> Dict:
-        jax_state = dict(pt_state)
-
-        # Need to change some parameters name to match Flax names so that we don't have to fork any layer
-        for key, tensor in pt_state.items():
-            # Key parts
-            key_parts = set(key.split("."))
-
-            # Every dense layer has "kernel" parameters instead of "weight"
-            if "dense.weight" in key:
-                del jax_state[key]
-                key = key.replace("weight", "kernel")
-                jax_state[key] = tensor
-
-            # SelfAttention needs also to replace "weight" by "kernel"
-            if {"query", "key", "value"} & key_parts:
-
-                # Flax SelfAttention decomposes the heads (num_head, size // num_heads)
-                if "bias" in key:
-                    jax_state[key] = tensor.reshape((config.num_attention_heads, -1))
-                elif "weight":
-                    del jax_state[key]
-                    key = key.replace("weight", "kernel")
-                    tensor = tensor.reshape((config.num_attention_heads, -1, config.hidden_size)).transpose((2, 0, 1))
-                    jax_state[key] = tensor
-
-            # SelfAttention output is not a separate layer, remove one nesting
-            if "attention.output.dense" in key:
-                del jax_state[key]
-                key = key.replace("attention.output.dense", "attention.self.out")
-                jax_state[key] = tensor
-
-            # SelfAttention output is not a separate layer, remove nesting on layer norm
-            if "attention.output.LayerNorm" in key:
-                del jax_state[key]
-                key = key.replace("attention.output.LayerNorm", "attention.LayerNorm")
-                jax_state[key] = tensor
-
-            # There are some transposed parameters w.r.t their PyTorch counterpart
-            if "intermediate.dense.kernel" in key or "output.dense.kernel" in key:
-                jax_state[key] = tensor.T
-
-            # Self Attention output projection needs to be transposed
-            if "out.kernel" in key:
-                jax_state[key] = tensor.reshape((config.hidden_size, config.num_attention_heads, -1)).transpose(
-                    1, 2, 0
-                )
-
-            # Pooler needs to transpose its kernel
-            if "pooler.dense.kernel" in key:
-                jax_state[key] = tensor.T
-
-            # Handle LayerNorm conversion
-            if "LayerNorm" in key:
-                del jax_state[key]
-
-                # Replace LayerNorm by layer_norm
-                new_key = key.replace("LayerNorm", "layer_norm")
-
-                if "weight" in key:
-                    new_key = new_key.replace("weight", "gamma")
-                elif "bias" in key:
-                    new_key = new_key.replace("bias", "beta")
-
-                jax_state[new_key] = tensor
-
-        return jax_state
-
-    def __init__(self, config: RobertaConfig, state: dict, seed: int = 0, **kwargs):
-        model = FlaxRobertaModule(
-            vocab_size=config.vocab_size,
-            hidden_size=config.hidden_size,
-            type_vocab_size=config.type_vocab_size,
-            max_length=config.max_position_embeddings,
-            num_encoder_layers=config.num_hidden_layers,
-            num_heads=config.num_attention_heads,
-            head_size=config.hidden_size,
-            intermediate_size=config.intermediate_size,
-        )
-
-        super().__init__(config, model, state, seed)
-
-    @property
-    def module(self) -> nn.Module:
-        return self._module
-
-    def __call__(self, input_ids, token_type_ids=None, position_ids=None, attention_mask=None):
-        if token_type_ids is None:
-            token_type_ids = jnp.ones_like(input_ids)
-
-        if position_ids is None:
-            position_ids = np.arange(
-                self.config.pad_token_id + 1, np.atleast_2d(input_ids).shape[-1] + self.config.pad_token_id + 1
-            )
-
-        if attention_mask is None:
-            attention_mask = jnp.ones_like(input_ids)
-
-        return self.model.apply(
-            {"params": self.params},
-            jnp.array(input_ids, dtype="i4"),
-            jnp.array(token_type_ids, dtype="i4"),
-            jnp.array(position_ids, dtype="i4"),
-            jnp.array(attention_mask, dtype="i4"),
-        )

src/transformers/modeling_flax_utils.py

@@ -1,194 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Google Flax Team Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-from abc import ABC, abstractmethod
-from pickle import UnpicklingError
-from typing import Dict
-
-import flax.linen as nn
-import jax
-import jax.numpy as jnp
-from flax.serialization import to_bytes
-from flax.traverse_util import unflatten_dict
-from jax.random import PRNGKey
-
-from .configuration_utils import PretrainedConfig
-from .file_utils import WEIGHTS_NAME, cached_path, hf_bucket_url, is_remote_url
-from .utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-
-@jax.jit
-def gelu(x):
-    r"""Gaussian error linear unit activation function.
-
-    Computes the element-wise function:
-
-    .. math::
-      \mathrm{gelu}(x) = \frac{x}{2} \left(1 + \mathrm{tanh} \left(
-        \sqrt{\frac{2}{\pi}} \left(x + 0.044715 x^3 \right) \right) \right)
-
-    We explicitly use the approximation rather than the exact formulation for
-    speed. For more information, see `Gaussian Error Linear Units (GELUs)
-    <https://arxiv.org/abs/1606.08415>`_, section 2.
-    """
-    return x * 0.5 * (1.0 + jax.lax.erf(x / jnp.sqrt(2.0)))
-
-
-ACT2FN = {
-    "gelu": nn.gelu,
-    "relu": nn.relu,
-    "swish": nn.swish,
-    "gelu_new": gelu,
-}
-
-
-class FlaxPreTrainedModel(ABC):
-    config_class = None
-    pretrained_model_archive_map = {}
-    base_model_prefix = ""
-    model_class = None
-
-    def __init__(self, config: PretrainedConfig, module: nn.Module, params: Dict, seed: int = 0):
-        if config is None:
-            raise ValueError("config cannot be None")
-
-        if module is None:
-            raise ValueError("module cannot be None")
-
-        if params is None:
-            raise ValueError("state cannot be None")
-
-        # Those are private to be exposed as typed property on derived classes.
-        self._config = config
-        self._module = module
-
-        # Those are public as their type is generic to every derived classes.
-        self.key = PRNGKey(seed)
-        self.params = params
-        self.model = module
-
-    @property
-    def config(self) -> PretrainedConfig:
-        return self._config
-
-    @staticmethod
-    @abstractmethod
-    def convert_from_pytorch(pt_state: Dict, config: PretrainedConfig) -> Dict:
-        raise NotImplementedError()
-
-    @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
-        r"""
-        Instantiate a pretrained Flax model from a pre-trained model configuration.
-        """
-        config = kwargs.pop("config", None)
-        # state_dict = kwargs.pop("state_dict", None)
-        cache_dir = kwargs.pop("cache_dir", None)
-        # from_tf = kwargs.pop("from_tf", False)
-        force_download = kwargs.pop("force_download", False)
-        resume_download = kwargs.pop("resume_download", False)
-        proxies = kwargs.pop("proxies", None)
-        # output_loading_info = kwargs.pop("output_loading_info", False)
-        local_files_only = kwargs.pop("local_files_only", False)
-        use_cdn = kwargs.pop("use_cdn", True)
-
-        # Load config if we don't provide a configuration
-        if not isinstance(config, PretrainedConfig):
-            config_path = config if config is not None else pretrained_model_name_or_path
-            config, model_kwargs = cls.config_class.from_pretrained(
-                config_path,
-                *model_args,
-                cache_dir=cache_dir,
-                return_unused_kwargs=True,
-                force_download=force_download,
-                resume_download=resume_download,
-                proxies=proxies,
-                local_files_only=local_files_only,
-                **kwargs,
-            )
-        else:
-            model_kwargs = kwargs
-
-        # Load model
-        if pretrained_model_name_or_path is not None:
-            if os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
-                archive_file = pretrained_model_name_or_path
-            else:
-                archive_file = hf_bucket_url(pretrained_model_name_or_path, filename=WEIGHTS_NAME, use_cdn=use_cdn)
-
-            # redirect to the cache, if necessary
-            try:
-                resolved_archive_file = cached_path(
-                    archive_file,
-                    cache_dir=cache_dir,
-                    force_download=force_download,
-                    proxies=proxies,
-                    resume_download=resume_download,
-                    local_files_only=local_files_only,
-                )
-            except EnvironmentError:
-                if pretrained_model_name_or_path in cls.pretrained_model_archive_map:
-                    msg = f"Couldn't reach server at '{archive_file}' to download pretrained weights."
-                else:
-                    msg = (
-                        f"Model name '{pretrained_model_name_or_path}' "
-                        f"was not found in model name list ({', '.join(cls.pretrained_model_archive_map.keys())}). "
-                        f"We assumed '{archive_file}' was a path or url to model weight files but "
-                        "couldn't find any such file at this path or url."
-                    )
-                raise EnvironmentError(msg)
-
-            if resolved_archive_file == archive_file:
-                logger.info(f"loading weights file {archive_file}")
-            else:
-                logger.info(f"loading weights file {archive_file} from cache at {resolved_archive_file}")
-        else:
-            resolved_archive_file = None
-
-        # Instantiate model.
-        with open(resolved_archive_file, "rb") as state_f:
-            try:
-                from flax.serialization import from_bytes
-
-                state = from_bytes(cls.model_class, state_f)
-            except TypeError:
-                try:
-                    import torch
-
-                    state = torch.load(state_f)
-                    state = {k: v.numpy() for k, v in state.items()}
-                    state = cls.convert_from_pytorch(state, config)
-                    state = unflatten_dict({tuple(k.split(".")[1:]): v for k, v in state.items()})
-                except UnpicklingError:
-                    raise EnvironmentError(
-                        f"Unable to convert model {archive_file} to Flax deserializable object. "
-                        "Supported format are PyTorch archive or Flax msgpack"
-                    )
-
-        return cls(config, state, *model_args, **model_kwargs)
-
-    def save_pretrained(self, folder):
-        folder_abs = os.path.abspath(folder)
-
-        if not os.path.exists(folder_abs):
-            os.mkdir(folder_abs)
-
-        with open(os.path.join(folder_abs, f"{self._config.model_type}.flax", "wb")) as f:
-            model_bytes = to_bytes(self.params)
-            f.write(model_bytes)

src/transformers/modeling_gpt2.py

@@ -33,7 +33,11 @@ from .file_utils import (
     add_start_docstrings_to_callable,
     replace_return_docstrings,
 )
-from .modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast
+from .modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+    SequenceClassifierOutputWithPast,
+)
 from .modeling_utils import (
     Conv1D,
     PreTrainedModel,
@@ -42,6 +46,7 @@ from .modeling_utils import (
     prune_conv1d_layer,
 )
 from .utils import logging
+from .utils.model_parallel_utils import assert_device_map, get_device_map
 
 
 logger = logging.get_logger(__name__)
@@ -124,7 +129,10 @@ class Attention(nn.Module):
         # [switch nx => n_state from Block to Attention to keep identical to TF implem]
         assert n_state % config.n_head == 0
         self.register_buffer(
-            "bias", torch.tril(torch.ones((n_ctx, n_ctx), dtype=torch.uint8)).view(1, 1, n_ctx, n_ctx)
+            "bias",
+            torch.tril(torch.ones((n_ctx, n_ctx), dtype=torch.uint8)).view(
+                1, 1, n_ctx, n_ctx
+            ),
         )
         self.register_buffer("masked_bias", torch.tensor(-1e4))
         self.n_head = config.n_head
@@ -147,7 +155,9 @@ class Attention(nn.Module):
         heads, index = find_pruneable_heads_and_indices(
             heads, self.n_head, self.split_size // self.n_head, self.pruned_heads
         )
-        index_attn = torch.cat([index, index + self.split_size, index + (2 * self.split_size)])
+        index_attn = torch.cat(
+            [index, index + self.split_size, index + (2 * self.split_size)]
+        )
 
         # Prune conv1d layers
         self.c_attn = prune_conv1d_layer(self.c_attn, index_attn, dim=1)
@@ -158,7 +168,9 @@ class Attention(nn.Module):
         self.n_head = self.n_head - len(heads)
         self.pruned_heads = self.pruned_heads.union(heads)
 
-    def _attn(self, q, k, v, attention_mask=None, head_mask=None, output_attentions=False):
+    def _attn(
+        self, q, k, v, attention_mask=None, head_mask=None, output_attentions=False
+    ):
         w = torch.matmul(q, k)
         if self.scale:
             w = w / (float(v.size(-1)) ** 0.5)
@@ -214,7 +226,9 @@ class Attention(nn.Module):
                 self, "q_attn"
             ), "If class is used as cross attention, the weights `q_attn` have to be defined. Please make sure to instantiate class with `Attention(..., is_cross_attention=True)`."
             query = self.q_attn(hidden_states)
-            key, value = self.c_attn(encoder_hidden_states).split(self.split_size, dim=2)
+            key, value = self.c_attn(encoder_hidden_states).split(
+                self.split_size, dim=2
+            )
             attention_mask = encoder_attention_mask
         else:
             query, key, value = self.c_attn(hidden_states).split(self.split_size, dim=2)
@@ -223,16 +237,23 @@ class Attention(nn.Module):
         key = self.split_heads(key, k=True)
         value = self.split_heads(value)
         if layer_past is not None:
-            past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1]  # transpose back cf below
+            past_key, past_value = (
+                layer_past[0].transpose(-2, -1),
+                layer_past[1],
+            )  # transpose back cf below
             key = torch.cat((past_key, key), dim=-1)
             value = torch.cat((past_value, value), dim=-2)
 
         if use_cache is True:
-            present = torch.stack((key.transpose(-2, -1), value))  # transpose to have same shapes for stacking
+            present = torch.stack(
+                (key.transpose(-2, -1), value)
+            )  # transpose to have same shapes for stacking
         else:
             present = (None,)
 
-        attn_outputs = self._attn(query, key, value, attention_mask, head_mask, output_attentions)
+        attn_outputs = self._attn(
+            query, key, value, attention_mask, head_mask, output_attentions
+        )
         a = attn_outputs[0]
 
         a = self.merge_heads(a)
@@ -267,8 +288,12 @@ class Block(nn.Module):
         self.attn = Attention(hidden_size, n_ctx, config, scale)
         self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
         if config.add_cross_attention:
-            self.crossattention = Attention(hidden_size, n_ctx, config, scale, is_cross_attention=True)
-            self.ln_cross_attn = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
+            self.crossattention = Attention(
+                hidden_size, n_ctx, config, scale, is_cross_attention=True
+            )
+            self.ln_cross_attn = nn.LayerNorm(
+                hidden_size, eps=config.layer_norm_epsilon
+            )
         self.mlp = MLP(inner_dim, config)
 
     def forward(
@@ -311,7 +336,9 @@ class Block(nn.Module):
             attn_output = cross_attn_outputs[0]
             # residual connection
             hidden_states = hidden_states + attn_output
-            outputs = outputs + cross_attn_outputs[1:]  # add cross attentions if we output attention weights
+            outputs = (
+                outputs + cross_attn_outputs[1:]
+            )  # add cross attentions if we output attention weights
 
         feed_forward_hidden_states = self.mlp(self.ln_2(hidden_states))
         # residual connection
@@ -472,6 +499,48 @@ GPT2_INPUTS_DOCSTRING = r"""
             Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple.
 """
 
+PARALLELIZE_DOCSTRING = r"""
+    Uses a device map to distribute attention modules of the model across several devices. If no device map is given, it
+    will evenly distribute blocks across all devices.
+    Args:
+        device_map (:obj:`Dict[int, list]`, optional, defaults to None):
+            A dictionary that maps attention modules to devices. Note that the embedding module and LMHead are
+            always automatically mapped to the first device (for esoteric reasons). That means that the first
+            device should have fewer attention modules mapped to it than other devices.
+
+            For reference, the gpt2 models have the following number of attention modules:
+
+                - gpt2: 12
+                - gpt2-medium: 24
+                - gpt2-large: 36
+                - gpt2-xl: 48
+
+    Example::
+        Here is an example of a device map on a machine with 4 GPUs using gpt2-xl, which has a total of 48 attention modules:
+
+            model = GPT2LMHeadModel.from_pretrained('gpt2-xl')
+            device_map = {0: [0, 1, 2, 3, 4, 5, 6, 7, 8],
+                          1: [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21],
+                          2: [22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34],
+                          3: [35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]}
+            model.parallelize(device_map)
+"""
+
+DEPARALLELIZE_DOCSTRING = r"""
+    Moves the model to cpu from a model parallel state.
+
+    Example::
+        On a 4 GPU machine with gpt2-large:
+
+        model = GPT2LMHeadModel.from_pretrained('gpt2-large')
+        device_map = {0: [0, 1, 2, 3, 4, 5, 6, 7],
+                    1: [8, 9, 10, 11, 12, 13, 14, 15],
+                    2: [16, 17, 18, 19, 20, 21, 22, 23],
+                    3: [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]}
+        model.parallelize(device_map) # Splits the model across several devices
+        model.deparallelize() # Put the model back on cpu and cleans memory by calling torch.cuda.empty_cache()
+"""
+
 
 @add_start_docstrings(
     "The bare GPT2 Model transformer outputting raw hidden-states without any specific head on top.",
@@ -484,11 +553,58 @@ class GPT2Model(GPT2PreTrainedModel):
         self.wte = nn.Embedding(config.vocab_size, config.n_embd)
         self.wpe = nn.Embedding(config.n_positions, config.n_embd)
         self.drop = nn.Dropout(config.embd_pdrop)
-        self.h = nn.ModuleList([Block(config.n_ctx, config, scale=True) for _ in range(config.n_layer)])
+        self.h = nn.ModuleList(
+            [Block(config.n_ctx, config, scale=True) for _ in range(config.n_layer)]
+        )
         self.ln_f = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
 
         self.init_weights()
 
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+
+    @add_start_docstrings(PARALLELIZE_DOCSTRING)
+    def parallelize(self, device_map=None):
+        # Check validity of device_map
+        self.device_map = (
+            get_device_map(len(self.h), range(torch.cuda.device_count()))
+            if device_map is None
+            else device_map
+        )
+        assert_device_map(self.device_map, len(self.h))
+
+        self.model_parallel = True
+        self.first_device = (
+            "cpu"
+            if "cpu" in self.device_map.keys()
+            else "cuda:" + str(min(self.device_map.keys()))
+        )
+        self.last_device = "cuda:" + str(max(self.device_map.keys()))
+        self.wte = self.wte.to(self.first_device)
+        self.wpe = self.wpe.to(self.first_device)
+        # Load onto devices
+        for k, v in self.device_map.items():
+            for block in v:
+                cuda_device = "cuda:" + str(k)
+                self.h[block] = self.h[block].to(cuda_device)
+        # ln_f to last
+        self.ln_f = self.ln_f.to(self.last_device)
+
+    @add_start_docstrings(DEPARALLELIZE_DOCSTRING)
+    def deparallelize(self):
+        self.model_parallel = False
+        self.device_map = None
+        self.first_device = "cpu"
+        self.last_device = "cpu"
+        self.wte = self.wte.to("cpu")
+        self.wpe = self.wpe.to("cpu")
+        for index in range(len(self.h)):
+            self.h[index] = self.h[index].to("cpu")
+
+        self.ln_f = self.ln_f.to("cpu")
+        torch.cuda.empty_cache()
+
     def get_input_embeddings(self):
         return self.wte
 
@@ -534,15 +650,25 @@ class GPT2Model(GPT2PreTrainedModel):
             past_key_values = kwargs.pop("past")
         assert kwargs == {}, f"Unexpected keyword arguments: {list(kwargs.keys())}."
 
-        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
         output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
         )
         use_cache = use_cache if use_cache is not None else self.config.use_cache
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
 
         if input_ids is not None and inputs_embeds is not None:
-            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+            raise ValueError(
+                "You cannot specify both input_ids and inputs_embeds at the same time"
+            )
         elif input_ids is not None:
             input_shape = input_ids.size()
             input_ids = input_ids.view(-1, input_shape[-1])
@@ -565,7 +691,12 @@ class GPT2Model(GPT2PreTrainedModel):
             past_length = past_key_values[0][0].size(-2)
         if position_ids is None:
             device = input_ids.device if input_ids is not None else inputs_embeds.device
-            position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
+            position_ids = torch.arange(
+                past_length,
+                input_shape[-1] + past_length,
+                dtype=torch.long,
+                device=device,
+            )
             position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
 
         # Attention mask.
@@ -590,7 +721,11 @@ class GPT2Model(GPT2PreTrainedModel):
         # If a 2D ou 3D attention mask is provided for the cross-attention
         # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length]
         if self.config.add_cross_attention and encoder_hidden_states is not None:
-            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
+            (
+                encoder_batch_size,
+                encoder_sequence_length,
+                _,
+            ) = encoder_hidden_states.size()
             encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
             if encoder_attention_mask is None:
                 encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
@@ -620,15 +755,33 @@ class GPT2Model(GPT2PreTrainedModel):
         all_attentions = () if output_attentions else None
         all_hidden_states = () if output_hidden_states else None
         for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
+            # Model parallel
+            if self.model_parallel:
+                torch.cuda.set_device(hidden_states.device)
+                # Ensure layer_past is on same device as hidden_states (might not be correct)
+                if layer_past is not None:
+                    layer_past = layer_past.to(hidden_states.device)
+                # Ensure that attention_mask is always on the same device as hidden_states
+                if attention_mask is not None:
+                    attention_mask = attention_mask.to(hidden_states.device)
+
+                if isinstance(head_mask, torch.Tensor):
+                    head_mask = head_mask.to(hidden_states.device)
+
             if output_hidden_states:
-                all_hidden_states = all_hidden_states + (hidden_states.view(*output_shape),)
+                all_hidden_states = all_hidden_states + (
+                    hidden_states.view(*output_shape),
+                )
 
             if getattr(self.config, "gradient_checkpointing", False):
 
                 def create_custom_forward(module):
                     def custom_forward(*inputs):
                         # checkpointing only works with tuple returns, not with lists
-                        return tuple(output for output in module(*inputs, use_cache, output_attentions))
+                        return tuple(
+                            output
+                            for output in module(*inputs, use_cache, output_attentions)
+                        )
 
                     return custom_forward
 
@@ -660,6 +813,12 @@ class GPT2Model(GPT2PreTrainedModel):
             if output_attentions:
                 all_attentions = all_attentions + (outputs[2],)
 
+            # Model Parallel: If it's the last layer for that device, put things on the next device
+            if self.model_parallel:
+                for k, v in self.device_map.items():
+                    if i == v[-1] and "cuda:" + str(k) != self.last_device:
+                        hidden_states = hidden_states.to("cuda:" + str(k + 1))
+
         hidden_states = self.ln_f(hidden_states)
 
         hidden_states = hidden_states.view(*output_shape)
@@ -668,7 +827,11 @@ class GPT2Model(GPT2PreTrainedModel):
             all_hidden_states = all_hidden_states + (hidden_states,)
 
         if not return_dict:
-            return tuple(v for v in [hidden_states, presents, all_hidden_states, all_attentions] if v is not None)
+            return tuple(
+                v
+                for v in [hidden_states, presents, all_hidden_states, all_attentions]
+                if v is not None
+            )
 
         return BaseModelOutputWithPast(
             last_hidden_state=hidden_states,
@@ -693,6 +856,29 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
 
         self.init_weights()
 
+        self.model_parallel = False
+
+    @add_start_docstrings(PARALLELIZE_DOCSTRING)
+    def parallelize(self, device_map=None):
+        self.device_map = (
+            get_device_map(len(self.transformer.h), range(torch.cuda.device_count()))
+            if device_map is None
+            else device_map
+        )
+        assert_device_map(self.device_map, len(self.transformer.h))
+
+        self.transformer.parallelize(self.device_map)
+        self.lm_head = self.lm_head.to(self.transformer.first_device)
+        self.model_parallel = True
+
+    @add_start_docstrings(DEPARALLELIZE_DOCSTRING)
+    def deparallelize(self):
+        self.transformer.deparallelize()
+        self.transformer = self.transformer.to("cpu")
+        self.lm_head = self.lm_head.to("cpu")
+        self.model_parallel = False
+        torch.cuda.empty_cache()
+
     def get_output_embeddings(self):
         return self.lm_head
 
@@ -701,23 +887,10 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
         if past:
             input_ids = input_ids[:, -1].unsqueeze(-1)
 
-        attention_mask = kwargs.get("attention_mask", None)
-        position_ids = kwargs.get("position_ids", None)
-
-        if attention_mask is not None and position_ids is None:
-            # create postion_ids on the fly for batch generation
-            position_ids = attention_mask.long().cumsum(-1) - 1
-            position_ids.masked_fill_(attention_mask == 0, 1)
-            if past:
-                position_ids = position_ids[:, -1].unsqueeze(-1)
-        else:
-            position_ids = None
         return {
             "input_ids": input_ids,
             "past_key_values": past,
             "use_cache": kwargs.get("use_cache"),
-            "position_ids": position_ids,
-            "attention_mask": attention_mask,
         }
 
     @add_start_docstrings_to_callable(GPT2_INPUTS_DOCSTRING)
@@ -760,7 +933,9 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
             )
             past_key_values = kwargs.pop("past")
         assert kwargs == {}, f"Unexpected keyword arguments: {list(kwargs.keys())}."
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
 
         transformer_outputs = self.transformer(
             input_ids,
@@ -779,6 +954,11 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
         )
         hidden_states = transformer_outputs[0]
 
+        # Set device for model parallelism
+        if self.model_parallel:
+            torch.cuda.set_device(self.transformer.first_device)
+            hidden_states = hidden_states.to(self.lm_head.weight.device)
+
         lm_logits = self.lm_head(hidden_states)
 
         loss = None
@@ -788,7 +968,9 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
             shift_labels = labels[..., 1:].contiguous()
             # Flatten the tokens
             loss_fct = CrossEntropyLoss()
-            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
+            )
 
         if not return_dict:
             output = (lm_logits,) + transformer_outputs[1:]
@@ -836,7 +1018,9 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
         }
 
     @add_start_docstrings_to_callable(GPT2_INPUTS_DOCSTRING)
-    @replace_return_docstrings(output_type=GPT2DoubleHeadsModelOutput, config_class=_CONFIG_FOR_DOC)
+    @replace_return_docstrings(
+        output_type=GPT2DoubleHeadsModelOutput, config_class=_CONFIG_FOR_DOC
+    )
     def forward(
         self,
         input_ids=None,
@@ -912,7 +1096,9 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
             )
             past_key_values = kwargs.pop("past")
         assert kwargs == {}, f"Unexpected keyword arguments: {list(kwargs.keys())}."
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
 
         transformer_outputs = self.transformer(
             input_ids,
@@ -936,13 +1122,17 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
         mc_loss = None
         if mc_labels is not None:
             loss_fct = CrossEntropyLoss()
-            mc_loss = loss_fct(mc_logits.view(-1, mc_logits.size(-1)), mc_labels.view(-1))
+            mc_loss = loss_fct(
+                mc_logits.view(-1, mc_logits.size(-1)), mc_labels.view(-1)
+            )
         lm_loss = None
         if labels is not None:
             shift_logits = lm_logits[..., :-1, :].contiguous()
             shift_labels = labels[..., 1:].contiguous()
             loss_fct = CrossEntropyLoss()
-            lm_loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+            lm_loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
+            )
 
         if not return_dict:
             output = (lm_logits, mc_logits) + transformer_outputs[1:]
@@ -1016,7 +1206,9 @@ class GPT2ForSequenceClassification(GPT2PreTrainedModel):
             If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
             If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
         """
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
 
         transformer_outputs = self.transformer(
             input_ids,
@@ -1046,7 +1238,9 @@ class GPT2ForSequenceClassification(GPT2PreTrainedModel):
             sequence_lengths = -1
         else:
             if input_ids is not None:
-                sequence_lengths = torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1
+                sequence_lengths = (
+                    torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1
+                )
             else:
                 sequence_lengths = -1
                 logger.warning(
@@ -1064,7 +1258,9 @@ class GPT2ForSequenceClassification(GPT2PreTrainedModel):
                 loss = loss_fct(pooled_logits.view(-1), labels.view(-1))
             else:
                 loss_fct = CrossEntropyLoss()
-                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+                loss = loss_fct(
+                    pooled_logits.view(-1, self.num_labels), labels.view(-1)
+                )
 
         if not return_dict:
             output = (pooled_logits,) + transformer_outputs[1:]

src/transformers/modeling_lxmert.py

@@ -15,6 +15,7 @@
 """ PyTorch LXMERT model. """
 
 
+import logging
 import math
 import os
 from dataclasses import dataclass
@@ -34,10 +35,9 @@ from .file_utils import (
     replace_return_docstrings,
 )
 from .modeling_utils import PreTrainedModel
-from .utils import logging
 
 
-logger = logging.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 _CONFIG_FOR_DOC = "LxmertConfig"
 _TOKENIZER_FOR_DOC = "LxmertTokenizer"