refactor encoder layer

delete unnecessary files
finish new model outputs
2020-10-13 17:39:02 +00:00 · 2020-10-12 22:44:07 +00:00 · 2020-10-12 22:43:39 +00:00 · 2020-10-12 13:26:09 +00:00 · 2020-10-09 17:00:44 +00:00 · 2020-10-09 09:52:16 +00:00
160 changed files with 4743 additions and 2969 deletions
@@ -37,27 +37,25 @@ members/contributors which may be interested in your PR.
 If you know how to use git blame, that is the easiest way, otherwise, here is a rough guide of **who to tag**.
 Please tag fewer than 3 people.

- albert, bert, XLM: @LysandreJik 
- GPT2: @LysandreJik, @patrickvonplaten
+ albert, bert, GPT2, XLM: @LysandreJik 
 tokenizers: @mfuntowicz
 Trainer: @sgugger
- Benchmarks: @patrickvonplaten
+ Speed and Memory Benchmarks: @patrickvonplaten
 Model Cards: @julien-c
 Translation: @sshleifer
 Summarization: @sshleifer
+ TextGeneration: @TevenLeScao 
 examples/distillation: @VictorSanh
 nlp datasets: [different repo](https://github.com/huggingface/nlp)
 rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)
- Text Generation: @patrickvonplaten, @TevenLeScao
+ Text Generation: @TevenLeScao
 Blenderbot, Bart, Marian, Pegasus: @sshleifer
 T5: @patrickvonplaten
- Rag: @patrickvonplaten, @lhoestq
- EncoderDecoder: @patrickvonplaten
- Longformer, Reformer: @patrickvonplaten
- TransfoXL, XLNet: @TevenLeScao, @patrickvonplaten
+ Longformer/Reformer: @patrickvonplaten
+ TransfoXL/XLNet: @TevenLeScao 
 examples/seq2seq: @sshleifer
 examples/bert-loses-patience: @JetRunner
 tensorflow: @jplu
 examples/token-classification: @stefan-it
 documentation: @sgugger
- -->
+ -->
@@ -11,109 +11,63 @@ on:
  # pull_request:
  repository_dispatch:

+
 jobs:
-  run_tests_tf_gpu:
+  run_tests_torch_and_tf_gpu:
    runs-on: [self-hosted, single-gpu]
    steps:
-      - uses: actions/checkout@v2
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install torch!=1.6.0
-          pip install .[sklearn,testing,onnxruntime]
-          pip install git+https://github.com/huggingface/datasets
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-      - name: Run all tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-          USE_CUDA: yes
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/
+    - uses: actions/checkout@v2
+    - name: Python version
+      run: |
+        which python
+        python --version
+        pip --version
+    - name: Current dir
+      run: pwd
+    - run: nvidia-smi

-  run_tests_torch_gpu:
-    runs-on: [self-hosted, single-gpu]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install torch!=1.6.0
-          pip install .[sklearn,testing,onnxruntime]
-          pip install git+https://github.com/huggingface/datasets
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-      - name: Run all tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-          USE_CUDA: yes
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/
+    - name: Loading cache.
+      uses: actions/cache@v2
+      id: cache
+      with:
+        path: .env
+        key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }}

-  run_tests_tf_multiple_gpu:
+    - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
+      run: |
+        python -m venv .env
+        source .env/bin/activate
+        which python
+        python --version
+        pip --version
+    - name: Install dependencies
+      run: |
+        source .env/bin/activate
+        pip install --upgrade pip
+        pip install torch!=1.6.0
+        pip install .[sklearn,testing,onnxruntime]
+        pip install git+https://github.com/huggingface/datasets
+
+    - name: Are GPUs recognized by our DL frameworks
+      run: |
+        source .env/bin/activate
+        python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
+        python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+
+    - name: Run all non-slow tests on GPU
+      env:
+        TF_FORCE_GPU_ALLOW_GROWTH: "true"
+        # TF_GPU_MEMORY_LIMIT: 4096
+        OMP_NUM_THREADS: 1
+        USE_CUDA: yes
+      run: |
+        source .env/bin/activate
+        python -m pytest -n 2 --dist=loadfile -s ./tests/
+
+  run_tests_torch_and_tf_multiple_gpu:
    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
      - name: Python version
        run: |
          which python
@@ -122,54 +76,15 @@ jobs:
      - name: Current dir
        run: pwd
      - run: nvidia-smi
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install torch!=1.6.0
-          pip install .[sklearn,testing,onnxruntime]
-          pip install git+https://github.com/huggingface/datasets
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-      - name: Run all tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-          USE_CUDA: yes
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/

-  run_tests_torch_multiple_gpu:
-    runs-on: [self-hosted, multi-gpu]
-    steps:
-      - uses: actions/checkout@v2
      - name: Loading cache.
        uses: actions/cache@v2
        id: cache
        with:
          path: .env
-          key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
+          key: v0-tests_tf_torch_multiple_gpu-${{ hashFiles('setup.py') }}
+
      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m venv .env
          source .env/bin/activate
@@ -183,15 +98,19 @@ jobs:
          pip install torch!=1.6.0
          pip install .[sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets
+
      - name: Are GPUs recognized by our DL frameworks
        run: |
          source .env/bin/activate
          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-      - name: Run all tests on GPU
+
+      - name: Run all non-slow tests on GPU
        env:
+          TF_FORCE_GPU_ALLOW_GROWTH: "true"
+          # TF_GPU_MEMORY_LIMIT: 4096
          OMP_NUM_THREADS: 1
          USE_CUDA: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/
+          python -m pytest -n 2 --dist=loadfile -s ./tests/
@@ -9,16 +9,82 @@ on:
    - cron: "0 0 * * *"

 jobs:
-  run_all_tests_torch_gpu:
+  run_all_tests_torch_and_tf_gpu:
    runs-on: [self-hosted, single-gpu]
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Loading cache.
+      uses: actions/cache@v2
+      id: cache
+      with:
+        path: .env
+        key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
+
+    - name: Python version
+      run: |
+        which python
+        python --version
+        pip --version
+    - name: Current dir
+      run: pwd
+    - run: nvidia-smi
+    - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        python -m venv .env
+        source .env/bin/activate
+        which python
+        python --version
+        pip --version
+    - name: Install dependencies
+      run: |
+        source .env/bin/activate
+        pip install --upgrade pip
+        pip install torch!=1.6.0
+        pip install .[sklearn,testing,onnxruntime]
+        pip install git+https://github.com/huggingface/datasets
+
+    - name: Are GPUs recognized by our DL frameworks
+      run: |
+        source .env/bin/activate
+        python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
+        python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+
+
+    - name: Run all tests on GPU
+      env:
+        TF_FORCE_GPU_ALLOW_GROWTH: "true"
+        OMP_NUM_THREADS: 1
+        RUN_SLOW: yes
+        USE_CUDA: yes
+      run: |
+        source .env/bin/activate
+        python -m pytest -n 1 --dist=loadfile -s ./tests/
+
+    - name: Run examples tests on GPU
+      env:
+        TF_FORCE_GPU_ALLOW_GROWTH: "true"
+        OMP_NUM_THREADS: 1
+        RUN_SLOW: yes
+        USE_CUDA: yes
+      run: |
+        source .env/bin/activate
+        pip install -r examples/requirements.txt
+        python -m pytest -n 1 --dist=loadfile -s examples
+
+  run_all_tests_torch_and_tf_multiple_gpu:
+    runs-on: [self-hosted, multi-gpu]
    steps:
      - uses: actions/checkout@v2
+
      - name: Loading cache.
        uses: actions/cache@v2
        id: cache
        with:
          path: .env
-          key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
+          key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }}
+
      - name: Python version
        run: |
          which python
@@ -42,13 +108,16 @@ jobs:
          pip install torch!=1.6.0
          pip install .[sklearn,testing,onnxruntime]
          pip install git+https://github.com/huggingface/datasets
+
      - name: Are GPUs recognized by our DL frameworks
        run: |
          source .env/bin/activate
          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-      - name: Run examples tests on GPU
+
+      - name: Run all tests on GPU
        env:
+          TF_FORCE_GPU_ALLOW_GROWTH: "true"
          OMP_NUM_THREADS: 1
          RUN_SLOW: yes
          USE_CUDA: yes
@@ -56,235 +125,9 @@ jobs:
          source .env/bin/activate
          python -m pytest -n 1 --dist=loadfile -s ./tests/

-  run_examples_torch_and_tf_gpu:
-    runs-on: [self-hosted, single-gpu]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install torch!=1.6.0
-          pip install .[tf,sklearn,testing,onnxruntime]
-          pip install git+https://github.com/huggingface/datasets
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-      - name: Run examples tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-          RUN_SLOW: yes
-          USE_CUDA: yes
-        run: |
-          source .env/bin/activate
-          pip install -r examples/requirements.txt
-          python -m pytest -n 1 --dist=loadfile -s examples
-
-  run_all_tests_tf_gpu:
-    runs-on: [self-hosted, single-gpu]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install torch!=1.6.0
-          pip install .[sklearn,testing,onnxruntime]
-          pip install git+https://github.com/huggingface/datasets
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-      - name: Run examples tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-          RUN_SLOW: yes
-          USE_CUDA: yes
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/
-
-  run_all_tests_torch_multiple_gpu:
-    runs-on: [self-hosted, multi-gpu]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }}
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install torch!=1.6.0
-          pip install .[sklearn,testing,onnxruntime]
-          pip install git+https://github.com/huggingface/datasets
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-      - name: Run examples tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-          RUN_SLOW: yes
-          USE_CUDA: yes
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/
-
-  run_all_tests_tf_multiple_gpu:
-    runs-on: [self-hosted, multi-gpu]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }}
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install torch!=1.6.0
-          pip install .[sklearn,testing,onnxruntime]
-          pip install git+https://github.com/huggingface/datasets
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
-      - name: Run examples tests on GPU
-        env:
-          OMP_NUM_THREADS: 1
-          RUN_SLOW: yes
-          USE_CUDA: yes
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s ./tests/
-
-  run_examples_torch_and_tf_multiple_gpu:
-    runs-on: [self-hosted, multi-gpu]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Loading cache.
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: .env
-          key: v0-slow_tests_tf_torch_multi_gpu-${{ hashFiles('setup.py') }}
-      - name: Python version
-        run: |
-          which python
-          python --version
-          pip --version
-      - name: Current dir
-        run: pwd
-      - run: nvidia-smi
-      - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          python -m venv .env
-          source .env/bin/activate
-          which python
-          python --version
-          pip --version
-      - name: Install dependencies
-        run: |
-          source .env/bin/activate
-          pip install --upgrade pip
-          pip install torch!=1.6.0
-          pip install .[tf,sklearn,testing,onnxruntime]
-          pip install git+https://github.com/huggingface/datasets
-      - name: Are GPUs recognized by our DL frameworks
-        run: |
-          source .env/bin/activate
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
      - name: Run examples tests on GPU
        env:
+          TF_FORCE_GPU_ALLOW_GROWTH: "true"
          OMP_NUM_THREADS: 1
          RUN_SLOW: yes
          USE_CUDA: yes
@@ -12,7 +12,6 @@ __pycache__/
 tests/fixtures
 logs/
 lightning_logs/
-lang_code_data/

 # Distribution / packaging
 .Python
@@ -161,34 +161,36 @@ If you'd like to play with the examples, you must [install the library from sour
 1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.
 1. **[BART](https://huggingface.co/transformers/model_doc/bart.html)** (from Facebook) released with the paper [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension](https://arxiv.org/pdf/1910.13461.pdf) by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer.
 1. **[BERT](https://huggingface.co/transformers/model_doc/bert.html)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.
-1. **[BERT For Sequence Generation](https://huggingface.co/transformers/model_doc/bertgeneration.html)** (from Google) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn.
-1. **[Blenderbot](https://huggingface.co/transformers/master/model_doc/blenderbot.html)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston.
+1. **[BERT For Sequence Generation](https://tfhub.dev/s?module-type=text-generation&subtype=module,placeholder)** (from Google) released with the paper [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn.
 1. **[CamemBERT](https://huggingface.co/transformers/model_doc/camembert.html)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
 1. **[CTRL](https://huggingface.co/transformers/model_doc/ctrl.html)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
-1. **[DeBERTa](https://huggingface.co/transformers/master/model_doc/deberta.html)** (from Microsoft Research) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
+1. **[DeBERTa](https://huggingface.co/transformers/model_doc/deberta.html)** (from Microsoft Research) released with the paper [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
 1. **[DialoGPT](https://huggingface.co/transformers/model_doc/dialogpt.html)** (from Microsoft Research) released with the paper [DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation](https://arxiv.org/abs/1911.00536) by Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan.
 1. **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/master/examples/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT.
-1. **[DPR](https://huggingface.co/transformers/model_doc/dpr.html)** (from Facebook) released with the paper [Dense Passage Retrieval
+1. **[DPR](https://github.com/facebookresearch/DPR)** (from Facebook) released with the paper [Dense Passage Retrieval
 for Open-Domain Question Answering](https://arxiv.org/abs/2004.04906) by Vladimir Karpukhin, Barlas Oğuz, Sewon
 Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
 1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning.
 1. **[FlauBERT](https://huggingface.co/transformers/model_doc/flaubert.html)** (from CNRS) released with the paper [FlauBERT: Unsupervised Language Model Pre-training for French](https://arxiv.org/abs/1912.05372) by Hang Le, Loïc Vial, Jibril Frej, Vincent Segonne, Maximin Coavoux, Benjamin Lecouteux, Alexandre Allauzen, Benoît Crabbé, Laurent Besacier, Didier Schwab.
-1. **[Funnel Transformer](https://huggingface.co/transformers/model_doc/funnel.html)** (from CMU/Google Brain) released with the paper [Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing](https://arxiv.org/abs/2006.03236) by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
+1. **[Funnel Transformer](https://github.com/laiguokun/Funnel-Transformer)** (from CMU/Google Brain) released with the paper [Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing](https://arxiv.org/abs/2006.03236) by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
 1. **[GPT](https://huggingface.co/transformers/model_doc/gpt.html)** (from OpenAI) released with the paper [Improving Language Understanding by Generative Pre-Training](https://blog.openai.com/language-unsupervised/) by Alec Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever.
 1. **[GPT-2](https://huggingface.co/transformers/model_doc/gpt2.html)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**.
-1. **[LayoutLM](https://huggingface.co/transformers/model_doc/layoutlm.html)** (from Microsoft Research Asia) released with the paper [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou.
+1. **[LayoutLM](https://github.com/microsoft/unilm/tree/master/layoutlm)** (from Microsoft Research Asia) released with the paper [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou.
 1. **[Longformer](https://huggingface.co/transformers/model_doc/longformer.html)** (from AllenAI) released with the paper [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, Arman Cohan.
-1. **[LXMERT](https://huggingface.co/transformers/model_doc/lxmert.html)** (from UNC Chapel Hill) released with the paper [LXMERT: Learning Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal.
+1. **[LXMERT](https://github.com/airsplay/lxmert)** (from UNC Chapel Hill) released with the paper [LXMERT: Learning Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal.
 1. **[MarianMT](https://huggingface.co/transformers/model_doc/marian.html)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team.
-1. **[MBart](https://huggingface.co/transformers/model_doc/mbart.html)** (from Facebook) released with the paper  [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-1. **[Pegasus](https://huggingface.co/transformers/model_doc/pegasus.html)** (from Google) released with the paper [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization](https://arxiv.org/abs/1912.08777)> by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
+1. **[MBart](https://github.com/pytorch/fairseq/tree/master/examples/mbart)** (from Facebook) released with the paper  [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
+1. **[MMBT](https://github.com/facebookresearch/mmbt/)** (from Facebook), released together with the paper a [Supervised Multimodal Bitransformers for Classifying Images and Text](https://arxiv.org/pdf/1909.02950.pdf) by Douwe Kiela, Suvrat Bhooshan, Hamed Firooz, Davide Testuggine.
+1. **[Pegasus](https://github.com/google-research/pegasus)** (from Google) released with the paper [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization](https://arxiv.org/abs/1912.08777)> by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
+1. **[ProphetNet](https://github.com/microsoft/ProphetNet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
 1. **[Reformer](https://huggingface.co/transformers/model_doc/reformer.html)** (from Google Research) released with the paper [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
 1. **[RoBERTa](https://huggingface.co/transformers/model_doc/roberta.html)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
 ultilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT.
-1. **[SqueezeBert](https://huggingface.co/transformers/master/model_doc/squeezebert.html)** released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://arxiv.org/abs/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
+1. **[SqueezeBert](https://huggingface.co/transformers/model_doc/squeezebert.html)** released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://arxiv.org/abs/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
 1. **[T5](https://huggingface.co/transformers/model_doc/t5.html)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
 1. **[Transformer-XL](https://huggingface.co/transformers/model_doc/transformerxl.html)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
 1. **[XLM](https://huggingface.co/transformers/model_doc/xlm.html)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau.
+1. **[XLMProphetNet](https://github.com/microsoft/ProphetNet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
 1. **[XLM-RoBERTa](https://huggingface.co/transformers/model_doc/xlmroberta.html)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov.
 1. **[XLNet](https://huggingface.co/transformers/model_doc/xlnet.html)** (from Google/CMU) released with the paper [XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
 1. **[Other community models](https://huggingface.co/models)**, contributed by the [community](https://huggingface.co/users).
@@ -57,104 +57,119 @@ conversion utilities for the following models:
 ..
    This list is updated automatically from the README with `make fix-copies`. Do not update manually!

-1. :doc:`ALBERT <model_doc/albert>` (from Google Research and the Toyota Technological Institute at Chicago) released
-   with the paper `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations
-   <https://arxiv.org/abs/1909.11942>`__, by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush
-   Sharma, Radu Soricut.
-2. :doc:`BART <model_doc/bart>` (from Facebook) released with the paper `BART: Denoising Sequence-to-Sequence
-   Pre-training for Natural Language Generation, Translation, and Comprehension
+1. `ALBERT <https://huggingface.co/transformers/model_doc/albert.html>`__ (from Google Research and the Toyota
+   Technological Institute at Chicago) released with the paper `ALBERT: A Lite BERT for Self-supervised Learning of
+   Language Representations <https://arxiv.org/abs/1909.11942>`__, by Zhenzhong Lan, Mingda Chen, Sebastian Goodman,
+   Kevin Gimpel, Piyush Sharma, Radu Soricut.
+2. `BART <https://huggingface.co/transformers/model_doc/bart.html>`__ (from Facebook) released with the paper `BART:
+   Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension
   <https://arxiv.org/pdf/1910.13461.pdf>`__ by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman
   Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer.
-3. :doc:`BERT <model_doc/bert>` (from Google) released with the paper `BERT: Pre-training of Deep Bidirectional
-   Transformers for Language Understanding <https://arxiv.org/abs/1810.04805>`__ by Jacob Devlin, Ming-Wei Chang,
-   Kenton Lee and Kristina Toutanova.
-4. :doc:`BERT For Sequence Generation <model_doc/bertgeneration>` (from Google) released with the paper `Leveraging
-   Pre-trained Checkpoints for Sequence Generation Tasks <https://arxiv.org/abs/1907.12461>`__ by Sascha Rothe, Shashi
-   Narayan, Aliaksei Severyn.
-5. `Blenderbot <https://huggingface.co/transformers/master/model_doc/blenderbot.html>`__ (from Facebook) released with
-   the paper `Recipes for building an open-domain chatbot <https://arxiv.org/abs/2004.13637>`__ by Stephen Roller,
-   Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan
-   Boureau, Jason Weston.
-6. :doc:`CamemBERT <model_doc/camembert>` (from Inria/Facebook/Sorbonne) released with the paper `CamemBERT: a Tasty
-   French Language Model <https://arxiv.org/abs/1911.03894>`__ by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz
-   Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot.
-7. :doc:`CTRL <model_doc/ctrl>` (from Salesforce) released with the paper `CTRL: A Conditional Transformer Language
-   Model for Controllable Generation <https://arxiv.org/abs/1909.05858>`__ by Nitish Shirish Keskar*, Bryan McCann*,
-   Lav R. Varshney, Caiming Xiong and Richard Socher.
-8. `DeBERTa <https://huggingface.co/transformers/master/model_doc/deberta.html>`__ (from Microsoft Research) released
-   with the paper `DeBERTa: Decoding-enhanced BERT with Disentangled Attention <https://arxiv.org/abs/2006.03654>`__ by
+3. `BERT <https://huggingface.co/transformers/model_doc/bert.html>`__ (from Google) released with the paper `BERT:
+   Pre-training of Deep Bidirectional Transformers for Language Understanding <https://arxiv.org/abs/1810.04805>`__ by
+   Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova.
+4. `BERT For Sequence Generation <https://tfhub.dev/s?module-type=text-generation&subtype=module,placeholder>`__ (from
+   Google) released with the paper `Leveraging Pre-trained Checkpoints for Sequence Generation Tasks
+   <https://arxiv.org/abs/1907.12461>`__ by Sascha Rothe, Shashi Narayan, Aliaksei Severyn.
+5. `CamemBERT <https://huggingface.co/transformers/model_doc/camembert.html>`__ (from Inria/Facebook/Sorbonne) released
+   with the paper `CamemBERT: a Tasty French Language Model <https://arxiv.org/abs/1911.03894>`__ by Louis Martin*,
+   Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé
+   Seddah and Benoît Sagot.
+6. `CTRL <https://huggingface.co/transformers/model_doc/ctrl.html>`__ (from Salesforce) released with the paper `CTRL:
+   A Conditional Transformer Language Model for Controllable Generation <https://arxiv.org/abs/1909.05858>`__ by Nitish
+   Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher.
+7. `DeBERTa <https://huggingface.co/transformers/model_doc/deberta.html>`__ (from Microsoft Research) released with the
+   paper `DeBERTa: Decoding-enhanced BERT with Disentangled Attention <https://arxiv.org/abs/2006.03654>`__ by
   Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen.
-9. :doc:`DialoGPT <model_doc/dialogpt>` (from Microsoft Research) released with the paper `DialoGPT: Large-Scale
-   Generative Pre-training for Conversational Response Generation <https://arxiv.org/abs/1911.00536>`__ by Yizhe Zhang,
-   Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan.
-10. :doc:`DistilBERT <model_doc/distilbert>` (from HuggingFace), released together with the paper `DistilBERT, a
-    distilled version of BERT: smaller, faster, cheaper and lighter <https://arxiv.org/abs/1910.01108>`__ by Victor
-    Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into `DistilGPT2
-    <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__, RoBERTa into `DistilRoBERTa
-    <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__, Multilingual BERT into
-    `DistilmBERT <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German
-    version of DistilBERT.
-11. :doc:`DPR <model_doc/dpr>` (from Facebook) released with the paper `Dense Passage Retrieval for Open-Domain
-    Question Answering <https://arxiv.org/abs/2004.04906>`__ by Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick
-    Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
-12. :doc:`ELECTRA <model_doc/electra>` (from Google Research/Stanford University) released with the paper `ELECTRA:
-    Pre-training text encoders as discriminators rather than generators <https://arxiv.org/abs/2003.10555>`__ by Kevin
-    Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning.
-13. :doc:`FlauBERT <model_doc/flaubert>` (from CNRS) released with the paper `FlauBERT: Unsupervised Language Model
-    Pre-training for French <https://arxiv.org/abs/1912.05372>`__ by Hang Le, Loïc Vial, Jibril Frej, Vincent Segonne,
-    Maximin Coavoux, Benjamin Lecouteux, Alexandre Allauzen, Benoît Crabbé, Laurent Besacier, Didier Schwab.
-14. :doc:`Funnel Transformer <model_doc/funnel>` (from CMU/Google Brain) released with the paper `Funnel-Transformer:
-    Filtering out Sequential Redundancy for Efficient Language Processing <https://arxiv.org/abs/2006.03236>`__ by
-    Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
-15. :doc:`GPT <model_doc/gpt>` (from OpenAI) released with the paper `Improving Language Understanding by Generative
-    Pre-Training <https://blog.openai.com/language-unsupervised/>`__ by Alec Radford, Karthik Narasimhan, Tim Salimans
-    and Ilya Sutskever.
-16. :doc:`GPT-2 <model_doc/gpt2>` (from OpenAI) released with the paper `Language Models are Unsupervised Multitask
-    Learners <https://blog.openai.com/better-language-models/>`__ by Alec Radford*, Jeffrey Wu*, Rewon Child, David
-    Luan, Dario Amodei** and Ilya Sutskever**.
-17. :doc:`LayoutLM <model_doc/layoutlm>` (from Microsoft Research Asia) released with the paper `LayoutLM: Pre-training
-    of Text and Layout for Document Image Understanding <https://arxiv.org/abs/1912.13318>`__ by Yiheng Xu, Minghao Li,
-    Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou.
-18. :doc:`Longformer <model_doc/longformer>` (from AllenAI) released with the paper `Longformer: The Long-Document
-    Transformer <https://arxiv.org/abs/2004.05150>`__ by Iz Beltagy, Matthew E. Peters, Arman Cohan.
-19. :doc:`LXMERT <model_doc/lxmert>` (from UNC Chapel Hill) released with the paper `LXMERT: Learning Cross-Modality
-    Encoder Representations from Transformers for Open-Domain Question Answering <https://arxiv.org/abs/1908.07490>`__
-    by Hao Tan and Mohit Bansal.
-20. :doc:`MarianMT <model_doc/marian>` Machine translation models trained using `OPUS <http://opus.nlpl.eu/>`__ data by
-    Jörg Tiedemann. The `Marian Framework <https://marian-nmt.github.io/>`__ is being developed by the Microsoft
-    Translator Team.
-21. :doc:`MBart <model_doc/mbart>` (from Facebook) released with the paper  `Multilingual Denoising Pre-training for
-    Neural Machine Translation <https://arxiv.org/abs/2001.08210>`__ by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li,
-    Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-22. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
-    Gap-sentences for Abstractive Summarization <https://arxiv.org/abs/1912.08777>`__> by Jingqing Zhang, Yao Zhao,
-    Mohammad Saleh and Peter J. Liu.
-23. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
-    Transformer <https://arxiv.org/abs/2001.04451>`__ by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
-24. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
-    Pretraining Approach <https://arxiv.org/abs/1907.11692>`__ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar
-    Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. ultilingual BERT into `DistilmBERT
+8. `DialoGPT <https://huggingface.co/transformers/model_doc/dialogpt.html>`__ (from Microsoft Research) released with
+   the paper `DialoGPT: Large-Scale Generative Pre-training for Conversational Response Generation
+   <https://arxiv.org/abs/1911.00536>`__ by Yizhe Zhang, Siqi Sun, Michel Galley, Yen-Chun Chen, Chris Brockett, Xiang
+   Gao, Jianfeng Gao, Jingjing Liu, Bill Dolan.
+9. `DistilBERT <https://huggingface.co/transformers/model_doc/distilbert.html>`__ (from HuggingFace), released together
+   with the paper `DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter
+   <https://arxiv.org/abs/1910.01108>`__ by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been
+   applied to compress GPT2 into `DistilGPT2
+   <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__, RoBERTa into `DistilRoBERTa
+   <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__, Multilingual BERT into
+   `DistilmBERT <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German version
+   of DistilBERT.
+10. `DPR <https://github.com/facebookresearch/DPR>`__ (from Facebook) released with the paper `Dense Passage Retrieval
+    for Open-Domain Question Answering <https://arxiv.org/abs/2004.04906>`__ by Vladimir Karpukhin, Barlas Oğuz, Sewon
+    Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
+11. `ELECTRA <https://huggingface.co/transformers/model_doc/electra.html>`__ (from Google Research/Stanford University)
+    released with the paper `ELECTRA: Pre-training text encoders as discriminators rather than generators
+    <https://arxiv.org/abs/2003.10555>`__ by Kevin Clark, Minh-Thang Luong, Quoc V. Le, Christopher D. Manning.
+12. `FlauBERT <https://huggingface.co/transformers/model_doc/flaubert.html>`__ (from CNRS) released with the paper
+    `FlauBERT: Unsupervised Language Model Pre-training for French <https://arxiv.org/abs/1912.05372>`__ by Hang Le,
+    Loïc Vial, Jibril Frej, Vincent Segonne, Maximin Coavoux, Benjamin Lecouteux, Alexandre Allauzen, Benoît Crabbé,
+    Laurent Besacier, Didier Schwab.
+13. `Funnel Transformer <https://github.com/laiguokun/Funnel-Transformer>`__ (from CMU/Google Brain) released with the
+    paper `Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing
+    <https://arxiv.org/abs/2006.03236>`__ by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
+14. `GPT <https://huggingface.co/transformers/model_doc/gpt.html>`__ (from OpenAI) released with the paper `Improving
+    Language Understanding by Generative Pre-Training <https://blog.openai.com/language-unsupervised/>`__ by Alec
+    Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever.
+15. `GPT-2 <https://huggingface.co/transformers/model_doc/gpt2.html>`__ (from OpenAI) released with the paper `Language
+    Models are Unsupervised Multitask Learners <https://blog.openai.com/better-language-models/>`__ by Alec Radford*,
+    Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**.
+16. `LayoutLM <https://github.com/microsoft/unilm/tree/master/layoutlm>`__ (from Microsoft Research Asia) released with
+    the paper `LayoutLM: Pre-training of Text and Layout for Document Image Understanding
+    <https://arxiv.org/abs/1912.13318>`__ by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, Ming Zhou.
+17. `Longformer <https://huggingface.co/transformers/model_doc/longformer.html>`__ (from AllenAI) released with the
+    paper `Longformer: The Long-Document Transformer <https://arxiv.org/abs/2004.05150>`__ by Iz Beltagy, Matthew E.
+    Peters, Arman Cohan.
+18. `LXMERT <https://github.com/airsplay/lxmert>`__ (from UNC Chapel Hill) released with the paper `LXMERT: Learning
+    Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering
+    <https://arxiv.org/abs/1908.07490>`__ by Hao Tan and Mohit Bansal.
+19. `MarianMT <https://huggingface.co/transformers/model_doc/marian.html>`__ Machine translation models trained using
+    `OPUS <http://opus.nlpl.eu/>`__ data by Jörg Tiedemann. The `Marian Framework <https://marian-nmt.github.io/>`__ is
+    being developed by the Microsoft Translator Team.
+20. `MBart <https://github.com/pytorch/fairseq/tree/master/examples/mbart>`__ (from Facebook) released with the paper 
+    `Multilingual Denoising Pre-training for Neural Machine Translation <https://arxiv.org/abs/2001.08210>`__ by Yinhan
+    Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
+21. `MMBT <https://github.com/facebookresearch/mmbt/>`__ (from Facebook), released together with the paper a
+    `Supervised Multimodal Bitransformers for Classifying Images and Text <https://arxiv.org/pdf/1909.02950.pdf>`__ by
+    Douwe Kiela, Suvrat Bhooshan, Hamed Firooz, Davide Testuggine.
+22. `Pegasus <https://github.com/google-research/pegasus>`__ (from Google) released with the paper `PEGASUS:
+    Pre-training with Extracted Gap-sentences for Abstractive Summarization <https://arxiv.org/abs/1912.08777>`__> by
+    Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
+23. `ProphetNet <https://github.com/microsoft/ProphetNet>`__ (from Microsoft Research) released with the paper
+    `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__
+    by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
+24. `Reformer <https://huggingface.co/transformers/model_doc/reformer.html>`__ (from Google Research) released with the
+    paper `Reformer: The Efficient Transformer <https://arxiv.org/abs/2001.04451>`__ by Nikita Kitaev, Łukasz Kaiser,
+    Anselm Levskaya.
+25. `RoBERTa <https://huggingface.co/transformers/model_doc/roberta.html>`__ (from Facebook), released together with
+    the paper a `Robustly Optimized BERT Pretraining Approach <https://arxiv.org/abs/1907.11692>`__ by Yinhan Liu, Myle
+    Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov.
+    ultilingual BERT into `DistilmBERT
    <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German version of
    DistilBERT.
-25. `SqueezeBert <https://huggingface.co/transformers/master/model_doc/squeezebert.html>`__ released with the paper
+26. `SqueezeBert <https://huggingface.co/transformers/model_doc/squeezebert.html>`__ released with the paper
    `SqueezeBERT: What can computer vision teach NLP about efficient neural networks?
    <https://arxiv.org/abs/2006.11316>`__ by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer.
-26. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
-    Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by Colin Raffel and Noam Shazeer and Adam
-    Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
-27. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
-    Attentive Language Models Beyond a Fixed-Length Context <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*,
-    Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
-28. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
-    Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis Conneau.
-29. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
-    Cross-lingual Representation Learning at Scale <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay
-    Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke
-    Zettlemoyer and Veselin Stoyanov.
-30. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `XLNet: Generalized Autoregressive
-    Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by Zhilin Yang*, Zihang Dai*, Yiming
-    Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
-31. `Other community models <https://huggingface.co/models>`__, contributed by the `community
+27. `T5 <https://huggingface.co/transformers/model_doc/t5.html>`__ (from Google AI) released with the paper `Exploring
+    the Limits of Transfer Learning with a Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by
+    Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi
+    Zhou and Wei Li and Peter J. Liu.
+28. `Transformer-XL <https://huggingface.co/transformers/model_doc/transformerxl.html>`__ (from Google/CMU) released
+    with the paper `Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context
+    <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le,
+    Ruslan Salakhutdinov.
+29. `XLM <https://huggingface.co/transformers/model_doc/xlm.html>`__ (from Facebook) released together with the paper
+    `Cross-lingual Language Model Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis
+    Conneau.
+30. `XLMProphetNet <https://github.com/microsoft/ProphetNet>`__ (from Microsoft Research) released with the paper
+    `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__
+    by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
+31. `XLM-RoBERTa <https://huggingface.co/transformers/model_doc/xlmroberta.html>`__ (from Facebook AI), released
+    together with the paper `Unsupervised Cross-lingual Representation Learning at Scale
+    <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary,
+    Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov.
+32. `XLNet <https://huggingface.co/transformers/model_doc/xlnet.html>`__ (from Google/CMU) released with the paper
+    `XLNet: Generalized Autoregressive Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by
+    Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
+33. `Other community models <https://huggingface.co/models>`__, contributed by the `community
    <https://huggingface.co/users>`__.

 .. toctree::
@@ -19,9 +19,3 @@ Callbacks internals
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.trainer_callback.CallbackHandler
-
-Distributed Evaluation
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.trainer_pt_utils.DistributedTensorGatherer
-    :members:
@@ -12,7 +12,7 @@ subclass :class:`~transformers.Trainer` and override the methods you need (see :

 By default a :class:`~transformers.Trainer` will use the following callbacks:

- :class:`~transformers.DefaultFlowCallback` which handles the default behavior for logging, saving and evaluation.
+- :class:`~transformers.DefaultFlowCallback` which handles the default beahvior for logging, saving and evaluation.
 - :class:`~transformers.PrinterCallback` or :class:`~transformers.ProrgressCallback` to display progress and print the
  logs (the first one is used if you deactivate tqdm through the :class:`~transformers.TrainingArguments`, otherwise
  it's the second one).
@@ -15,7 +15,7 @@ Both :class:`~transformers.Trainer` and :class:`~transformers.TFTrainer` contain
 previous features. To inject custom behavior you can subclass them and override the following methods:

 - **get_train_dataloader**/**get_train_tfdataset** -- Creates the training DataLoader (PyTorch) or TF Dataset.
- **get_eval_dataloader**/**get_eval_tfdataset** -- Creates the evaluation DataLoader (PyTorch) or TF Dataset.
+- **get_eval_dataloader**/**get_eval_tfdataset** -- Creates the evaulation DataLoader (PyTorch) or TF Dataset.
 - **get_test_dataloader**/**get_test_tfdataset** -- Creates the test DataLoader (PyTorch) or TF Dataset.
 - **log** -- Logs information on the various objects watching training.
 - **create_optimizer_and_scheduler** -- Setups the optimizer and learning rate scheduler if they were not passed at
@@ -1,5 +1,5 @@
 Blenderbot
-----------------------------------------------------------------------------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 **DISCLAIMER:** If you see something strange,
 file a `Github Issue <https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ .

@@ -104,13 +104,6 @@ OpenAIGPTDoubleHeadsModel
    :members: forward


-OpenAIGPTForSequenceClassification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.OpenAIGPTForSequenceClassification
-    :members: forward
-
-
 TFOpenAIGPTModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

@@ -66,7 +66,7 @@ The library is built around three types of classes for each model:
 All these classes can be instantiated from pretrained instances and saved locally using two methods:

 - :obj:`from_pretrained()` lets you instantiate a model/configuration/tokenizer from a pretrained version either
-  provided by the library itself (the supported models are provided in the list :doc:`here <pretrained_models>`
+  provided by the library itself (the suported models are provided in the list :doc:`here <pretrained_models>`
  or stored locally (or on a server) by the user,
 - :obj:`save_pretrained()` lets you save a model/configuration/tokenizer locally so that it can be reloaded using
  :obj:`from_pretrained()`.
@@ -294,10 +294,10 @@ For a list that includes community-uploaded models, refer to `https://huggingfac
 |                    | ``t5-11B``                                                 | | ~11B parameters with 24-layers, 1024-hidden-state, 65536 feed-forward hidden-state, 128-heads,                                      |
 |                    |                                                            | | Trained on English text: the Colossal Clean Crawled Corpus (C4)                                                                     |
 +--------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
-| XLM-RoBERTa        | ``xlm-roberta-base``                                       | | ~270M parameters with 12-layers, 768-hidden-state, 3072 feed-forward hidden-state, 8-heads,                                         |
+| XLM-RoBERTa        | ``xlm-roberta-base``                                       | | ~125M parameters with 12-layers, 768-hidden-state, 3072 feed-forward hidden-state, 8-heads,                                         |
 |                    |                                                            | | Trained on on 2.5 TB of newly created clean CommonCrawl data in 100 languages                                                       |
 |                    +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
-|                    | ``xlm-roberta-large``                                      | | ~550M parameters with 24-layers, 1024-hidden-state, 4096 feed-forward hidden-state, 16-heads,                                       |
+|                    | ``xlm-roberta-large``                                      | | ~355M parameters with 24-layers, 1027-hidden-state, 4096 feed-forward hidden-state, 16-heads,                                       |
 |                    |                                                            | | Trained on 2.5 TB of newly created clean CommonCrawl data in 100 languages                                                          |
 +--------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 | FlauBERT           | ``flaubert/flaubert_small_cased``                          | | 6-layer, 512-hidden, 8-heads, 54M parameters                                                                                        |
@@ -24,11 +24,8 @@ import logging
 import math
 import os
 from dataclasses import dataclass, field
-from glob import glob
 from typing import Optional

-from torch.utils.data import ConcatDataset
-
 from transformers import (
    CONFIG_MAPPING,
    MODEL_WITH_LM_HEAD_MAPPING,
@@ -90,13 +87,6 @@ class DataTrainingArguments:
    train_data_file: Optional[str] = field(
        default=None, metadata={"help": "The input training data file (a text file)."}
    )
-    train_data_files: Optional[str] = field(
-        default=None,
-        metadata={
-            "help": "The input training data files (multiple files in glob format). "
-            "Very often splitting large files to smaller files can prevent tokenizer going out of memory"
-        },
-    )
    eval_data_file: Optional[str] = field(
        default=None,
        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
@@ -141,24 +131,17 @@ def get_dataset(
    evaluate: bool = False,
    cache_dir: Optional[str] = None,
 ):
-    def _dataset(file_path):
-        if args.line_by_line:
-            return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
-        else:
-            return TextDataset(
-                tokenizer=tokenizer,
-                file_path=file_path,
-                block_size=args.block_size,
-                overwrite_cache=args.overwrite_cache,
-                cache_dir=cache_dir,
-            )
-
-    if evaluate:
-        return _dataset(args.eval_data_file)
-    elif args.train_data_files:
-        return ConcatDataset([_dataset(f) for f in glob(args.train_data_files)])
+    file_path = args.eval_data_file if evaluate else args.train_data_file
+    if args.line_by_line:
+        return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
    else:
-        return _dataset(args.train_data_file)
+        return TextDataset(
+            tokenizer=tokenizer,
+            file_path=file_path,
+            block_size=args.block_size,
+            overwrite_cache=args.overwrite_cache,
+            cache_dir=cache_dir,
+        )


 def main():
@@ -119,7 +119,7 @@ class BaseTransformer(pl.LightningModule):
    def get_lr_scheduler(self):
        get_schedule_func = arg_to_scheduler[self.hparams.lr_scheduler]
        scheduler = get_schedule_func(
-            self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=self.total_steps()
+            self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=self.total_steps
        )
        scheduler = {"scheduler": scheduler, "interval": "step", "frequency": 1}
        return scheduler
@@ -159,20 +159,19 @@ class BaseTransformer(pl.LightningModule):
    def test_epoch_end(self, outputs):
        return self.validation_end(outputs)

+    @property
    def total_steps(self) -> int:
        """The number of total training steps that will be run. Used for lr scheduler purposes."""
        num_devices = max(1, self.hparams.gpus)  # TODO: consider num_tpu_cores
        effective_batch_size = self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
-        return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs
+        dataset_size = len(self.train_loader.dataset)
+        return (dataset_size / effective_batch_size) * self.hparams.max_epochs

    def setup(self, mode):
-        if mode == "test":
-            self.dataset_size = len(self.test_dataloader().dataset)
-        else:
+        if mode == "fit":
            self.train_loader = self.get_dataloader("train", self.hparams.train_batch_size, shuffle=True)
-            self.dataset_size = len(self.train_loader.dataset)

-    def get_dataloader(self, type_path: str, batch_size: int, shuffle: bool = False):
+    def get_dataloader(self, type_path, batch_size, shuffle=False):
        raise NotImplementedError("You must implement this for your task")

    def train_dataloader(self):
@@ -65,41 +65,26 @@ Does He Love You	Does He Love You	Red Sandy Spika dress of Reba McEntire	Greates
 We demonstrate how to evaluate retrieval against DPR evaluation data. You can download respective files from links listed [here](https://github.com/facebookresearch/DPR/blob/master/data/download_data.py#L39-L45).

 1. Download and unzip the gold data file. We use the `biencoder-nq-dev` from https://dl.fbaipublicfiles.com/dpr/data/retriever/biencoder-nq-dev.json.gz.
-    ```bash
-    wget https://dl.fbaipublicfiles.com/dpr/data/retriever/biencoder-nq-dev.json.gz && gzip -d biencoder-nq-dev.json.gz
-   ```
-
 2. Parse the unziped file using the `parse_dpr_relevance_data.py`
    ```bash
-    mkdir output # or wherever you want to save this
    python examples/rag/parse_dpr_relevance_data.py \
-        --src_path biencoder-nq-dev.json \
-        --evaluation_set output/biencoder-nq-dev.questions \
-        --gold_data_path output/biencoder-nq-dev.pages
+        --src_path path/to/unziped/biencoder-nq-dev.json \
+        --evaluation_set path/to/output/biencoder-nq-dev.questions \
+        --gold_data_path path/to/output/biencoder-nq-dev.pages
    ```
 3. Run evaluation:
-    ```bash    
-    python examples/rag/eval_rag.py \
-        --model_name_or_path facebook/rag-sequence-nq \
-        --model_type rag_sequence \
-        --evaluation_set output/biencoder-nq-dev.questions \
-        --gold_data_path output/biencoder-nq-dev.pages \
-        --predictions_path output/retrieval_preds.tsv  \
-        --eval_mode retrieval \
-        --k 1
-    ```
-   ```bash
-   # EXPLANATION
+    ```bash
    python examples/rag/eval_rag.py \
        --model_name_or_path facebook/rag-sequence-nq \ # model name or path of the model we're evaluating
        --model_type rag_sequence \ # RAG model type (rag_token or rag_sequence)
-        --evaluation_set output/biencoder-nq-dev.questions \ # an input dataset for evaluation
-        --gold_data_path poutput/biencoder-nq-dev.pages \ # a dataset containing ground truth answers for samples from the evaluation_set
-        --predictions_path output/retrieval_preds.tsv  \ # name of file where predictions will be stored
+        --evaluation_set path/to/output/biencoder-nq-dev.questions \ # an input dataset for evaluation
+        --gold_data_path path/to/output/biencoder-nq-dev.pages \ # a dataset containing ground truth answers for samples from the evaluation_set
+        --predictions_path path/to/retrieval_preds.tsv  \ # name of file where predictions will be stored
        --eval_mode retrieval \ # indicates whether we're performing retrieval evaluation or e2e evaluation
        --k 1 # parameter k for the precision@k metric
-   
    ```
+
+
 ## End-to-end evaluation

 We support two formats of the gold data file (controlled by the `gold_data_mode` parameter):
@@ -112,9 +97,7 @@ who is the owner of reading football club	['Xiu Li Dai', 'Dai Yongge', 'Dai Xiul
 Xiu Li Dai
 ```

-Predictions of the model for the samples from the `evaluation_set` will be saved under the path specified by the `predictions_path` parameter. 
-If this path already exists, the script will use saved predictions to calculate metrics. 
-Add `--recalculate` parameter to force the script to perform inference from scratch.
+Predictions of the model for the samples from the `evaluation_set` will be saved under the path specified by the `predictions_path` parameter. If this path already exists, the script will use saved predictions to calculate metrics. Add `--recalculate` parameter to force the script to perform inference from scratch.

 An example e2e evaluation run could look as follows:
 ```bash
@@ -1,5 +0,0 @@
-import os
-import sys
-
-
-sys.path.insert(1, os.path.dirname(os.path.realpath(__file__)))
@@ -15,7 +15,7 @@ from transformers import logging as transformers_logging


 sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # isort:skip
-from utils import exact_match_score, f1_score  # noqa: E402 # isort:skip
+from examples.rag.utils import exact_match_score, f1_score  # noqa: E402 # isort:skip


 logger = logging.getLogger(__name__)
@@ -31,13 +31,16 @@ from transformers import (
 from transformers import logging as transformers_logging


-from callbacks import (  # noqa: E402 # isort:skipq
+sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip
+
+from examples.lightning_base import BaseTransformer, add_generic_args, generic_train  # noqa: E402 # isort:skip
+from examples.rag.callbacks import (  # noqa: E402 # isort:skip
    get_checkpoint_callback,
    get_early_stopping_callback,
    Seq2SeqLoggingCallback,
 )
-from distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
-from utils import (  # noqa: E402 # isort:skip
+from examples.rag.distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
+from examples.rag.utils import (  # noqa: E402 # isort:skip
    calculate_exact_match,
    flatten_list,
    get_git_info,
@@ -50,11 +53,6 @@ from utils import (  # noqa: E402 # isort:skip
    Seq2SeqDataset,
 )

-# need the parent dir module
-sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
-from lightning_base import BaseTransformer, add_generic_args, generic_train  # noqa
-
-
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)

@@ -23,7 +23,7 @@ from transformers.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FI

 sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip

-from distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip
+from examples.rag.distributed_retriever import RagPyTorchDistributedRetriever  # noqa: E402 # isort:skip


 def require_distributed_retrieval(test_case):
@@ -5,7 +5,7 @@ psutil
 sacrebleu
 rouge-score
 tensorflow_datasets
-pytorch-lightning==0.9.0
+pytorch-lightning==0.8.5
 matplotlib
 git-python==1.0.3
 faiss-cpu
@@ -12,13 +12,14 @@ For `bertabs` instructions, see [`bertabs/README.md`](bertabs/README.md).
 - `MBartForConditionalGeneration`
 - `FSMTForConditionalGeneration`
 - `T5ForConditionalGeneration`
+    

 ## Datasets

 #### XSUM:
 ```bash
 cd examples/seq2seq
-wget https://cdn-datasets.huggingface.co/summarization/xsum.tar.gz
+wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/xsum.tar.gz
 tar -xzvf xsum.tar.gz
 export XSUM_DIR=${PWD}/xsum
 ```
@@ -28,7 +29,7 @@ To use your own data, copy that files format. Each article to be summarized is o
 #### CNN/DailyMail
 ```bash
 cd examples/seq2seq
-wget https://cdn-datasets.huggingface.co/summarization/cnn_dm_v2.tgz
+wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/cnn_dm_v2.tgz
 tar -xzvf cnn_dm_v2.tgz  # empty lines removed
 mv cnn_cln cnn_dm
 export CNN_DIR=${PWD}/cnn_dm
@@ -38,7 +39,7 @@ this should make a directory called `cnn_dm/` with 6 files.
 #### WMT16 English-Romanian Translation Data:
 download with this command:
 ```bash
-wget https://cdn-datasets.huggingface.co/translation/wmt_en_ro.tar.gz
+wget https://s3.amazonaws.com/datasets.huggingface.co/translation/wmt_en_ro.tar.gz
 tar -xzvf wmt_en_ro.tar.gz
 export ENRO_DIR=${PWD}/wmt_en_ro
 ```
@@ -46,7 +47,7 @@ this should make a directory called `wmt_en_ro/` with 6 files.

 #### WMT English-German:
 ```bash
-wget https://cdn-datasets.huggingface.co/translation/wmt_en_de.tgz
+wget https://s3.amazonaws.com/datasets.huggingface.co/translation/wmt_en_de.tgz
 tar -xzvf wmt_en_de.tgz
 export DATA_DIR=${PWD}/wmt_en_de
 ```
@@ -99,7 +100,7 @@ All finetuning bash scripts call finetune.py (or distillation.py) with reasonabl
 To see all the possible command line options, run:

 ```bash
-./finetune.py --help
+ ./finetune.py --help 
 ```

 ### Finetuning Training Params
@@ -191,7 +192,7 @@ model = AutoModelForSeq2SeqLM.from_pretrained(f'{output_dir}/best_tfmr')
 ### Fine-tuning using Seq2SeqTrainer
 To use `Seq2SeqTrainer` for fine-tuning you should use the `finetune_trainer.py` script. It subclasses `Trainer` to extend it for seq2seq training. Except the `Trainer` releated `TrainingArguments`, it shares the same argument names as that of `finetune.py` file. One notable difference is that, calculating generative metrics (BLEU, ROUGE) is optional and is controlled using the `--predict_with_generate` argument, set this argument to calculate BLEU and ROUGE metrics.

-With PyTorch 1.6+ it'll automatically use `native AMP` when `--fp16` is set.
+With PyTorch 1.6+ it'll automatically use `native AMP` when `--fp16` is set. 

 To see all the possible command line options, run:

@@ -264,7 +265,6 @@ export DATA_DIR=cnn_dm
    --fp16 \
    --bs 32
 ```
-
 ### Multi-GPU Evaluation
 here is a command to run xsum evaluation on 8 GPUS. It is more than linearly faster than run_eval.py in some cases 
 because it uses SortishSampler to minimize padding. You can also use it on 1 GPU. `data_dir` must have 
@@ -391,17 +391,6 @@ runtime: 13H on V-100 16GB GPU.
 pytest examples/seq2seq/
 ```

-### Converting pytorch-lightning checkpoints
-pytorch lightning ``-do_predict`` often fails, after you are done training, the best way to evaluate your model is to convert it.
-
-This should be done for you, with a file called `{save_dir}/best_tfmr`. 
-
-If that file doesn't exist but you have a lightning `.ckpt` file, you can run
-```bash
-python convert_pl_checkpoint_to_hf.py PATH_TO_CKPT  randomly_initialized_hf_model_path save_dir/best_tfmr
-```
-Then either `run_eval` or `run_distributed_eval` with `save_dir/best_tfmr` (see previous sections)
-

 ## Experimental Features 
 These features are harder to use and not always useful.
@@ -430,3 +419,4 @@ uses 12,723 batches of length 48 and takes slightly more time 9.5 minutes.
 The feature is still experimental, because:
 + we can make it much more robust if we have memory mapped/preprocessed datasets.
 + The speedup over sortish sampler is not that large at the moment.
+
@@ -39,7 +39,7 @@ python run_summarization.py \
    --compute_rouge true
 ```

-The scripts executes on GPU if one is available and if `no_cuda` is not set to `true`. Inference on multiple GPUs is not supported yet. The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. The script takes 30 hours to compute with a single Tesla V100 GPU and a batch size of 10 (300,000 texts to summarize).
+The scripts executes on GPU if one is available and if `no_cuda` is not set to `true`. Inference on multiple GPUs is not suported yet. The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. The script takes 30 hours to compute with a single Tesla V100 GPU and a batch size of 10 (300,000 texts to summarize).

 ## Summarize any text

@@ -17,7 +17,7 @@ from finetune import main as ft_main
 from make_student import create_student_by_copying_alternating_layers, get_layers_to_supervise
 from transformers import AutoModelForSeq2SeqLM, MBartTokenizer, T5ForConditionalGeneration
 from transformers.modeling_bart import shift_tokens_right
-from utils import calculate_bleu, freeze_params, label_smoothed_nll_loss, use_task_specific_params
+from utils import calculate_bleu, freeze_params, label_smoothed_nll_loss, pickle_load, use_task_specific_params


 # need the parent dir module
@@ -264,6 +264,30 @@ def create_module(args):
    return model


+def evaluate_checkpoint(ckpt_path: Path, dest_dir=None):
+    # TODO(SS): DELETE? Better to convert_pl_ckpt_to_hf and run_eval.py
+    exp_dir = ckpt_path.parent
+    if dest_dir is None:
+        dest_dir = exp_dir
+    clash = list(dest_dir.glob("test_generations*"))
+    if clash:
+        print(f"SKIPPING to avoid overwriting {clash}")
+    ckpt = torch.load(ckpt_path, map_location="cpu")
+    if "hparams" in ckpt:
+        args = argparse.Namespace(**ckpt["hparams"])
+    else:
+        args = argparse.Namespace(**pickle_load(exp_dir / "hparams.pkl"))
+    args.resume_from_checkpoint = str(ckpt_path)
+    args.do_train = False
+    args.output_dir = str(dest_dir)
+    args.n_gpu = 1
+    args.eval_batch_size = 16
+    Path(args.output_dir).mkdir(exist_ok=True)
+    model = create_module(args)
+    trainer: pl.Trainer = generic_train(model, args, early_stopping_callback=False)
+    trainer.test(model)
+
+
 def distill_main(args):
    Path(args.output_dir).mkdir(exist_ok=True)
    if len(os.listdir(args.output_dir)) > 3 and args.do_train:
@@ -181,7 +181,6 @@ class SummarizationModule(BaseTransformer):
        return self._generative_step(batch)

    def validation_epoch_end(self, outputs, prefix="val") -> Dict:
-
        self.step_count += 1
        losses = {k: torch.stack([x[k] for x in outputs]).mean() for k in self.loss_names}
        loss = losses["loss"]
@@ -1,7 +1,7 @@
 # Script for verifying that run_bart_sum can be invoked from its directory

 # Get tiny dataset with cnn_dm format (4 examples for train, val, test)
-wget https://cdn-datasets.huggingface.co/summarization/cnn_tiny.tgz
+wget https://s3.amazonaws.com/datasets.huggingface.co/summarization/cnn_tiny.tgz
 tar -xzvf cnn_tiny.tgz
 rm cnn_tiny.tgz

@@ -4,7 +4,7 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional

-from seq2seq_trainer import Seq2SeqTrainer, arg_to_scheduler_choices
+from seq2seq_trainer import Seq2SeqTrainer
 from transformers import (
    AutoConfig,
    AutoModelForSeq2SeqLM,
@@ -63,9 +63,6 @@ class Seq2SeqTrainingArguments(TrainingArguments):
    attention_dropout: Optional[float] = field(
        default=None, metadata={"help": "Attention dropout probability. Goes into model.config."}
    )
-    lr_scheduler: Optional[str] = field(
-        default="linear", metadata={"help": f"Which lr scheduler to use. Selected in {arg_to_scheduler_choices}"}
-    )


@dataclass
@@ -4,24 +4,24 @@ These are the generations of various large models on various large **training**
 ### Available Pseudo-labels
 | Dataset | Model                       | Link                                                                                   | Rouge Scores       | Notes                                                                                                       
 |---------|-----------------------------|----------------------------------------------------------------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------
-| XSUM    | `facebook/bart-large-xsum`    | [download](https://cdn-datasets.huggingface.co/pseudo/xsum/bart_xsum_pl.tgz)          | 49.8/28.0/42.5     |                                                                                                             
-| XSUM    | `google/pegasus-xsum`         | [download](https://cdn-datasets.huggingface.co/pseudo/xsum/pegasus_xsum.tgz)          | 53.3/32.7/46.5     |                                                                                                             
-| XSUM    | `facebook/bart-large-xsum`    | [download](https://cdn-datasets.huggingface.co/pseudo/xsum/xsum_pl2_bart.tgz)         |                   | Bart pseudolabels filtered to those with Rouge2 > 10.0 w GT.                                                 
-| CNN/DM  | `sshleifer/pegasus-cnn-ft-v2` | [download](https://cdn-datasets.huggingface.co/pseudo/cnn_dm/pegasus_cnn_cnn_pls.tgz) | 47.316/26.65/44.56 | do not worry about the fact that train.source is one line shorter.                                          
-| CNN/DM  | `facebook/bart-large-cnn`     | [download](https://cdn-datasets.huggingface.co/pseudo/cnn_dm/cnn_bart_pl.tgz)         |                    | 5K (2%) are missing, there should be 282173                                                                 
-| CNN/DM  | `google/pegasus-xsum`         | [download](https://cdn-datasets.huggingface.co/pseudo/cnn_dm/pegasus_xsum_on_cnn.tgz) | 21.5/6.76/25       | extra labels for xsum distillation  Used max_source_length=512, (and all other pegasus-xsum configuration). 
-| EN-RO   | `Helsinki-NLP/opus-mt-en-ro`  | [download](https://cdn-datasets.huggingface.co/pseudo/wmt_en_ro/opus_mt_en_ro.tgz) |       |  
-| EN-RO   | `facebook/mbart-large-en-ro`  | [download](https://cdn-datasets.huggingface.co/pseudo/wmt_en_ro/mbart_large_en_ro.tgz) |       |  
+| XSUM    | `facebook/bart-large-xsum`    | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/xsum/bart_xsum_pl.tgz)          | 49.8/28.0/42.5     |                                                                                                             
+| XSUM    | `google/pegasus-xsum`         | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/xsum/pegasus_xsum.tgz)          | 53.3/32.7/46.5     |                                                                                                             
+| XSUM    | `facebook/bart-large-xsum`    | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/xsum/xsum_pl2_bart.tgz)         |                   | Bart pseudolabels filtered to those with Rouge2 > 10.0 w GT.                                                 
+| CNN/DM  | `sshleifer/pegasus-cnn-ft-v2` | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/cnn_dm/pegasus_cnn_cnn_pls.tgz) | 47.316/26.65/44.56 | do not worry about the fact that train.source is one line shorter.                                          
+| CNN/DM  | `facebook/bart-large-cnn`     | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/cnn_dm/cnn_bart_pl.tgz)         |                    | 5K (2%) are missing, there should be 282173                                                                 
+| CNN/DM  | `google/pegasus-xsum`         | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/cnn_dm/pegasus_xsum_on_cnn.tgz) | 21.5/6.76/25       | extra labels for xsum distillation  Used max_source_length=512, (and all other pegasus-xsum configuration). 
+| EN-RO   | `Helsinki-NLP/opus-mt-en-ro`  | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/wmt_en_ro/opus_mt_en_ro.tgz) |       |  
+| EN-RO   | `facebook/mbart-large-en-ro`  | [download](https://s3.amazonaws.com/datasets.huggingface.co/pseudo/wmt_en_ro/mbart_large_en_ro.tgz) |       |  


 (EN_RO = WMT 2016 English-Romanian).

 Example Download Command:
 ```bash
-curl -S https://cdn-datasets.huggingface.co/pseudo/xsum/bart_xsum_pl.tgz | tar -xvz -C .
+curl -S https://s3.amazonaws.com/datasets.huggingface.co/pseudo/xsum/bart_xsum_pl.tgz | tar -xvz -C .
 ```
 ### Generating New Pseudolabels
-Here is the command I used to generate the pseudolabels in the second row of the table, after downloading XSUM from [here](https://cdn-datasets.huggingface.co/summarization/xsum.tar.gz). 
+Here is the command I used to generate the pseudolabels in the second row of the table, after downloading XSUM from [here](https://s3.amazonaws.com/datasets.huggingface.co/summarization/xsum.tar.gz). 

 ```bash                                                                         
 python -m torch.distributed.launch --nproc_per_node=8 run_distributed_eval.py \
@@ -8,16 +8,7 @@ from torch.utils.data import DistributedSampler, RandomSampler
 from transformers import Trainer
 from transformers.configuration_fsmt import FSMTConfig
 from transformers.file_utils import is_torch_tpu_available
-from transformers.optimization import (
-    Adafactor,
-    AdamW,
-    get_constant_schedule,
-    get_constant_schedule_with_warmup,
-    get_cosine_schedule_with_warmup,
-    get_cosine_with_hard_restarts_schedule_with_warmup,
-    get_linear_schedule_with_warmup,
-    get_polynomial_decay_schedule_with_warmup,
-)
+from transformers.optimization import Adafactor, AdamW, get_linear_schedule_with_warmup
 from transformers.trainer_pt_utils import get_tpu_sampler


@@ -29,16 +20,6 @@ except ImportError:

 logger = logging.getLogger(__name__)

-arg_to_scheduler = {
-    "linear": get_linear_schedule_with_warmup,
-    "cosine": get_cosine_schedule_with_warmup,
-    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
-    "polynomial": get_polynomial_decay_schedule_with_warmup,
-    "constant": get_constant_schedule,
-    "constant_w_warmup": get_constant_schedule_with_warmup,
-}
-arg_to_scheduler_choices = sorted(arg_to_scheduler.keys())
-

 class Seq2SeqTrainer(Trainer):
    def __init__(self, config, data_args, *args, **kwargs):
@@ -81,21 +62,9 @@ class Seq2SeqTrainer(Trainer):
                )

        if self.lr_scheduler is None:
-            self.lr_scheduler = self._get_lr_scheduler(num_training_steps)
-        else:  # ignoring --lr_scheduler
-            logger.warn("scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.")
-
-    def _get_lr_scheduler(self, num_training_steps):
-        schedule_func = arg_to_scheduler[self.args.lr_scheduler]
-        if self.args.lr_scheduler == "constant":
-            scheduler = schedule_func(self.optimizer)
-        elif self.args.lr_scheduler == "constant_w_warmup":
-            scheduler = schedule_func(self.optimizer, num_warmup_steps=self.args.warmup_steps)
-        else:
-            scheduler = schedule_func(
+            self.lr_scheduler = get_linear_schedule_with_warmup(
                self.optimizer, num_warmup_steps=self.args.warmup_steps, num_training_steps=num_training_steps
            )
-        return scheduler

    def _get_train_sampler(self) -> Optional[torch.utils.data.sampler.Sampler]:
        if isinstance(self.train_dataset, torch.utils.data.IterableDataset):
@@ -174,7 +143,7 @@ class Seq2SeqTrainer(Trainer):
            # Call forward again to get loss # TODO: avoidable?
            outputs = model(**inputs, use_cache=False)
            loss = self._compute_loss(outputs[1], labels_out)
-            loss = loss.mean().detach()
+            loss = loss.mean().item()
            if self.args.prediction_loss_only:
                return (loss, None, None)

@@ -13,7 +13,7 @@ import torch

 import lightning_base
 from convert_pl_checkpoint_to_hf import convert_pl_to_hf
-from distillation import distill_main
+from distillation import distill_main, evaluate_checkpoint
 from finetune import SummarizationModule, main
 from run_eval import generate_summaries_or_translations, run_generate
 from run_eval_search import run_search
@@ -178,6 +178,7 @@ class TestSummarizationDistiller(unittest.TestCase):
        generate_summaries_or_translations(examples, out_path, str(model.output_dir / "best_tfmr"))
        self.assertTrue(Path(out_path).exists())

+        evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
        out_path_new = tempfile.mkdtemp()
        convert_pl_to_hf(ckpts[0], transformer_ckpts[0].parent, out_path_new)
        assert os.path.exists(os.path.join(out_path_new, "pytorch_model.bin"))
@@ -226,6 +227,8 @@ class TestSummarizationDistiller(unittest.TestCase):
        assert len(all_files) > 2
        self.assertEqual(len(transformer_ckpts), 2)

+        evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
+
    def test_distill_t5(self):
        updates = dict(
            student_encoder_layers=1,
@@ -1,22 +0,0 @@
-import tempfile
-import unittest
-
-from transformers.convert_marian_tatoeba_to_pytorch import TatoebaConverter
-from transformers.file_utils import cached_property
-from transformers.testing_utils import slow
-
-
-class TatoebaConversionTester(unittest.TestCase):
-    @cached_property
-    def resolver(self):
-        tmp_dir = tempfile.mkdtemp()
-        return TatoebaConverter(save_dir=tmp_dir)
-
-    @slow
-    def test_resolver(self):
-        self.resolver.convert_models(["heb-eng"])
-
-    @slow
-    def test_model_card(self):
-        content, mmeta = self.resolver.write_model_card("opus-mt-he-en", dry_run=True)
-        assert mmeta["long_pair"] == "heb-eng"
@@ -116,8 +116,8 @@ class ExamplesTests(TestCasePlus):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
-            result = run_pl_glue.main()[0]
-            # for now just testing that the script can run to completion
+            result = run_pl_glue.main()
+            # for now just testing that the script can run to a completion
            self.assertGreater(result["acc"], 0.25)
            #
            # TODO: this fails on CI - doesn't get acc/f1>=0.75:
@@ -59,7 +59,7 @@ class TorchXLAExamplesTests(unittest.TestCase):
            --model_name_or_path=bert-base-cased
            --per_device_train_batch_size=64
            --per_device_eval_batch_size=64
-            --evaluation_strategy steps
+            --evaluate_during_training
            --overwrite_cache
            """.split()
        with patch.object(sys, "argv", testargs):
@@ -81,14 +81,3 @@ class TorchXLAExamplesTests(unittest.TestCase):

            # Assert that the script takes less than 300 seconds to make sure it doesn't hang.
            self.assertLess(end - start, 300)
-
-    def test_trainer_tpu(self):
-        import xla_spawn
-
-        testargs = """
-            transformers/tests/test_trainer_tpu.py
-            --num_cores=8
-            transformers/tests/test_trainer_tpu.py
-            """.split()
-        with patch.object(sys, "argv", testargs):
-            xla_spawn.main()
@@ -43,7 +43,7 @@ python run_tf_text_classification.py \
  --do_eval \
  --do_predict \
  --logging_steps 10 \
-  --evaluation_strategy steps \
+  --evaluate_during_training \
  --save_steps 10 \
  --overwrite_output_dir \
  --max_seq_length 128
@@ -60,7 +60,7 @@ def get_tfds(
        for k in files.keys():
            transformed_ds[k] = ds[k].map(
                lambda example: tokenizer.batch_encode_plus(
-                    (example[features_name[0]], example[features_name[1]]),
+                    (example[features_name[0]], features_name[1]),
                    truncation=True,
                    max_length=max_seq_length,
                    padding="max_length",
@@ -96,9 +96,6 @@ def get_tfds(
        else None
    )

-    if train_ds is not None:
-        train_ds = train_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TRAIN])))
-
    val_ds = (
        tf.data.Dataset.from_generator(
            gen_val,
@@ -109,9 +106,6 @@ def get_tfds(
        else None
    )

-    if val_ds is not None:
-        val_ds = val_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.VALIDATION])))
-
    test_ds = (
        tf.data.Dataset.from_generator(
            gen_test,
@@ -122,9 +116,6 @@ def get_tfds(
        else None
    )

-    if test_ds is not None:
-        test_ds = test_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TEST])))
-
    return train_ds, val_ds, test_ds, label2id


@@ -1,3 +0,0 @@
---
-license: apache-2.0
---
@@ -4,5 +4,4 @@ datasets:
 - squad
 metrics:
 - squad
-license: apache-2.0
 ---
@@ -1,5 +1,4 @@
 ---
 language: de
-license: apache-2.0
 ---
 ## distilbert-base-german-cased
@@ -6,5 +6,4 @@ widget:
  context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
 - text: "How many square kilometers of rainforest is covered in the basin?"
  context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
-license: apache-2.0
 ---
@@ -1,3 +0,0 @@
---
-license: apache-2.0
---
@@ -1,3 +0,0 @@
---
-license: mit
---
@@ -1,19 +0,0 @@
-TinyBERT: Distilling BERT for Natural Language Understanding
-======== 
-TinyBERT is 7.5x smaller and 9.4x faster on inference than BERT-base and achieves competitive performances in the tasks of natural language understanding. It performs a novel transformer distillation at both the pre-training and task-specific learning stages. In general distillation, we use the original BERT-base without fine-tuning as the teacher and a large-scale text corpus as the learning data. By performing the Transformer distillation on the text from general domain, we obtain a general TinyBERT which provides a good initialization for the task-specific distillation. We here provide the general TinyBERT for your tasks at hand.
-
-For more details about the techniques of TinyBERT, refer to our paper:
-[TinyBERT: Distilling BERT for Natural Language Understanding](https://arxiv.org/abs/1909.10351)
-
-
-Citation
-========
-If you find TinyBERT useful in your research, please cite the following paper:
-```
-@article{jiao2019tinybert,
-  title={Tinybert: Distilling bert for natural language understanding},
-  author={Jiao, Xiaoqi and Yin, Yichun and Shang, Lifeng and Jiang, Xin and Chen, Xiao and Li, Linlin and Wang, Fang and Liu, Qun},
-  journal={arXiv preprint arXiv:1909.10351},
-  year={2019}
-}
-```
@@ -9,7 +9,6 @@ datasets:
 - xnli
 widget:
 - text: "За кого вы голосуете в 2020 году? <sep> This text is about politique."
-license: mit
 ---

 # xlm-roberta-large-xnli
@@ -0,0 +1,30 @@
+## prophetnet-large-uncased-cnndm
+Fine-tuned weights(converted from [original fairseq version repo](https://github.com/microsoft/ProphetNet)) for [ProphetNet](https://arxiv.org/abs/2001.04063) on summarization task CNN/DailyMail.  
+ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
+ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
+
+### Usage
+A quick usage is like: 
+```
+from transformers import ProphetNetTokenizer, ProphetNetForConditionalGeneration, ProphetNetConfig
+
+model = ProphetNetForConditionalGeneration.from_pretrained('microsoft/prophetnet-large-uncased-cnndm')
+tokenizer = ProphetNetTokenizer.from_pretrained('microsoft/prophetnet-large-uncased-cnndm')
+
+ARTICLE_TO_SUMMARIZE = "USTC was founded in Beijing by the Chinese Academy of Sciences (CAS) in September 1958. The Director of CAS, Mr. Guo Moruo was appointed the first president of USTC. USTC's founding mission was to develop a high-level science and technology workforce, as deemed critical for development of China's economy, defense, and science and technology education. The establishment was hailed as \"A Major Event in the History of Chinese Education and Science.\" CAS has supported USTC by combining most of its institutes with the departments of the university. USTC is listed in the top 16 national key universities, becoming the youngest national key university.".lower()
+inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=100, return_tensors='pt')
+
+# Generate Summary
+summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=512, early_stopping=True)
+print([tokenizer.decode(g) for g in summary_ids])
+```
+Here, [X_SEP] is used as a special token to seperate sentences.
+### Citation
+```bibtex
+@article{yan2020prophetnet,
+  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
+  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
+  journal={arXiv preprint arXiv:2001.04063},
+  year={2020}
+}
+```
@@ -0,0 +1,17 @@
+## prophetnet-large-uncased
+Pretrained weights for [ProphetNet](https://arxiv.org/abs/2001.04063).  
+ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
+ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
+
+### Usage
+Please see [the official repository](https://github.com/microsoft/ProphetNet) for details.
+
+### Citation
+```bibtex
+@article{yan2020prophetnet,
+  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
+  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
+  journal={arXiv preprint arXiv:2001.04063},
+  year={2020}
+}
+```
@@ -0,0 +1,32 @@
+## xprophetnet-large-wiki100-cased-xglue-ntg
+Cross-lingual version [ProphetNet](https://arxiv.org/abs/2001.04063), pretrained on [wiki100 xGLUE dataset](https://arxiv.org/abs/2004.01401) and finetuned on xGLUE cross-lingual News Titles Generation task.  
+ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
+ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
+
+xProphetNet is also served as the baseline model for xGLUE cross-lingual natural language generation tasks.  
+For xGLUE corss-lingual NLG tasks, xProphetNet is finetuned with English data, but inference with both English and other zero-shot language data.  
+### Usage
+A quick usage is like: 
+```
+from transformers import ProphetNetTokenizer, ProphetNetForConditionalGeneration, ProphetNetConfig
+
+model = ProphetNetForConditionalGeneration.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-ntg')
+tokenizer = ProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-ntg')
+
+EN_SENTENCE = "Microsoft Corporation intends to officially end free support for the Windows 7 operating system after January 14, 2020, according to the official portal of the organization. From that day, users of this system will not be able to receive security updates, which could make their computers vulnerable to cyber attacks."
+RU_SENTENCE = "орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7 после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми к кибератакам."
+ZH_SENTENCE = "根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。"
+inputs = tokenizer([EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE], padding=True, max_length=256, return_tensors='pt')
+
+summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=100, early_stopping=True)
+print([tokenizer.decode(g) for g in summary_ids])  
+```
+### Citation
+```bibtex
+@article{yan2020prophetnet,
+  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
+  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
+  journal={arXiv preprint arXiv:2001.04063},
+  year={2020}
+}
+```
@@ -0,0 +1,31 @@
+## xprophetnet-large-wiki100-cased-xglue-ntg
+Cross-lingual version [ProphetNet](https://arxiv.org/abs/2001.04063), pretrained on [wiki100 xGLUE dataset](https://arxiv.org/abs/2004.01401) and finetuned on xGLUE cross-lingual Question Generation task.  
+ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
+ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
+
+xProphetNet is also served as the baseline model for xGLUE cross-lingual natural language generation tasks.  
+For xGLUE corss-lingual NLG tasks, xProphetNet is finetuned with English data, but inference with both English and other zero-shot language data.  
+### Usage
+A quick usage is like: 
+```
+from transformers import ProphetNetTokenizer, ProphetNetForConditionalGeneration, ProphetNetConfig
+
+model = ProphetNetForConditionalGeneration.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-qg')
+tokenizer = ProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-qg')
+
+EN_SENTENCE = "Google left China in 2010"
+ZH_SENTENCE = "Google在2010年离开中国"
+inputs = tokenizer([EN_SENTENCE, ZH_SENTENCE], padding=True, max_length=256, return_tensors='pt')
+
+summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=100, early_stopping=True)
+print([tokenizer.decode(g) for g in summary_ids])  
+```
+### Citation
+```bibtex
+@article{yan2020prophetnet,
+  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
+  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
+  journal={arXiv preprint arXiv:2001.04063},
+  year={2020}
+}
+```
@@ -0,0 +1,19 @@
+## xprophetnet-large-wiki100-cased
+Cross-lingual version [ProphetNet](https://arxiv.org/abs/2001.04063), pretrained on [wiki100 xGLUE dataset](https://arxiv.org/abs/2004.01401).  
+ProphetNet is a new pre-trained language model for sequence-to-sequence learning with a novel self-supervised objective called future n-gram prediction.  
+ProphetNet is able to predict more future tokens with a n-stream decoder. The original implementation is Fairseq version at [github repo](https://github.com/microsoft/ProphetNet).   
+
+xProphetNet is also served as the baseline model for xGLUE cross-lingual natural language generation tasks.  
+For xGLUE corss-lingual NLG tasks, xProphetNet is finetuned with English data, but inference with both English and other zero-shot language data.  
+### Usage
+Please see [the official repository](https://github.com/microsoft/ProphetNet/tree/master/xProphetNet) for details.
+
+### Citation
+```bibtex
+@article{yan2020prophetnet,
+  title={Prophetnet: Predicting future n-gram for sequence-to-sequence pre-training},
+  author={Yan, Yu and Qi, Weizhen and Gong, Yeyun and Liu, Dayiheng and Duan, Nan and Chen, Jiusheng and Zhang, Ruofei and Zhou, Ming},
+  journal={arXiv preprint arXiv:2001.04063},
+  year={2020}
+}
+```
@@ -6,8 +6,6 @@ language:
 - fr
 - it
 - es
-
-license: mit
 ---

 # bert-base-multilingual-uncased-sentiment
@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-
-# this script builds a small sample spm file tests/fixtures/test_sentencepiece_no_bos.model, with features needed by pegasus 
-
-# 1. pip install sentencepiece
-# 
-# 2. wget https://raw.githubusercontent.com/google/sentencepiece/master/data/botchan.txt
-
-# 3. build
-import sentencepiece as spm
-
-# pegasus:
-# 1. no bos
-# 2. eos_id is 1
-# 3. unk_id is 2
-# build a sample spm file accordingly
-spm.SentencePieceTrainer.train('--input=botchan.txt --model_prefix=test_sentencepiece_no_bos --bos_id=-1 --unk_id=2  --eos_id=1  --vocab_size=1000')
-
-# 4. now update the fixture
-# mv test_sentencepiece_no_bos.model ../../tests/fixtures/
@@ -1,44 +0,0 @@
-Setup transformers following instructions in README.md, (I would fork first).
-```bash
-git clone git@github.com:huggingface/transformers.git
-cd transformers
-pip install -e .
-pip install pandas
-```
-
-Get required metadata
-```
-curl https://cdn-datasets.huggingface.co/language_codes/language-codes-3b2.csv  > language-codes-3b2.csv
-curl https://cdn-datasets.huggingface.co/language_codes/iso-639-3.csv > iso-639-3.csv
-```
-
-Install Tatoeba-Challenge repo inside transformers
-```bash
-git clone git@github.com:Helsinki-NLP/Tatoeba-Challenge.git
-```
-
-To convert a few models, call the conversion script from command line:
-```bash
-python src/transformers/convert_marian_tatoeba_to_pytorch.py --models heb-eng eng-heb --save_dir converted
-```
-
-To convert lots of models you can pass your list of Tatoeba model names to `resolver.convert_models` in a python client or script.
-
-```python
-from transformers.convert_marian_tatoeba_to_pytorch import TatoebaConverter
-resolver = TatoebaConverter(save_dir='converted')
-resolver.convert_models(['heb-eng', 'eng-heb'])
-```
-
-
-### Upload converted models
-```bash
-cd converted
-transformers-cli login
-for FILE in *; do transformers-cli upload $FILE; done
-```
-
-
-### Modifications
- To change naming logic, change the code near `os.rename`. The model card creation code may also need to change.
- To change model card content, you must modify `TatoebaCodeResolver.write_model_card`
@@ -3,10 +3,7 @@ Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/m

 To create the package for pypi.

-1. Change the version in __init__.py, setup.py as well as docs/source/conf.py. Remove the master from the links in
-   the new models of the README:
-   (https://huggingface.co/transformers/master/model_doc/ -> https://huggingface.co/transformers/model_doc/)
-   then run `make fix-copies` to fix the index of the documentation.
+1. Change the version in __init__.py, setup.py as well as docs/source/conf.py.

 2. Unpin specific versions from setup.py that use a git install.

@@ -134,7 +131,9 @@ setup(
        "sacremoses",
    ],
    extras_require=extras,
-    entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
+    entry_points={
+        "console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]
+    },
    python_requires=">=3.6.0",
    classifiers=[
        "Development Status :: 5 - Production/Stable",
@@ -54,6 +54,7 @@ from .configuration_mmbt import MMBTConfig
 from .configuration_mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig
 from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
 from .configuration_pegasus import PegasusConfig
+from .configuration_prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig
 from .configuration_rag import RagConfig
 from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig
 from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
@@ -63,6 +64,7 @@ from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config
 from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
 from .configuration_utils import PretrainedConfig
 from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig
+from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig
 from .configuration_xlm_roberta import XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMRobertaConfig
 from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig
 from .data import (
@@ -185,6 +187,7 @@ from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFas
 from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
 from .tokenization_pegasus import PegasusTokenizer, PegasusTokenizerFast
 from .tokenization_phobert import PhobertTokenizer
+from .tokenization_prophetnet import ProphetNetTokenizer
 from .tokenization_rag import RagTokenizer
 from .tokenization_reformer import ReformerTokenizer, ReformerTokenizerFast
 from .tokenization_retribert import RetriBertTokenizer, RetriBertTokenizerFast
@@ -203,6 +206,7 @@ from .tokenization_utils_base import (
 )
 from .tokenization_utils_fast import PreTrainedTokenizerFast
 from .tokenization_xlm import XLMTokenizer
+from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer
 from .tokenization_xlm_roberta import XLMRobertaTokenizer, XLMRobertaTokenizerFast
 from .tokenization_xlnet import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast

@@ -437,13 +441,18 @@ if is_torch_available():
    from .modeling_openai import (
        OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
        OpenAIGPTDoubleHeadsModel,
-        OpenAIGPTForSequenceClassification,
        OpenAIGPTLMHeadModel,
        OpenAIGPTModel,
        OpenAIGPTPreTrainedModel,
        load_tf_weights_in_openai_gpt,
    )
    from .modeling_pegasus import PegasusForConditionalGeneration
+    from .modeling_prophetnet import (
+        PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
+        ProphetNetForConditionalGeneration,
+        ProphetNetModel,
+        ProphetNetPreTrainedModel,
+    )
    from .modeling_rag import RagModel, RagSequenceForGeneration, RagTokenForGeneration
    from .modeling_reformer import (
        REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
@@ -504,6 +513,11 @@ if is_torch_available():
        XLMPreTrainedModel,
        XLMWithLMHeadModel,
    )
+    from .modeling_xlm_prophetnet import (
+        XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
+        XLMProphetNetForConditionalGeneration,
+        XLMProphetNetModel,
+    )
    from .modeling_xlm_roberta import (
        XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
        XLMRobertaForCausalLM,
@@ -41,6 +41,7 @@ from .configuration_mbart import MBART_PRETRAINED_CONFIG_ARCHIVE_MAP, MBartConfi
 from .configuration_mobilebert import MobileBertConfig
 from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
 from .configuration_pegasus import PegasusConfig
+from .configuration_prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig
 from .configuration_rag import RagConfig
 from .configuration_reformer import ReformerConfig
 from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
@@ -50,6 +51,7 @@ from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config
 from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
 from .configuration_utils import PretrainedConfig
 from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig
+from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig
 from .configuration_xlm_roberta import XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMRobertaConfig
 from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

@@ -84,6 +86,8 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
        DPR_PRETRAINED_CONFIG_ARCHIVE_MAP,
        DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP,
        SQUEEZEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
+        XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
+        PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
    ]
    for key, value, in pretrained_map.items()
 )
@@ -116,6 +120,8 @@ CONFIG_MAPPING = OrderedDict(
        ("gpt2", GPT2Config),
        ("transfo-xl", TransfoXLConfig),
        ("xlnet", XLNetConfig),
+        ("xlm-prophetnet", XLMProphetNetConfig),
+        ("prophetnet", ProphetNetConfig),
        ("xlm", XLMConfig),
        ("ctrl", CTRLConfig),
        ("electra", ElectraConfig),
@@ -164,6 +170,8 @@ MODEL_NAMES_MAPPING = OrderedDict(
        ("layoutlm", "LayoutLM"),
        ("dpr", "DPR"),
        ("rag", "RAG"),
+        ("xlm-prophetnet", "XLMProphetNet"),
+        ("prophetnet", "ProphetNet"),
    ]
 )

@@ -31,7 +31,7 @@ class MMBTConfig(object):
            Config of the underlying Transformer models. Its values are copied over to use a single config.
        num_labels (:obj:`int`, `optional`):
            Size of final Linear layer for classification.
-        modal_hidden_size (:obj:`int`, `optional`, defaults to 2048):
+        modal_hidden_size (:obj:`int`, `optional`, defautls to 2048):
            Embedding dimension of the non-text modality encoder.
    """

@@ -0,0 +1,112 @@
+# coding=utf-8
+# Copyright 2020 The Microsoft Authors and The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" ProphetNet model configuration """
+
+
+import logging
+
+from .configuration_utils import PretrainedConfig
+
+
+logger = logging.getLogger(__name__)
+
+PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "microsoft/prophetnet-large-uncased": "https://s3.amazonaws.com/models.huggingface.co/bert/microsoft/prophetnet-large-uncased/config.json",
+}
+
+
+class ProphetNetConfig(PretrainedConfig):
+    r"""
+    Configuration class for ProphetNet. Parameters are renamed from the fairseq implementation
+    """
+    model_type = "prophetnet"
+
+    def __init__(
+        self,
+        activation_dropout=0.1,
+        activation_function="gelu",
+        vocab_size=30522,
+        hidden_size=1024,
+        encoder_ffn_dim=4096,
+        num_encoder_layers=12,
+        num_encoder_attention_heads=16,
+        decoder_ffn_dim=4096,
+        num_decoder_layers=12,
+        num_decoder_attention_heads=16,
+        encoder_layerdrop=0.1,
+        decoder_layerdrop=0.1,
+        attention_dropout=0.1,
+        dropout=0.1,
+        encoder_max_position_embeddings=512,
+        decoder_max_position_embeddings=512,
+        init_std=0.02,
+        is_encoder_decoder=True,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        decoder_start_token_id=0,
+        ngram=2,
+        num_buckets=32,
+        relative_max_distance=128,
+        disable_ngram_loss=False,
+        eps=0.0,
+        **common_kwargs
+    ):
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            is_encoder_decoder=is_encoder_decoder,
+            decoder_start_token_id=decoder_start_token_id,
+            **common_kwargs,
+        )
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.encoder_ffn_dim = encoder_ffn_dim
+        self.num_encoder_layers = num_encoder_layers
+        self.num_encoder_attention_heads = num_encoder_attention_heads
+        self.encoder_layerdrop = encoder_layerdrop
+        self.decoder_layerdrop = decoder_layerdrop
+        self.decoder_ffn_dim = decoder_ffn_dim
+        self.num_decoder_layers = num_decoder_layers
+        self.num_decoder_attention_heads = num_decoder_attention_heads
+        self.encoder_max_position_embeddings = encoder_max_position_embeddings
+        self.decoder_max_position_embeddings = decoder_max_position_embeddings
+        self.init_std = init_std  # Normal(0, this parameter)
+        self.activation_function = activation_function
+
+        # parameters for prophetnet
+        self.ngram = ngram
+        self.num_buckets = num_buckets
+        self.relative_max_distance = relative_max_distance
+        self.disable_ngram_loss = disable_ngram_loss
+        self.eps = eps
+
+        # 3 Types of Dropout
+        self.attention_dropout = attention_dropout
+        self.activation_dropout = activation_dropout
+        self.dropout = dropout
+
+    @property
+    def max_position_embeddings(self) -> int:
+        return self.encoder_max_position_embeddings
+
+    @property
+    def num_attention_heads(self) -> int:
+        return self.num_encoder_attention_heads
+
+    @property
+    def num_hidden_layers(self) -> int:
+        return self.num_encoder_layers + self.num_decoder_layers
@@ -274,7 +274,7 @@ class PretrainedConfig(object):
                Path to a directory in which a downloaded pretrained model configuration should be cached if the
                standard cache should not be used.
            force_download (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Whether or not to force to (re-)download the configuration files and override the cached versions if they
+                Wheter or not to force to (re-)download the configuration files and override the cached versions if they
                exist.
            resume_download (:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not to delete incompletely received file. Attempts to resume the download if such a file
@@ -0,0 +1,22 @@
+""" XLM-ProphetNet model configuration """
+
+
+import logging
+
+from .configuration_prophetnet import ProphetNetConfig
+
+
+logger = logging.getLogger(__name__)
+
+XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "microsoft/xprophetnet-large-wiki100-cased": "https://s3.amazonaws.com/models.huggingface.co/bert/microsoft/xprophetnet-large-wiki100-cased/config.json",
+}
+
+
+class XLMProphetNetConfig(ProphetNetConfig):
+    """
+    This class overrides :class:`~transformers.RobertaConfig`. Please check the
+    superclass for the appropriate documentation alongside usage examples.
+    """
+
+    model_type = "xlm-prophetnet"
@@ -204,7 +204,7 @@ class XLNetConfig(PretrainedConfig):
        if mem_len is None or mem_len == 0:
            warnings.warn(
                "This config doesn't use attention memories, a core feature of XLNet."
-                " Consider setting `mem_len` to a non-zero value, for example "
+                " Consider setting `men_len` to a non-zero value, for example "
                "`xlnet = XLNetLMHeadModel.from_pretrained('xlnet-base-cased'', mem_len=1024)`,"
                " for accurate training performance as well as an order of magnitude faster inference."
                " Starting from version 3.5.0, the default parameter will be 1024, following"
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+path=$(realpath ${1})
+python ./convert_prophetnet_original_pytorch_checkpoint_to_pytorch.py --prophetnet_checkpoint_path "${path}_old" --pytorch_dump_folder_path "${path}"
@@ -44,8 +44,7 @@ class DPRContextEncoderState(DPRState):
        print("Loading DPR biencoder from {}".format(self.src_file))
        saved_state = load_states_from_checkpoint(self.src_file)
        encoder, prefix = model.ctx_encoder, "ctx_model."
-        # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
-        state_dict = {"bert_model.embeddings.position_ids": model.ctx_encoder.bert_model.embeddings.position_ids}
+        state_dict = {}
        for key, value in saved_state.model_dict.items():
            if key.startswith(prefix):
                key = key[len(prefix) :]
@@ -62,8 +61,7 @@ class DPRQuestionEncoderState(DPRState):
        print("Loading DPR biencoder from {}".format(self.src_file))
        saved_state = load_states_from_checkpoint(self.src_file)
        encoder, prefix = model.question_encoder, "question_model."
-        # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
-        state_dict = {"bert_model.embeddings.position_ids": model.question_encoder.bert_model.embeddings.position_ids}
+        state_dict = {}
        for key, value in saved_state.model_dict.items():
            if key.startswith(prefix):
                key = key[len(prefix) :]
@@ -79,10 +77,7 @@ class DPRReaderState(DPRState):
        model = DPRReader(DPRConfig(**BertConfig.get_config_dict("bert-base-uncased")[0]))
        print("Loading DPR reader from {}".format(self.src_file))
        saved_state = load_states_from_checkpoint(self.src_file)
-        # Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
-        state_dict = {
-            "encoder.bert_model.embeddings.position_ids": model.span_predictor.encoder.bert_model.embeddings.position_ids
-        }
+        state_dict = {}
        for key, value in saved_state.model_dict.items():
            if key.startswith("encoder.") and not key.startswith("encoder.encode_proj"):
                key = "encoder.bert_model." + key[len("encoder.") :]
@@ -211,7 +211,7 @@ def load_graph_from_args(pipeline_name: str, framework: str, model: str, tokeniz
        pipeline_name: The kind of pipeline to use (ner, question-answering, etc.)
        framework: The actual model to convert the pipeline from ("pt" or "tf")
        model: The model name which will be loaded by the pipeline
-        tokenizer: The tokenizer name which will be loaded by the pipeline, default to the model's value
+        tokenizer: The tokenizer name which will be loaded by the pipeline, defaut to the model's value

    Returns: Pipeline object

@@ -1,11 +1,12 @@
 import argparse
 import json
 import os
+import shutil
 import socket
 import time
 import warnings
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Dict, List, Tuple, Union
 from zipfile import ZipFile

 import numpy as np
@@ -22,6 +23,85 @@ def remove_suffix(text: str, suffix: str):
    return text  # or whatever


+def _process_benchmark_table_row(x):
+    fields = lmap(str.strip, x.replace("\t", "").split("|")[1:-1])
+    assert len(fields) == 3
+    return (fields[0], float(fields[1]), float(fields[2]))
+
+
+def process_last_benchmark_table(readme_path) -> List[Tuple[str, float, float]]:
+    md_content = Path(readme_path).open().read()
+    entries = md_content.split("## Benchmarks")[-1].strip().split("\n")[2:]
+    data = lmap(_process_benchmark_table_row, entries)
+    return data
+
+
+def check_if_models_are_dominated(old_repo_path="OPUS-MT-train/models", new_repo_path="Tatoeba-Challenge/models/"):
+    """Make a blacklist for models where we have already ported the same language pair, and the ported model has higher BLEU score."""
+    import pandas as pd
+
+    newest_released, old_reg, released = get_released_df(new_repo_path, old_repo_path)
+
+    short_to_new_bleu = newest_released.set_index("short_pair").bleu
+
+    assert released.groupby("short_pair").pair.nunique().max() == 1
+
+    short_to_long = released.groupby("short_pair").pair.first().to_dict()
+
+    overlap_short = old_reg.index.intersection(released.short_pair.unique())
+    overlap_long = [short_to_long[o] for o in overlap_short]
+    new_reported_bleu = [short_to_new_bleu[o] for o in overlap_short]
+
+    def get_old_bleu(o) -> float:
+        pat = old_repo_path + "/{}/README.md"
+        bm_data = process_last_benchmark_table(pat.format(o))
+        tab = pd.DataFrame(bm_data, columns=["testset", "bleu", "chr-f"])
+        tato_bleu = tab.loc[lambda x: x.testset.str.startswith("Tato")].bleu
+        if tato_bleu.shape[0] > 0:
+            return tato_bleu.iloc[0]
+        else:
+            return np.nan
+
+    old_bleu = [get_old_bleu(o) for o in overlap_short]
+    cmp_df = pd.DataFrame(
+        dict(short=overlap_short, long=overlap_long, old_bleu=old_bleu, new_bleu=new_reported_bleu)
+    ).fillna(-1)
+
+    dominated = cmp_df[cmp_df.old_bleu > cmp_df.new_bleu]
+    whitelist_df = cmp_df[cmp_df.old_bleu <= cmp_df.new_bleu]
+    blacklist = dominated.long.unique().tolist()  # 3 letter codes
+    return whitelist_df, dominated, blacklist
+
+
+def get_released_df(new_repo_path, old_repo_path):
+    import pandas as pd
+
+    released_cols = [
+        "url_base",
+        "pair",  # (ISO639-3/ISO639-5 codes),
+        "short_pair",  # (reduced codes),
+        "chrF2_score",
+        "bleu",
+        "brevity_penalty",
+        "ref_len",
+        "src_name",
+        "tgt_name",
+    ]
+    released = pd.read_csv(f"{new_repo_path}/released-models.txt", sep="\t", header=None).iloc[:-1]
+    released.columns = released_cols
+    old_reg = make_registry(repo_path=old_repo_path)
+    old_reg = pd.DataFrame(old_reg, columns=["id", "prepro", "url_model", "url_test_set"])
+    assert old_reg.id.value_counts().max() == 1
+    old_reg = old_reg.set_index("id")
+    released["fname"] = released["url_base"].apply(
+        lambda x: remove_suffix(remove_prefix(x, "https://object.pouta.csc.fi/Tatoeba-Challenge/opus"), ".zip")
+    )
+    released["2m"] = released.fname.str.startswith("2m")
+    released["date"] = pd.to_datetime(released["fname"].apply(lambda x: remove_prefix(remove_prefix(x, "2m-"), "-")))
+    newest_released = released.dsort("date").drop_duplicates(["short_pair"], keep="first")
+    return newest_released, old_reg, released
+
+
 def remove_prefix(text: str, prefix: str):
    if text.startswith(prefix):
        return text[len(prefix) :]
@@ -103,11 +183,7 @@ def find_model_file(dest_dir):  # this one better


 # Group Names Logic: change long opus model names to something shorter, like opus-mt-en-ROMANCE
-ROM_GROUP = (
-    "fr+fr_BE+fr_CA+fr_FR+wa+frp+oc+ca+rm+lld+fur+lij+lmo+es+es_AR+es_CL+es_CO+es_CR+es_DO+es_EC+es_ES+es_GT"
-    "+es_HN+es_MX+es_NI+es_PA+es_PE+es_PR+es_SV+es_UY+es_VE+pt+pt_br+pt_BR+pt_PT+gl+lad+an+mwl+it+it_IT+co"
-    "+nap+scn+vec+sc+ro+la"
-)
+ROM_GROUP = "fr+fr_BE+fr_CA+fr_FR+wa+frp+oc+ca+rm+lld+fur+lij+lmo+es+es_AR+es_CL+es_CO+es_CR+es_DO+es_EC+es_ES+es_GT+es_HN+es_MX+es_NI+es_PA+es_PE+es_PR+es_SV+es_UY+es_VE+pt+pt_br+pt_BR+pt_PT+gl+lad+an+mwl+it+it_IT+co+nap+scn+vec+sc+ro+la"
 GROUPS = [
    ("cmn+cn+yue+ze_zh+zh_cn+zh_CN+zh_HK+zh_tw+zh_TW+zh_yue+zhs+zht+zh", "ZH"),
    (ROM_GROUP, "ROMANCE"),
@@ -145,15 +221,13 @@ ORG_NAME = "Helsinki-NLP/"


 def convert_opus_name_to_hf_name(x):
-    """For OPUS-MT-Train/ DEPRECATED"""
    for substr, grp_name in GROUPS:
        x = x.replace(substr, grp_name)
    return x.replace("+", "_")


 def convert_hf_name_to_opus_name(hf_model_name):
-    """Relies on the assumption that there are no language codes like pt_br in models that are not in
-    GROUP_TO_OPUS_NAME."""
+    """Relies on the assumption that there are no language codes like pt_br in models that are not in GROUP_TO_OPUS_NAME."""
    hf_model_name = remove_prefix(hf_model_name, ORG_NAME)
    if hf_model_name in GROUP_TO_OPUS_NAME:
        opus_w_prefix = GROUP_TO_OPUS_NAME[hf_model_name]
@@ -173,9 +247,8 @@ def get_system_metadata(repo_root):
    )


-FRONT_MATTER_TEMPLATE = """---
-language:
-{}
+front_matter = """---
+language: {}
 tags:
 - translation

@@ -183,13 +256,11 @@ license: apache-2.0
 ---

 """
-DEFAULT_REPO = "Tatoeba-Challenge"
-DEFAULT_MODEL_DIR = os.path.join(DEFAULT_REPO, "models")


 def write_model_card(
    hf_model_name: str,
-    repo_root=DEFAULT_REPO,
+    repo_root="OPUS-MT-train",
    save_dir=Path("marian_converted"),
    dry_run=False,
    extra_metadata={},
@@ -223,10 +294,7 @@ def write_model_card(

    # combine with opus markdown

-    extra_markdown = (
-        f"### {hf_model_name}\n\n* source group: {metadata['src_name']} \n* target group: "
-        f"{metadata['tgt_name']} \n*  OPUS readme: [{opus_name}]({readme_url})\n"
-    )
+    extra_markdown = f"### {hf_model_name}\n\n* source group: {metadata['src_name']} \n* target group: {metadata['tgt_name']} \n*  OPUS readme: [{opus_name}]({readme_url})\n"

    content = opus_readme_path.open().read()
    content = content.split("\n# ")[-1]  # Get the lowest level 1 header in the README -- the most recent model.
@@ -234,7 +302,7 @@ def write_model_card(
    print(splat[3])
    content = "*".join(splat)
    content = (
-        FRONT_MATTER_TEMPLATE.format(metadata["src_alpha2"])
+        front_matter.format(metadata["src_alpha2"])
        + extra_markdown
        + "\n* "
        + content.replace("download", "download original weights")
@@ -255,6 +323,48 @@ def write_model_card(
    return content, metadata


+def get_clean_model_id_mapping(multiling_model_ids):
+    return {x: convert_opus_name_to_hf_name(x) for x in multiling_model_ids}
+
+
+def expand_group_to_two_letter_codes(grp_name):
+    raise NotImplementedError()
+
+
+def get_two_letter_code(three_letter_code):
+    raise NotImplementedError()
+    # return two_letter_code
+
+
+def get_tags(code, ref_name):
+    if len(code) == 2:
+        assert "languages" not in ref_name, f"{code}: {ref_name}"
+        return [code], False
+    elif "languages" in ref_name:
+        group = expand_group_to_two_letter_codes(code)
+        group.append(code)
+        return group, True
+    else:  # zho-> zh
+        raise ValueError(f"Three letter monolingual code: {code}")
+
+
+def resolve_lang_code(r):
+    """R is a row in ported"""
+    short_pair = r.short_pair
+    src, tgt = short_pair.split("-")
+    src_tags, src_multilingual = get_tags(src, r.src_name)
+    assert isinstance(src_tags, list)
+    tgt_tags, tgt_multilingual = get_tags(src, r.tgt_name)
+    assert isinstance(tgt_tags, list)
+    if src_multilingual:
+        src_tags.append("multilingual_src")
+    if tgt_multilingual:
+        tgt_tags.append("multilingual_tgt")
+    return src_tags + tgt_tags
+
+    # process target
+
+
 def make_registry(repo_path="Opus-MT-train/models"):
    if not (Path(repo_path) / "fr-en" / "README.md").exists():
        raise ValueError(
@@ -272,25 +382,36 @@ def make_registry(repo_path="Opus-MT-train/models"):
    return [(k, v["pre-processing"], v["download"], v["download"][:-4] + ".test.txt") for k, v in results.items()]


-def convert_all_sentencepiece_models(model_list=None, repo_path=None, dest_dir=Path("marian_converted")):
+def make_tatoeba_registry(repo_path="Tatoeba-Challenge/models"):
+    if not (Path(repo_path) / "zho-eng" / "README.md").exists():
+        raise ValueError(
+            f"repo_path:{repo_path} does not exist: "
+            "You must run: git clone git@github.com:Helsinki-NLP/Tatoeba-Challenge.git before calling."
+        )
+    results = {}
+    for p in Path(repo_path).iterdir():
+        if len(p.name) != 7:
+            continue
+        lns = list(open(p / "README.md").readlines())
+        results[p.name] = _parse_readme(lns)
+    return [(k, v["pre-processing"], v["download"], v["download"][:-4] + ".test.txt") for k, v in results.items()]
+
+
+def convert_all_sentencepiece_models(model_list=None, repo_path=None):
    """Requires 300GB"""
    save_dir = Path("marian_ckpt")
-    dest_dir = Path(dest_dir)
+    dest_dir = Path("marian_converted")
    dest_dir.mkdir(exist_ok=True)
-    save_paths = []
    if model_list is None:
        model_list: list = make_registry(repo_path=repo_path)
    for k, prepro, download, test_set_url in tqdm(model_list):
        if "SentencePiece" not in prepro:  # dont convert BPE models.
            continue
-        if not os.path.exists(save_dir / k):
+        if not os.path.exists(save_dir / k / "pytorch_model.bin"):
            download_and_unzip(download, save_dir / k)
        pair_name = convert_opus_name_to_hf_name(k)
        convert(save_dir / k, dest_dir / f"opus-mt-{pair_name}")

-        save_paths.append(dest_dir / f"opus-mt-{pair_name}")
-    return save_paths
-

 def lmap(f, x) -> List:
    return list(map(f, x))
@@ -372,6 +493,15 @@ def add_special_tokens_to_vocab(model_dir: Path) -> None:
    save_tokenizer_config(model_dir)


+def save_tokenizer(self, save_directory):
+    dest = Path(save_directory)
+    src_path = Path(self.init_kwargs["source_spm"])
+
+    for dest_name in {"source.spm", "target.spm", "tokenizer_config.json"}:
+        shutil.copyfile(src_path.parent / dest_name, dest / dest_name)
+    save_json(self.encoder, dest / "vocab.json")
+
+
 def check_equal(marian_cfg, k1, k2):
    v1, v2 = marian_cfg[k1], marian_cfg[k2]
    assert v1 == v2, f"hparams {k1},{k2} differ: {v1} != {v2}"
@@ -568,14 +698,14 @@ def convert(source_dir: Path, dest_dir):

    add_special_tokens_to_vocab(source_dir)
    tokenizer = MarianTokenizer.from_pretrained(str(source_dir))
-    tokenizer.save_pretrained(dest_dir)
+    save_tokenizer(tokenizer, dest_dir)

    opus_state = OpusState(source_dir)
    assert opus_state.cfg["vocab_size"] == len(
        tokenizer.encoder
    ), f"Original vocab size {opus_state.cfg['vocab_size']} and new vocab size {len(tokenizer.encoder)} mismatched"
    # save_json(opus_state.cfg, dest_dir / "marian_original_config.json")
-    # ^^ Uncomment to save human readable marian config for debugging
+    # ^^ Save human readable marian config for debugging

    model = opus_state.load_marian_model()
    model = model.half()
@@ -602,11 +732,15 @@ def unzip(zip_path: str, dest_dir: str) -> None:

 if __name__ == "__main__":
    """
-    Tatoeba conversion instructions in scripts/tatoeba/README.md
+    To bulk convert, run
+    >>> from transformers.convert_marian_to_pytorch import make_tatoeba_registry, convert_all_sentencepiece_models
+    >>> reg = make_tatoeba_registry()
+    >>> convert_all_sentencepiece_models(model_list=reg)  # saves to marian_converted
+    (bash) aws s3 sync marian_converted s3://models.huggingface.co/bert/Helsinki-NLP/ --dryrun
    """
    parser = argparse.ArgumentParser()
    # Required parameters
-    parser.add_argument("--src", type=str, help="path to marian model sub dir", default="en-de")
+    parser.add_argument("--src", type=str, help="path to marian model dir", default="en-de")
    parser.add_argument("--dest", type=str, default=None, help="Path to the output PyTorch model.")
    args = parser.parse_args()

@@ -0,0 +1,157 @@
+# coding=utf-8
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Convert RoBERTa checkpoint."""
+
+
+import argparse
+
+import torch
+
+from transformers import logging
+from transformers.modeling_prophetnet import ProphetNetForConditionalGeneration
+from transformers.modeling_xlm_prophetnet import XLMProphetNetForConditionalGeneration
+
+# transformers_old should correspond to branch `save_old_prophetnet_model_structure` here
+from transformers_old.modeling_prophetnet import (
+    ProphetNetForConditionalGeneration as ProphetNetForConditionalGenerationOld,
+)
+from transformers_old.modeling_xlm_prophetnet import (
+    XLMProphetNetForConditionalGeneration as XLMProphetNetForConditionalGenerationOld,
+)
+
+
+logger = logging.get_logger(__name__)
+logging.set_verbosity_info()
+
+
+def convert_prophetnet_checkpoint_to_pytorch(prophetnet_checkpoint_path: str, pytorch_dump_folder_path: str):
+    """
+    Copy/paste/tweak prohpetnet's weights to our prophetnet structure.
+    """
+    if "xprophetnet" in prophetnet_checkpoint_path:
+        prophet_old = XLMProphetNetForConditionalGenerationOld.from_pretrained(prophetnet_checkpoint_path)
+        prophet, loading_info = XLMProphetNetForConditionalGeneration.from_pretrained(
+            prophetnet_checkpoint_path, output_loading_info=True
+        )
+    else:
+        prophet_old = ProphetNetForConditionalGenerationOld.from_pretrained(prophetnet_checkpoint_path)
+        prophet, loading_info = ProphetNetForConditionalGeneration.from_pretrained(
+            prophetnet_checkpoint_path, output_loading_info=True
+        )
+
+    special_keys = ["key_proj", "value_proj", "query_proj"]
+
+    mapping = {
+        "self_attn": "ngram_self_attn",
+        "cross_attn": "encoder_attn",
+        "cross_attn_layer_norm": "encoder_attn_layer_norm",
+        "feed_forward_layer_norm": "final_layer_norm",
+        "feed_forward": "",
+        "intermediate": "fc1",
+        "output": "fc2",
+        "key_proj": "k_proj",
+        "query_proj": "q_proj",
+        "value_proj": "v_proj",
+        "word_embeddings": "embed_tokens",
+        "embeddings_layer_norm": "emb_layer_norm",
+        "relative_pos_embeddings": "relative_linear",
+        "ngram_embeddings": "ngram_input_embed",
+        "position_embeddings": "embed_positions",
+    }
+
+    for key in loading_info["missing_keys"]:
+        attributes = key.split(".")
+
+        if attributes[0] == "lm_head":
+            model = prophet
+            old_model = prophet_old
+        else:
+            model = prophet.prophetnet
+            old_model = prophet_old.model
+
+        is_key_init = False
+        for attribute in attributes:
+            if attribute in mapping:
+                old_attribute = mapping[attribute]
+                if not hasattr(old_model, old_attribute) and len(old_attribute) > 0:
+                    old_attribute = attribute
+            elif hasattr(old_model, attribute):
+                old_attribute = attribute
+
+            if attribute == "weight":
+                assert old_model.weight.shape == model.weight.shape, "Shapes have to match!"
+                model.weight = old_model.weight
+                logger.info(f"{attribute} is initialized.")
+                is_key_init = True
+                break
+            elif attribute == "bias":
+                assert old_model.bias.shape == model.bias.shape, "Shapes have to match!"
+                model.bias = old_model.bias
+                logger.info(f"{attribute} is initialized")
+                is_key_init = True
+                break
+            elif attribute in special_keys and hasattr(old_model, "in_proj_weight"):
+                embed_dim = old_model.in_proj_weight.shape[0] // 3
+                param = getattr(model, attribute)
+                param.weight.shape == old_model.in_proj_weight[:embed_dim, :].shape, "Shapes have to match"
+                param.bias.shape == old_model.in_proj_bias[:embed_dim].shape, "Shapes have to match"
+                if attribute == "query_proj":
+                    model.query_proj.weight = torch.nn.Parameter(old_model.in_proj_weight[:embed_dim, :])
+                    model.query_proj.bias = torch.nn.Parameter(old_model.in_proj_bias[:embed_dim])
+
+                elif attribute == "key_proj":
+                    model.key_proj.weight = torch.nn.Parameter(old_model.in_proj_weight[embed_dim : 2 * embed_dim, :])
+                    model.key_proj.bias = torch.nn.Parameter(old_model.in_proj_bias[embed_dim : 2 * embed_dim])
+                elif attribute == "value_proj":
+                    model.value_proj.weight = torch.nn.Parameter(old_model.in_proj_weight[2 * embed_dim :, :])
+                    model.value_proj.bias = torch.nn.Parameter(old_model.in_proj_bias[2 * embed_dim :])
+                is_key_init = True
+                break
+            elif attribute == "position_embeddings":
+                model.position_embeddings.weight = torch.nn.Parameter(old_model.embed_positions.weight)
+                is_key_init = True
+                break
+
+            if attribute.isdigit():
+                model = model[int(attribute)]
+                old_model = old_model[int(old_attribute)]
+            else:
+                model = getattr(model, attribute)
+
+                if old_attribute == "":
+                    old_model = old_model
+                else:
+                    if not hasattr(old_model, old_attribute):
+                        raise ValueError(f"{old_model} does not have {old_attribute}")
+                    old_model = getattr(old_model, old_attribute)
+
+        if not is_key_init:
+            raise ValueError(f"{key} was not correctly initialized!")
+
+    print(f"Saving model to {pytorch_dump_folder_path}")
+    prophet.save_pretrained(pytorch_dump_folder_path)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    # Required parameters
+    parser.add_argument(
+        "--prophetnet_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
+    )
+    parser.add_argument(
+        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
+    )
+    args = parser.parse_args()
+    convert_prophetnet_checkpoint_to_pytorch(args.prophetnet_checkpoint_path, args.pytorch_dump_folder_path)
@@ -547,7 +547,6 @@ CONVERTERS = {
    "DPRReaderTokenizer": BertConverter,
    "DPRQuestionEncoderTokenizer": BertConverter,
    "DPRContextEncoderTokenizer": BertConverter,
-    "ElectraTokenizer": BertConverter,
    "FunnelTokenizer": FunnelConverter,
    "GPT2Tokenizer": GPT2Converter,
    "LxmertTokenizer": BertConverter,
@@ -560,7 +560,7 @@ class SquadProcessor(DataProcessor):

        Args:
            dataset: The tfds dataset loaded from `tensorflow_datasets.load("squad")`
-            evaluate: Boolean specifying if in evaluation mode or in training mode
+            evaluate: boolean specifying if in evaluation mode or in training mode

        Returns:
            List of SquadExample
@@ -1093,7 +1093,7 @@ def is_tensor(x):
 class ModelOutput(OrderedDict):
    """
    Base class for all model outputs as dataclass. Has a ``__getitem__`` that allows indexing by integer or slice (like
-    a tuple) or strings (like a dictionary) that will ignore the ``None`` attributes. Otherwise behaves like a
+    a tuple) or strings (like a dictionnary) that will ignore the ``None`` attributes. Otherwise behaves like a
    regular python dictionary.

    .. warning::
@@ -65,8 +65,7 @@ class HfArgumentParser(ArgumentParser):
                if field.default is not dataclasses.MISSING:
                    kwargs["default"] = field.default
            elif field.type is bool or field.type is Optional[bool]:
-                if field.type is bool or (field.default is not None and field.default is not dataclasses.MISSING):
-                    kwargs["action"] = "store_false" if field.default is True else "store_true"
+                kwargs["action"] = "store_false" if field.default is True else "store_true"
                if field.default is True:
                    field_name = f"--no-{field.name}"
                    kwargs["dest"] = field.name
@@ -2,11 +2,13 @@
 import math
 import os

+from .file_utils import is_torch_tpu_available
+from .trainer_callback import TrainerCallback
+from .trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun
+from .utils import logging

-# Import 3rd-party integrations first:

 try:
-    # Comet needs to be imported before any ML frameworks
    import comet_ml  # noqa: F401

    _has_comet = True
@@ -51,14 +53,6 @@ except ImportError:
    except ImportError:
        _has_tensorboard = False

-# No transformer imports above this point
-
-from .file_utils import is_torch_tpu_available
-from .trainer_callback import TrainerCallback
-from .trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun
-from .utils import logging
-
-
 logger = logging.get_logger(__name__)


@@ -197,7 +191,7 @@ class TensorBoardCallback(TrainerCallback):

    Args:
        tb_writer (:obj:`SummaryWriter`, `optional`):
-            The writer to use. Will instantiate one if not set.
+            The writer to use. Will instatiate one if not set.
    """

    def __init__(self, tb_writer=None):
@@ -539,7 +539,7 @@ ALBERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -43,6 +43,7 @@ from .configuration_auto import (
    MobileBertConfig,
    OpenAIGPTConfig,
    PegasusConfig,
+    ProphetNetConfig,
    ReformerConfig,
    RetriBertConfig,
    RobertaConfig,
@@ -50,6 +51,7 @@ from .configuration_auto import (
    T5Config,
    TransfoXLConfig,
    XLMConfig,
+    XLMProphetNetConfig,
    XLMRobertaConfig,
    XLNetConfig,
    replace_list_option_in_docstrings,
@@ -153,8 +155,9 @@ from .modeling_mobilebert import (
    MobileBertForTokenClassification,
    MobileBertModel,
 )
-from .modeling_openai import OpenAIGPTForSequenceClassification, OpenAIGPTLMHeadModel, OpenAIGPTModel
+from .modeling_openai import OpenAIGPTLMHeadModel, OpenAIGPTModel
 from .modeling_pegasus import PegasusForConditionalGeneration
+from .modeling_prophetnet import ProphetNetForConditionalGeneration, ProphetNetModel
 from .modeling_rag import (  # noqa: F401 - need to import all RagModels to be in globals() function
    RagModel,
    RagSequenceForGeneration,
@@ -194,6 +197,7 @@ from .modeling_xlm import (
    XLMModel,
    XLMWithLMHeadModel,
 )
+from .modeling_xlm_prophetnet import XLMProphetNetForConditionalGeneration, XLMProphetNetModel
 from .modeling_xlm_roberta import (
    XLMRobertaForCausalLM,
    XLMRobertaForMaskedLM,
@@ -247,6 +251,8 @@ MODEL_MAPPING = OrderedDict(
        (BertGenerationConfig, BertGenerationEncoder),
        (DebertaConfig, DebertaModel),
        (DPRConfig, DPRQuestionEncoder),
+        (XLMProphetNetConfig, XLMProphetNetModel),
+        (ProphetNetConfig, ProphetNetModel),
    ]
 )

@@ -359,6 +365,8 @@ MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
        (BartConfig, BartForConditionalGeneration),
        (FSMTConfig, FSMTForConditionalGeneration),
        (EncoderDecoderConfig, EncoderDecoderModel),
+        (XLMProphetNetConfig, XLMProphetNetForConditionalGeneration),
+        (ProphetNetConfig, ProphetNetForConditionalGeneration),
    ]
 )

@@ -381,7 +389,6 @@ MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
        (FunnelConfig, FunnelForSequenceClassification),
        (DebertaConfig, DebertaForSequenceClassification),
        (GPT2Config, GPT2ForSequenceClassification),
-        (OpenAIGPTConfig, OpenAIGPTForSequenceClassification),
    ]
 )

@@ -507,7 +514,7 @@ AUTO_MODEL_PRETRAINED_DOCSTRING = r"""
                :obj:`{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each
                request.
            output_loading_info(:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Whether ot not to also return a dictionary containing missing keys, unexpected keys and error
+                Whether ot not to also return a dictionnary containing missing keys, unexpected keys and error
                messages.
            local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not to only look at local files (e.g., not try doanloading the model).
@@ -533,7 +540,7 @@ AUTO_MODEL_PRETRAINED_DOCSTRING = r"""
 class AutoModel:
    r"""
    This is a generic model class that will be instantiated as one of the base model classes of the library
-    when created with the :meth:`~transformers.AutoModel.from_pretrained` class method or the
+    when created with the when created with the :meth:`~transformers.AutoModel.from_pretrained` class method or the
    :meth:`~transformers.AutoModel.from_config` class methods.

    This class cannot be instantiated directly using ``__init__()`` (throws an error).
@@ -113,7 +113,7 @@ BART_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, target_sequence_length)`, `optional`):
@@ -667,7 +667,7 @@ BERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -781,7 +781,7 @@ class BertModel(BertPreTrainedModel):
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
@@ -1012,7 +1012,7 @@ class BertLMHeadModel(BertPreTrainedModel):
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.
        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
            Labels for computing the left-to-right language modeling loss (next word prediction).
            Indices should be in ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
@@ -218,7 +218,7 @@ BERT_GENERATION_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -450,7 +450,7 @@ class BertGenerationDecoder(BertGenerationPreTrainedModel):
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.
        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
            Labels for computing the left-to-right language modeling loss (next word prediction).
            Indices should be in ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
@@ -273,7 +273,7 @@ CTRL_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
@@ -401,7 +401,7 @@ DISTILBERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`):
@@ -358,7 +358,7 @@ DPR_ENCODERS_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
@@ -403,7 +403,7 @@ DPR_READER_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(n_passages, sequence_length, hidden_size)`, `optional`):
@@ -611,7 +611,7 @@ ELECTRA_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -74,7 +74,7 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, target_sequence_length)`, `optional`):
@@ -81,7 +81,7 @@ FLAUBERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
@@ -224,7 +224,7 @@ FSMT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, target_sequence_length)`, `optional`):
@@ -857,7 +857,7 @@ FUNNEL_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -429,7 +429,7 @@ GPT2_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, input_ids_length)`, `optional`):
@@ -584,7 +584,7 @@ class GPT2Model(GPT2PreTrainedModel):
            # positions we want to attend and -10000.0 for masked positions.
            # Since we are adding it to the raw scores before the softmax, this is
            # effectively the same as removing these entirely.
-            attention_mask = attention_mask.to(dtype=self.dtype)  # fp16 compatibility
+            attention_mask = attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
            attention_mask = (1.0 - attention_mask) * -10000.0

        # If a 2D ou 3D attention mask is provided for the cross-attention
@@ -701,23 +701,10 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
        if past:
            input_ids = input_ids[:, -1].unsqueeze(-1)

-        attention_mask = kwargs.get("attention_mask", None)
-        position_ids = kwargs.get("position_ids", None)
-
-        if attention_mask is not None and position_ids is None:
-            # create postion_ids on the fly for batch generation
-            position_ids = attention_mask.long().cumsum(-1) - 1
-            position_ids.masked_fill_(attention_mask == 0, 1)
-            if past:
-                position_ids = position_ids[:, -1].unsqueeze(-1)
-        else:
-            position_ids = None
        return {
            "input_ids": input_ids,
            "past_key_values": past,
            "use_cache": kwargs.get("use_cache"),
-            "position_ids": position_ids,
-            "attention_mask": attention_mask,
        }

    @add_start_docstrings_to_callable(GPT2_INPUTS_DOCSTRING)
@@ -1018,7 +1018,7 @@ LONGFORMER_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        global_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`):
@@ -848,7 +848,7 @@ LXMERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        visual_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`):
@@ -856,7 +856,7 @@ LXMERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -964,7 +964,7 @@ class LxmertModel(LxmertPreTrainedModel):
        # Process the visual attention mask
        if visual_attention_mask is not None:
            extended_visual_attention_mask = visual_attention_mask.unsqueeze(1).unsqueeze(2)
-            extended_visual_attention_mask = extended_visual_attention_mask.to(dtype=self.dtype)
+            extended_visual_attention_mask = extended_visual_attention_mask.to(dtype=next(self.parameters()).dtype)
            extended_visual_attention_mask = (1.0 - extended_visual_attention_mask) * -10000.0
        else:
            extended_visual_attention_mask = None
@@ -123,7 +123,7 @@ MMBT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
@@ -167,7 +167,7 @@ MMBT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

        output_attentions (:obj:`bool`, `optional`):
            Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
@@ -756,7 +756,7 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -792,7 +792,7 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

        output_attentions (:obj:`bool`, `optional`):
            Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
@@ -25,7 +25,7 @@ from typing import Optional, Tuple

 import torch
 import torch.nn as nn
-from torch.nn import CrossEntropyLoss, MSELoss
+from torch.nn import CrossEntropyLoss

 from .activations import gelu_new, swish
 from .configuration_openai import OpenAIGPTConfig
@@ -36,7 +36,7 @@ from .file_utils import (
    add_start_docstrings_to_callable,
    replace_return_docstrings,
 )
-from .modeling_outputs import BaseModelOutput, CausalLMOutput, SequenceClassifierOutput
+from .modeling_outputs import BaseModelOutput, CausalLMOutput
 from .modeling_utils import (
    Conv1D,
    PreTrainedModel,
@@ -360,7 +360,7 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
@@ -732,113 +732,3 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel):
            hidden_states=transformer_outputs.hidden_states,
            attentions=transformer_outputs.attentions,
        )
-
-
-@add_start_docstrings(
-    """The Original OpenAI GPT Model transformer with a sequence classification head on top
-    (linear layer).
-    :class:`~transformers.OpenAIGPTForSequenceClassification` uses the last token in order to do the classification, as
-    other causal models (e.g. GPT-2) do.
-    Since it does classification on the last token, it requires to know the position of the last token.
-    If a :obj:`pad_token_id` is defined in the configuration, it finds the last token that is not a padding token
-    in each row. If no :obj:`pad_token_id` is defined, it simply takes the last value in each row of the batch.
-    Since it cannot guess the padding tokens when :obj:`inputs_embeds` are passed instead of :obj:`input_ids`, it
-    does the same (take the last value in each row of the batch).
-    """,
-    OPENAI_GPT_START_DOCSTRING,
-)
-class OpenAIGPTForSequenceClassification(OpenAIGPTPreTrainedModel):
-    def __init__(self, config):
-        super().__init__(config)
-        self.num_labels = config.num_labels
-        self.transformer = OpenAIGPTModel(config)
-        self.score = nn.Linear(config.n_embd, self.num_labels, bias=False)
-
-        self.init_weights()
-
-    @add_start_docstrings_to_callable(OPENAI_GPT_INPUTS_DOCSTRING)
-    @add_code_sample_docstrings(
-        tokenizer_class=_TOKENIZER_FOR_DOC,
-        checkpoint="openai-gpt",
-        output_type=SequenceClassifierOutput,
-        config_class=_CONFIG_FOR_DOC,
-    )
-    def forward(
-        self,
-        input_ids=None,
-        attention_mask=None,
-        token_type_ids=None,
-        position_ids=None,
-        head_mask=None,
-        inputs_embeds=None,
-        labels=None,
-        output_attentions=None,
-        output_hidden_states=None,
-        return_dict=None,
-    ):
-        r"""
-        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`):
-            Labels for computing the sequence classification/regression loss.
-            Indices should be in :obj:`[0, ..., config.num_labels - 1]`.
-            If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),
-            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
-        """
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-        transformer_outputs = self.transformer(
-            input_ids,
-            attention_mask=attention_mask,
-            token_type_ids=token_type_ids,
-            position_ids=position_ids,
-            head_mask=head_mask,
-            inputs_embeds=inputs_embeds,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
-
-        hidden_states = transformer_outputs[0]
-        logits = self.score(hidden_states)
-
-        if input_ids is not None:
-            batch_size, sequence_length = input_ids.shape[:2]
-        else:
-            batch_size, sequence_length = inputs_embeds.shape[:2]
-
-        assert (
-            self.config.pad_token_id is not None or batch_size == 1
-        ), "Cannot handle batch sizes > 1 if no padding token is defined."
-        if self.config.pad_token_id is None:
-            sequence_lengths = -1
-        else:
-            if input_ids is not None:
-                sequence_lengths = torch.ne(input_ids, self.config.pad_token_id).sum(-1) - 1
-            else:
-                sequence_lengths = -1
-                logger.warning(
-                    f"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be "
-                    f"unexpected if using padding tokens in conjuction with `inputs_embeds.`"
-                )
-
-        pooled_logits = logits[range(batch_size), sequence_lengths]
-
-        loss = None
-        if labels is not None:
-            if self.num_labels == 1:
-                #  We are doing regression
-                loss_fct = MSELoss()
-                loss = loss_fct(pooled_logits.view(-1), labels.view(-1))
-            else:
-                loss_fct = CrossEntropyLoss()
-                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
-
-        if not return_dict:
-            output = (pooled_logits,) + transformer_outputs[1:]
-            return ((loss,) + output) if loss is not None else output
-
-        return SequenceClassifierOutput(
-            loss=loss,
-            logits=pooled_logits,
-            hidden_states=transformer_outputs.hidden_states,
-            attentions=transformer_outputs.attentions,
-        )
@@ -406,7 +406,7 @@ RAG_FORWARD_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        encoder_outputs (:obj:`tuple(tuple(torch.FloatTensor)`, `optional`)
@@ -836,7 +836,7 @@ class RagSequenceForGeneration(RagPreTrainedModel):
                Mask values selected in ``[0, 1]``:

                - 1 for tokens that are **not masked**,
-                - 0 for tokens that are **masked**.
+                - 0 for tokens that are **maked**.

                `What are attention masks? <../glossary.html#attention-mask>`__
            context_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size * config.n_docs, config.max_combined_length)`, `optional`, returned when `output_retrieved=True`):
@@ -1221,7 +1221,7 @@ class RagTokenForGeneration(RagPreTrainedModel):
                Mask values selected in ``[0, 1]``:

                - 1 for tokens that are **not masked**,
-                - 0 for tokens that are **masked**.
+                - 0 for tokens that are **maked**.

                `What are attention masks? <../glossary.html#attention-mask>`__
            context_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size * config.n_docs, config.max_combined_length)`, `optional`, returned when `output_retrieved=True`):
@@ -1926,7 +1926,7 @@ REFORMER_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
@@ -185,7 +185,7 @@ class RetriBertModel(RetriBertPreTrainedModel):
                Mask values selected in ``[0, 1]``:

                - 1 for tokens that are **not masked**,
-                - 0 for tokens that are **masked**.
+                - 0 for tokens that are **maked**.

                `What are attention masks? <../glossary.html#attention-mask>`__
            input_ids_doc (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`):
@@ -506,7 +506,7 @@ ROBERTA_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -461,7 +461,7 @@ SQUEEZEBERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -843,7 +843,7 @@ T5_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        decoder_input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, target_sequence_length)`, `optional`):
@@ -690,7 +690,7 @@ ALBERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`Numpy array` or :obj:`tf.Tensor` of shape :obj:`({0})`, `optional`):
@@ -390,7 +390,7 @@ TF_AUTO_MODEL_PRETRAINED_DOCSTRING = r"""
                :obj:`{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each
                request.
            output_loading_info(:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Whether ot not to also return a dictionary containing missing keys, unexpected keys and error
+                Whether ot not to also return a dictionnary containing missing keys, unexpected keys and error
                messages.
            local_files_only(:obj:`bool`, `optional`, defaults to :obj:`False`):
                Whether or not to only look at local files (e.g., not try doanloading the model).
@@ -735,7 +735,7 @@ BERT_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`Numpy array` or :obj:`tf.Tensor` of shape :obj:`({0})`, `optional`):
@@ -495,7 +495,7 @@ CTRL_INPUTS_DOCSTRING = r"""
            Mask values selected in ``[0, 1]``:

            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
+            - 0 for tokens that are **maked**.

            `What are attention masks? <../glossary.html#attention-mask>`__
        token_type_ids (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(batch_size, sequence_length)`, `optional`):
Author	SHA1	Message	Date
Patrick von Platen	04cc1982f3	refactor encoder layer	2020-10-13 17:39:02 +00:00
Patrick von Platen	843633731a	delete unnecessary files	2020-10-12 22:44:07 +00:00
Patrick von Platen	962a5b6f7a	finish new model outputs	2020-10-12 22:43:39 +00:00
Patrick von Platen	5e2d277dd9	finish all models	2020-10-12 13:26:09 +00:00
Patrick von Platen	1d80765b8f	save intermediate	2020-10-09 17:00:44 +00:00
Patrick von Platen	9a83f3cc6e	final weight remapping	2020-10-09 09:52:16 +00:00
Patrick von Platen	268e8892cb	rename weights	2020-10-09 10:57:31 +02:00
Patrick von Platen	27d2a62a8c	add fast integration test	2020-10-09 08:16:52 +00:00
Patrick von Platen	9c1530504a	delete unnecessary code	2020-10-09 09:37:40 +02:00
Patrick von Platen	76f6992a1a	fix tests	2020-10-08 19:54:09 +02:00
Patrick von Platen	01b9bb39af	fix index.rst	2020-10-08 19:14:09 +02:00
Patrick von Platen	132230f340	merge conflicts	2020-10-08 17:00:45 +00:00
Patrick von Platen	8081439191	remove unused import	2020-10-08 16:50:52 +00:00
Patrick von Platen	19363190ef	correct n-gram stream attention mask as discussed with qweizhen	2020-10-08 16:49:58 +00:00
Patrick von Platen	135d42699e	no weight mapping	2020-10-08 16:36:16 +00:00
Patrick von Platen	acf4210266	finalize model structure	2020-10-08 14:51:16 +00:00
Patrick von Platen	fed52134b4	:wqal:Merge branch 'prophetnet_develop' of https://github.com/qiweizhen/transformers into prophetnet_develop	2020-10-08 13:13:58 +00:00
patrickvonplaten	81951c48a6	add tokenizer tests	2020-10-08 00:56:54 +02:00
Patrick von Platen	828636c6f6	change model architecture	2020-10-06 18:50:29 +00:00
Patrick von Platen	77b0997222	make common tests pass	2020-10-06 15:51:27 +00:00
Patrick von Platen	39c4753ccf	fix xlm tests	2020-10-05 23:30:56 +00:00
Patrick von Platen	26a16931d6	change decoder output logits	2020-10-06 01:24:15 +02:00
Patrick von Platen	4369f4ebf0	fix common tests	2020-10-05 17:11:39 +02:00
Patrick von Platen	2d162fc9de	fix use_cache vs. past_key_values	2020-10-02 16:42:17 +00:00
Patrick von Platen	248d24e303	add decoder_attention_mask	2020-10-02 14:44:54 +00:00
Patrick von Platen	f19e63e9e6	save parallel	2020-10-02 14:34:37 +00:00
Patrick von Platen	b09c6c18dd	continue refactor	2020-09-29 13:41:09 +00:00
Patrick von Platen	5f6bdb5ffe	further refactoring	2020-09-28 14:45:35 +00:00
Patrick von Platen	e3fb44d270	correct typo in init	2020-09-28 13:03:13 +00:00
Patrick von Platen	997077bc3c	add xlm config to init	2020-09-28 12:57:30 +00:00
Patrick von Platen	8430813073	clean tokenizers	2020-09-28 12:55:14 +00:00
Patrick von Platen	c9cb2a5dfb	fix merge conflicts	2020-09-28 09:38:52 +00:00
Patrick von Platen	540928f0d4	clean integration tests	2020-09-28 11:28:14 +02:00
weizhen	5343a97691	move prophetnet into the latest version	2020-09-16 13:21:51 +08:00
weizhen	2681bd96a2	still bugs, because of changed output formats of encoder and decoder	2020-09-15 23:28:07 +08:00
weizhen	09cdfb36fc	add new model prophetnet prophetnet modified modify codes as suggested v1 add prophetnet test files	2020-09-15 22:41:24 +08:00