style

.circleci/config.yml

@@ -77,17 +77,16 @@ jobs:
                       - v0.4-torch_and_tf-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
             - run: pip install .[sklearn,tf-cpu,torch,testing]
             - save_cache:
                 key: v0.4-{{ checksum "setup.py" }}
                 paths:
                     - '~/.cache/pip'
-            - run: RUN_PT_TF_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf ./tests/ -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt
+            - run: RUN_PT_TF_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pt_tf_cross_test --durations=0 | tee output.txt
             - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
     run_tests_torch:
         working_directory: ~/transformers
         docker:
@@ -103,12 +102,13 @@ jobs:
                       - v0.4-torch-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
             - run: pip install .[sklearn,torch,testing]
             - save_cache:
                   key: v0.4-torch-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -s --make_reports=tests ./tests/ | tee tests_output.txt
             - store_artifacts:
                   path: ~/transformers/tests_output.txt
             - store_artifacts:
@@ -129,17 +129,16 @@ jobs:
                       - v0.4-tf-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
             - run: pip install .[sklearn,tf-cpu,testing]
             - save_cache:
                   key: v0.4-tf-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_tf ./tests/ | tee tests_output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
             - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+               path: ~/transformers/output.txt
+               destination: test_output.txt
     run_tests_flax:
         working_directory: ~/transformers
         docker:
@@ -155,17 +154,16 @@ jobs:
                     - v0.4-flax-{{ checksum "setup.py" }}
                     - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
             - run: sudo pip install .[flax,sklearn,torch,testing]
             - save_cache:
                   key: v0.4-flax-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_flax ./tests/ | tee tests_output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
             - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
     run_tests_pipelines_torch:
         working_directory: ~/transformers
         docker:
@@ -181,17 +179,16 @@ jobs:
                       - v0.4-torch-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
             - run: pip install .[sklearn,torch,testing]
             - save_cache:
                   key: v0.4-torch-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
-            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test ./tests/ | tee tests_output.txt
+            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt
             - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
     run_tests_pipelines_tf:
         working_directory: ~/transformers
         docker:
@@ -207,17 +204,16 @@ jobs:
                       - v0.4-tf-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
             - run: pip install .[sklearn,tf-cpu,testing]
             - save_cache:
                   key: v0.4-tf-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
-            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf ./tests/ -m is_pipeline_test | tee tests_output.txt
+            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt
             - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+               path: ~/transformers/output.txt
+               destination: test_output.txt
     run_tests_custom_tokenizers:
         working_directory: ~/transformers
         docker:
@@ -237,12 +233,10 @@ jobs:
                   key: v0.4-custom_tokenizers-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
-            - run: python -m pytest -s --make-reports=tests_custom_tokenizers ./tests/test_tokenization_bert_japanese.py | tee tests_output.txt
+            - run: python -m pytest -s ./tests/test_tokenization_bert_japanese.py | tee output.txt
             - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+                path: ~/transformers/output.txt
+                destination: test_output.txt
     run_examples_torch:
         working_directory: ~/transformers
         docker:
@@ -264,12 +258,11 @@ jobs:
                   key: v0.4-torch_examples-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_torch ./examples/ | tee examples_output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -s --make_reports=examples ./examples/ | tee examples_output.txt
             - store_artifacts:
                   path: ~/transformers/examples_output.txt
             - store_artifacts:
                   path: ~/transformers/reports
-
     build_doc:
         working_directory: ~/transformers
         docker:
@@ -281,7 +274,7 @@ jobs:
                       - v0.4-build_doc-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install ."[all, docs]"
+            - run: pip install .[tf,torch,sentencepiece,docs]
             - save_cache:
                   key: v0.4-build_doc-{{ checksum "setup.py" }}
                   paths:
@@ -289,7 +282,6 @@ jobs:
             - run: cd docs && make html SPHINXOPTS="-W"
             - store_artifacts:
                 path: ./docs/_build
-
     deploy_doc:
         working_directory: ~/transformers
         docker:
@@ -303,13 +295,12 @@ jobs:
                   keys:
                       - v0.4-deploy_doc-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
-            - run: pip install ."[all,docs]"
+            - run: pip install .[tf,torch,sentencepiece,docs]
             - save_cache:
                   key: v0.4-deploy_doc-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
             - run: ./.circleci/deploy.sh
-
     check_code_quality:
         working_directory: ~/transformers
         docker:
@@ -336,7 +327,6 @@ jobs:
             - run: python utils/check_copies.py
             - run: python utils/check_dummies.py
             - run: python utils/check_repo.py
-
     check_repository_consistency:
         working_directory: ~/transformers
         docker:
@@ -367,7 +357,6 @@ jobs:
             - setup_remote_docker
             - *build_push_docker
             - *deploy_cluster
-
     cleanup-gke-jobs:
         docker:
             - image: circleci/python:3.6
@@ -377,7 +366,6 @@ jobs:
                   cluster: $GKE_CLUSTER
                   perform-login: true
             - *delete_gke_jobs
-
 workflow_filters: &workflow_filters
     filters:
         branches:

.circleci/deploy.sh

@@ -51,5 +51,4 @@ deploy_doc "7fb8bdf" v3.0.2
 deploy_doc "4b3ee9c" v3.1.0
 deploy_doc "3ebb1b3" v3.2.0
 deploy_doc "0613f05" v3.3.1
-deploy_doc "eb0e0ce" v3.4.0
-deploy_doc "818878d" # v3.5.1 Latest stable release
+deploy_doc "eb0e0ce" # v3.4.0 Latest stable release

.github/ISSUE_TEMPLATE/bug-report.md

@@ -30,22 +30,20 @@ assignees: ''
  Trainer: @sgugger
  Speed and Memory Benchmarks: @patrickvonplaten
  Model Cards: @julien-c
+ Translation: @sshleifer
+ Summarization: @sshleifer
  TextGeneration: @TevenLeScao 
  examples/distillation: @VictorSanh
  nlp datasets: [different repo](https://github.com/huggingface/nlp)
  rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)
- Text Generation: @patrickvonplaten @TevenLeScao
- Blenderbot: @patrickvonplaten
- Bart: @patrickvonplaten
- Marian: @patrickvonplaten
- Pegasus: @patrickvonplaten
- mBART: @patrickvonplaten
+ Text Generation: @TevenLeScao
+ blenderbot: @mariamabarham
+ Bart: @sshleifer
+ Marian: @sshleifer
  T5: @patrickvonplaten
  Longformer/Reformer: @patrickvonplaten
- TransfoXL/XLNet: @TevenLeScao
- RAG: @patrickvonplaten, @lhoestq
- FSMT: @stas00
- examples/seq2seq: @patil-suraj
+ TransfoXL/XLNet: @TevenLeScao 
+ examples/seq2seq: @sshleifer
  examples/bert-loses-patience: @JetRunner
  tensorflow: @jplu
  examples/token-classification: @stefan-it

.github/ISSUE_TEMPLATE/question-help.md

@@ -1,6 +1,6 @@
 ---
 name: "❓ Questions & Help"
-about: Post your general questions on the Hugging Face forum: https://discuss.huggingface.co/
+about: Post your general questions on the Hugging Face forum or Stack Overflow tagged huggingface-transformers
 title: ''
 labels: ''
 assignees: ''
@@ -10,17 +10,18 @@ assignees: ''
 # ❓ Questions & Help
 
 <!-- The GitHub issue tracker is primarly intended for bugs, feature requests,
-     new models, benchmarks, and migration questions. For all other questions,
+     new models and benchmarks, and migration questions. For all other questions,
      we direct you to the Hugging Face forum: https://discuss.huggingface.co/ .
+     You can also try Stack Overflow (SO) where a whole community of PyTorch and
+     Tensorflow enthusiast can help you out. In this case, make sure to tag your
+     question with the right deep learning framework as well as the
+     huggingface-transformers tag: 
+     https://stackoverflow.com/questions/tagged/huggingface-transformers 
      -->
 
 ## Details
-
 <!-- Description of your issue -->
 
-<!-- You should first ask your question on the forum, and only if
-     you didn't get an answer after a few days ask it here on GitHub. -->
-
-**A link to original question on the forum**:
-
-<!-- Your issue will be closed if you don't fill this part. -->
+<!-- You should first ask your question on the forum or SO, and only if
+     you didn't get an answer ask it here on GitHub. -->
+**A link to original question on the forum/Stack Overflow**:

.github/PULL_REQUEST_TEMPLATE.md

@@ -43,20 +43,21 @@ members/contributors which may be interested in your PR.
  Trainer: @sgugger
  Benchmarks: @patrickvonplaten
  Model Cards: @julien-c
+ Translation: @sshleifer
+ Summarization: @sshleifer
  examples/distillation: @VictorSanh
  nlp datasets: [different repo](https://github.com/huggingface/nlp)
  rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)
  Text Generation: @patrickvonplaten, @TevenLeScao
- Blenderbot, Bart, Marian, Pegasus: @patrickvonplaten
+ Blenderbot, Bart, Marian, Pegasus: @sshleifer
  T5: @patrickvonplaten
  Rag: @patrickvonplaten, @lhoestq
  EncoderDecoder: @patrickvonplaten
  Longformer, Reformer: @patrickvonplaten
  TransfoXL, XLNet: @TevenLeScao, @patrickvonplaten
- examples/seq2seq: @patil-suraj
+ examples/seq2seq: @sshleifer
  examples/bert-loses-patience: @JetRunner
  tensorflow: @jplu
  examples/token-classification: @stefan-it
  documentation: @sgugger
- FSTM: @stas00
  -->

.github/workflows/github-torch-hub.yml

@@ -8,9 +8,6 @@ on:
 jobs:
   torch_hub_integration:
     runs-on: ubuntu-latest
-    env:
-      # TODO quickfix but may need more investigation
-      ACTIONS_ALLOW_UNSECURE_COMMANDS: True
     steps:
     # no checkout necessary here.
     - name: Extract branch name

.github/workflows/self-push.yml

@@ -4,19 +4,17 @@ on:
   push:
     branches:
       - master
-      - model-templates
     paths:
       - "src/**"
       - "tests/**"
       - ".github/**"
-      - "templates/**"
   # pull_request:
   repository_dispatch:
 
 
 jobs:
   run_tests_torch_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
     steps:
       - uses: actions/checkout@v2
       - name: Python version
@@ -57,36 +55,16 @@ jobs:
           python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
           python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
 
-      - name: Create model files
-        run: |
-          source .env/bin/activate
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-
       - name: Run all non-slow tests on GPU
         env:
           OMP_NUM_THREADS: 1
           CUDA_VISIBLE_DEVICES: 0
         run: |
           source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_torch_gpu_failures_short.txt
-        
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_torch_gpu_test_reports
-          path: reports
-                  
+          python -m pytest -n 2 --dist=loadfile -s ./tests/
 
   run_tests_tf_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
     steps:
       - uses: actions/checkout@v2
       - name: Python version
@@ -126,35 +104,16 @@ jobs:
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
 
-      - name: Create model files
-        run: |
-          source .env/bin/activate
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
-          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
-
       - name: Run all non-slow tests on GPU
         env:
           OMP_NUM_THREADS: 1
           CUDA_VISIBLE_DEVICES: 0
         run: |
           source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_gpu tests
+          python -m pytest -n 2 --dist=loadfile -s ./tests/
 
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_gpu_failures_short.txt
-        
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_tf_gpu_test_reports
-          path: reports
-
-  run_tests_torch_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+  run_tests_torch_multiple_gpu:
+    runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
       - name: Python version
@@ -172,7 +131,7 @@ jobs:
         id: cache
         with:
           path: .env
-          key: v1.1-tests_torch_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v1.1-tests_torch_multiple_gpu-${{ hashFiles('setup.py') }}
 
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         run: |
@@ -199,21 +158,10 @@ jobs:
           OMP_NUM_THREADS: 1
         run: |
           source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_multi_gpu tests
+          python -m pytest -n 2 --dist=loadfile -s ./tests/
 
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_torch_multi_gpu_failures_short.txt          
-
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_torch_multi_gpu_test_reports
-          path: reports
-
-  run_tests_tf_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+  run_tests_tf_multiple_gpu:
+    runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
       - name: Python version
@@ -231,7 +179,7 @@ jobs:
         id: cache
         with:
           path: .env
-          key: v1.1-tests_tf_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v1.1-tests_tf_multiple_gpu-${{ hashFiles('setup.py') }}
 
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         run: |
@@ -258,16 +206,4 @@ jobs:
           OMP_NUM_THREADS: 1
         run: |
           source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_multi_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_failures_short.txt
-
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_tf_multi_gpu_test_reports
-          path: reports
-          
+          python -m pytest -n 2 --dist=loadfile -s ./tests/

.github/workflows/self-scheduled.yml

@@ -1,22 +1,16 @@
-# configuration notes:
-#
-# - `source .env/bin/activate` is currently needed to be run first thing first in each step. Otherwise
-# the step uses the system-wide python interpreter.
-
 name: Self-hosted runner (scheduled)
 
 on:
   push:
     branches:
       - ci_*
-      - framework-agnostic-tokenizers
   repository_dispatch:
   schedule:
     - cron: "0 0 * * *"
 
 jobs:
   run_all_tests_torch_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
     steps:
       - uses: actions/checkout@v2
 
@@ -66,11 +60,11 @@ jobs:
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=tests_torch tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_gpu_failures_short.txt
+        run: cat reports/report_test_torch_failures_short.txt
         
       - name: Run examples tests on GPU
         if: ${{ always() }}
@@ -80,11 +74,11 @@ jobs:
         run: |
           source .env/bin/activate
           pip install -r examples/requirements.txt
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_gpu examples
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=examples_torch examples
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/examples_torch_gpu_failures_short.txt
+        run: cat reports/report_examples_torch_failures_short.txt
 
       - name: Run all pipeline tests on GPU
         if: ${{ always() }}
@@ -95,11 +89,11 @@ jobs:
           RUN_PIPELINE_TESTS: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_torch_pipeline tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_pipeline_gpu_failures_short.txt
+        run: cat reports/report_tests_torch_pipeline_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}
@@ -110,7 +104,7 @@ jobs:
 
 
   run_all_tests_tf_gpu:
-    runs-on: [self-hosted, gpu, single-gpu]
+    runs-on: [self-hosted, single-gpu]
     steps:
       - uses: actions/checkout@v2
 
@@ -160,26 +154,11 @@ jobs:
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=tests_tf tests
           
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_gpu_failures_short.txt
-
-      - name: Run all pipeline tests on GPU
-        if: ${{ always() }}
-        env:
-          TF_FORCE_GPU_ALLOW_GROWTH: "true"
-          OMP_NUM_THREADS: 1
-          RUN_SLOW: yes
-          RUN_PIPELINE_TESTS: yes
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_pipelines_gpu_failures_short.txt
+        run: cat reports/report_test_tf_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}
@@ -188,8 +167,8 @@ jobs:
           name: run_all_tests_tf_gpu_test_reports
           path: reports
           
-  run_all_tests_torch_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+  run_all_tests_torch_multiple_gpu:
+    runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
 
@@ -198,7 +177,7 @@ jobs:
         id: cache
         with:
           path: .env
-          key: v1.1-slow_tests_torch_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v0.1-slow_tests_torch_multi_gpu-${{ hashFiles('setup.py') }}
 
       - name: Python version
         run: |
@@ -233,31 +212,33 @@ jobs:
           python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
           python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
 
-      - name: Run all tests on multi-GPU
+      - name: Run all tests on GPU
         env:
           OMP_NUM_THREADS: 1
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=tests_torch tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_multi_gpu_failures_short.txt
-
-      - name: Run examples tests on multi-GPU
+        run: cat reports/report_test_torch_failures_short.txt
+        
+      - name: Run examples tests on GPU
+        if: ${{ always() }}
         env:
           OMP_NUM_THREADS: 1
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_multi_gpu examples
+          pip install -r examples/requirements.txt
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=examples_torch examples
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/examples_torch_multi_gpu_failures_short.txt
+        run: cat reports/report_examples_torch_failures_short.txt
 
-      - name: Run all pipeline tests on multi-GPU
+      - name: Run all pipeline tests on GPU
         if: ${{ always() }}
         env:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
@@ -266,11 +247,11 @@ jobs:
           RUN_PIPELINE_TESTS: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_torch_pipeline tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_pipeline_multi_gpu_failures_short.txt
+        run: cat reports/report_tests_torch_pipeline_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}
@@ -279,8 +260,8 @@ jobs:
           name: run_all_tests_torch_multi_gpu_test_reports
           path: reports
 
-  run_all_tests_tf_multi_gpu:
-    runs-on: [self-hosted, gpu, multi-gpu]
+  run_all_tests_tf_multiple_gpu:
+    runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
 
@@ -289,7 +270,7 @@ jobs:
         id: cache
         with:
           path: .env
-          key: v1.1-slow_tests_tf_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v0.1-slow_tests_tf_multi_gpu-${{ hashFiles('setup.py') }}
 
       - name: Python version
         run: |
@@ -324,20 +305,15 @@ jobs:
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
 
-      - name: Run all tests on multi-GPU
+      - name: Run all tests on GPU
         env:
           OMP_NUM_THREADS: 1
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0
 
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_failures_short.txt
-
-      - name: Run all pipeline tests on multi-GPU
-        if: ${{ always() }}
+      - name: Run all pipeline tests on GPU
         env:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
           OMP_NUM_THREADS: 1
@@ -345,11 +321,11 @@ jobs:
           RUN_PIPELINE_TESTS: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s ./tests/ -m is_pipeline_test --durations=50
           
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_pipelines_failures_short.txt
+        run: cat reports/report_test_tf_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}
@@ -357,4 +333,4 @@ jobs:
         with:
           name: run_all_tests_tf_multi_gpu_test_reports
           path: reports
-          
+          

.gitignore

@@ -133,6 +133,7 @@ dmypy.json
 tensorflow_code
 
 # Models
+models
 proc_data
 
 # examples

CONTRIBUTING.md

@@ -308,12 +308,3 @@ Check our [documentation writing guide](https://github.com/huggingface/transform
 for more information.
 
 #### This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/master/CONTRIBUTING.md)
-
-
-### Develop on Windows
-
-One way one can run the make command on Window is to pass by MSYS2:
-
-1. [Download MSYS2](https://www.msys2.org/), we assume to have it installed in C:\msys64
-2. Open the command line C:\msys64\msys2.exe (it should be available from the start menu)
-3. Run in the shell: `pacman -Syu` and install make with `pacman -S make`

Makefile

@@ -3,9 +3,14 @@
 
 check_dirs := examples tests src utils
 
+# get modified files since the branch was made
+fork_point_sha := $(shell git merge-base --fork-point master)
+joined_dirs := $(shell echo $(check_dirs) | tr " " "|")
+modified_py_files := $(shell git diff --name-only $(fork_point_sha) | egrep '^($(joined_dirs))' | egrep '\.py$$')
+#$(info modified files are: $(modified_py_files))
+
 modified_only_fixup:
-	$(eval modified_py_files := $(shell python utils/get_modified_files.py $(check_dirs)))
-	@if test -n "$(modified_py_files)"; then \
+	@if [ -n "$(modified_py_files)" ]; then \
 		echo "Checking/fixing $(modified_py_files)"; \
 		black $(modified_py_files); \
 		isort $(modified_py_files); \

README.md

@@ -181,7 +181,6 @@ Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
 1. **[LXMERT](https://huggingface.co/transformers/model_doc/lxmert.html)** (from UNC Chapel Hill) released with the paper [LXMERT: Learning Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal.
 1. **[MarianMT](https://huggingface.co/transformers/model_doc/marian.html)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team.
 1. **[MBart](https://huggingface.co/transformers/model_doc/mbart.html)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-1. **[MT5](https://huggingface.co/transformers/model_doc/mt5.html)** (from Google AI) released with the paper [mT5: A massively multilingual pre-trained text-to-text transformer](https://arxiv.org/abs/2010.11934) by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
 1. **[Pegasus](https://huggingface.co/transformers/model_doc/pegasus.html)** (from Google) released with the paper [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization](https://arxiv.org/abs/1912.08777)> by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
 1. **[ProphetNet](https://huggingface.co/transformers/model_doc/prophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
 1. **[Reformer](https://huggingface.co/transformers/model_doc/reformer.html)** (from Google Research) released with the paper [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.

docs/source/_static/js/custom.js

@@ -1,11 +1,10 @@
 // These two things need to be updated at each release for the version selector.
 // Last stable version
-const stableVersion = "v3.5.0"
+const stableVersion = "v3.4.0"
 // Dictionary doc folder to label
 const versionMapping = {
     "master": "master",
-    "": "v3.5.0/v3.5.1",
-    "v3.4.0": "v3.4.0",
+    "": "v3.4.0",
     "v3.3.1": "v3.3.0/v3.3.1",
     "v3.2.0": "v3.2.0",
     "v3.1.0": "v3.1.0 (stable)",

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'3.5.0'
+release = u'3.4.0'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/index.rst

@@ -126,44 +126,41 @@ conversion utilities for the following models:
 21. :doc:`MBart <model_doc/mbart>` (from Facebook) released with the paper `Multilingual Denoising Pre-training for
     Neural Machine Translation <https://arxiv.org/abs/2001.08210>`__ by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li,
     Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
-22. :doc:`MT5 <model_doc/mt5>` (from Google AI) released with the paper `mT5: A massively multilingual pre-trained
-    text-to-text transformer <https://arxiv.org/abs/2010.11934>`__ by Linting Xue, Noah Constant, Adam Roberts, Mihir
-    Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
-23. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
+22. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
     Gap-sentences for Abstractive Summarization <https://arxiv.org/abs/1912.08777>`__> by Jingqing Zhang, Yao Zhao,
     Mohammad Saleh and Peter J. Liu.
-24. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
+23. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
     Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi,
     Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-25. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
+24. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
     Transformer <https://arxiv.org/abs/2001.04451>`__ by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
-26. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
+25. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
     Pretraining Approach <https://arxiv.org/abs/1907.11692>`__ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar
     Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. ultilingual BERT into `DistilmBERT
     <https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German version of
     DistilBERT.
-27. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
+26. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
     about efficient neural networks? <https://arxiv.org/abs/2006.11316>`__ by Forrest N. Iandola, Albert E. Shaw, Ravi
     Krishna, and Kurt W. Keutzer.
-28. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
+27. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
     Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by Colin Raffel and Noam Shazeer and Adam
     Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
-29. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
+28. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
     Attentive Language Models Beyond a Fixed-Length Context <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*,
     Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
-30. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
+29. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
     Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis Conneau.
-31. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
+30. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
     Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan,
     Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
-32. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
+31. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
     Cross-lingual Representation Learning at Scale <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay
     Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke
     Zettlemoyer and Veselin Stoyanov.
-33. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `​XLNet: Generalized Autoregressive
+32. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `​XLNet: Generalized Autoregressive
     Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by Zhilin Yang*, Zihang Dai*, Yiming
     Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
-34. `Other community models <https://huggingface.co/models>`__, contributed by the `community
+33. `Other community models <https://huggingface.co/models>`__, contributed by the `community
     <https://huggingface.co/users>`__.
 
 .. toctree::
@@ -251,7 +248,6 @@ conversion utilities for the following models:
     model_doc/marian
     model_doc/mbart
     model_doc/mobilebert
-    model_doc/mt5
     model_doc/gpt
     model_doc/gpt2
     model_doc/pegasus
@@ -276,4 +272,3 @@ conversion utilities for the following models:
     internal/pipelines_utils
     internal/tokenization_utils
     internal/trainer_utils
-    internal/generation_utils

docs/source/internal/generation_utils.rst

@@ -1,50 +0,0 @@
-Utilities for Generation
------------------------------------------------------------------------------------------------------------------------
-
-This page lists all the utility functions used by :meth:`~transformers.PretrainedModel.generate`,
-:meth:`~transformers.PretrainedModel.greedy_search`, :meth:`~transformers.PretrainedModel.sample`,
-:meth:`~transformers.PretrainedModel.beam_search`, and :meth:`~transformers.PretrainedModel.beam_sample`.
-
-Most of those are only useful if you are studying the code of the generate methods in the library.
-
-LogitsProcessor
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-A :class:`~transformers.LogitsProcessor` can be used to modify the prediction scores of a language model head for
-generation.
-
-.. autoclass:: transformers.LogitsProcessor
-    :members: __call__
-
-.. autoclass:: transformers.LogitsProcessorList
-    :members: __call__
-
-.. autoclass:: transformers.MinLengthLogitsProcessor
-    :members: __call__
-
-.. autoclass:: transformers.TemperatureLogitsWarper
-    :members: __call__
-
-.. autoclass:: transformers.RepetitionPenaltyLogitsProcessor
-    :members: __call__
-
-.. autoclass:: transformers.TopPLogitsWarper
-    :members: __call__
-
-.. autoclass:: transformers.TopKLogitsWarper
-    :members: __call__
-
-.. autoclass:: transformers.NoRepeatNGramLogitsProcessor
-    :members: __call__
-
-.. autoclass:: transformers.NoBadWordsLogitsProcessor
-    :members: __call__
-
-BeamSearch
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.BeamScorer
-    :members: process, finalize
-
-.. autoclass:: transformers.BeamSearchScorer
-    :members: process, finalize

docs/source/main_classes/model.rst

@@ -45,7 +45,7 @@ TFModelUtilsMixin
     :members:
 
 
-Generation
+Generative models
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.generation_utils.GenerationMixin

docs/source/main_classes/output.rst

@@ -65,34 +65,12 @@ BaseModelOutputWithPooling
     :members:
 
 
-BaseModelOutputWithCrossAttentions
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.modeling_outputs.BaseModelOutputWithCrossAttentions
-    :members:
-
-
-BaseModelOutputWithPoolingAndCrossAttentions
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.modeling_outputs.BaseModelOutputWithPoolingAndCrossAttentions
-    :members:
-
-
 BaseModelOutputWithPast
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.modeling_outputs.BaseModelOutputWithPast
     :members:
 
-
-BaseModelOutputWithPastAndCrossAttentions
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.modeling_outputs.BaseModelOutputWithPastAndCrossAttentions
-    :members:
-
-
 Seq2SeqModelOutput
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -107,20 +85,6 @@ CausalLMOutput
     :members:
 
 
-CausalLMOutputWithCrossAttentions
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.modeling_outputs.CausalLMOutputWithCrossAttentions
-    :members:
-
-
-CausalLMOutputWithPastAndCrossAttentions
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.modeling_outputs.CausalLMOutputWithPastAndCrossAttentions
-    :members:
-
-
 CausalLMOutputWithPast
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/main_classes/trainer.rst

@@ -35,7 +35,7 @@ Here is an example of how to customize :class:`~transformers.Trainer` using a cu
     class MyTrainer(Trainer):
         def compute_loss(self, model, inputs):
             labels = inputs.pop("labels")
-            outputs = model(**inputs)
+            outputs = models(**inputs)
             logits = outputs[0]
             return my_custom_loss(logits, labels)
 

docs/source/model_doc/albert.rst

@@ -51,10 +51,10 @@ AlbertTokenizer
 Albert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.albert.modeling_albert.AlbertForPreTrainingOutput
+.. autoclass:: transformers.modeling_albert.AlbertForPreTrainingOutput
     :members:
 
-.. autoclass:: transformers.models.albert.modeling_tf_albert.TFAlbertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_albert.TFAlbertForPreTrainingOutput
     :members:
 
 

docs/source/model_doc/auto.rst

@@ -81,13 +81,6 @@ AutoModelForMultipleChoice
     :members:
 
 
-AutoModelForNextSentencePrediction
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.AutoModelForNextSentencePrediction
-    :members:
-
-
 AutoModelForTokenClassification
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/bart.rst

@@ -3,7 +3,7 @@ BART
 
 **DISCLAIMER:** If you see something strange, file a `Github Issue
 <https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ and assign
-@patrickvonplaten
+@sshleifer
 
 Overview
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -26,51 +26,22 @@ According to the abstract,
 The Authors' code can be found `here <https://github.com/pytorch/fairseq/tree/master/examples/bart>`__.
 
 
-Examples
-_______________________________________________________________________________________________________________________
-
-- Examples and scripts for fine-tuning BART and other models for sequence to sequence tasks can be found in
-  `examples/seq2seq/ <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
-- An example of how to train :class:`~transformers.BartForConditionalGeneration` with a Hugging Face :obj:`datasets`
-  object can be found in this `forum discussion
-  <https://discuss.huggingface.co/t/train-bart-for-conditional-generation-e-g-summarization/1904>`__.
-- `Distilled checkpoints <https://huggingface.co/models?search=distilbart>`__ are described in this `paper
-  <https://arxiv.org/abs/2010.13002>`__.
-
-
 Implementation Notes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Bart doesn't use :obj:`token_type_ids` for sequence classification. Use :class:`~transformers.BartTokenizer` or
   :meth:`~transformers.BartTokenizer.encode` to get the proper splitting.
 - The forward pass of :class:`~transformers.BartModel` will create decoder inputs (using the helper function
-  :func:`transformers.models.bart.modeling_bart._prepare_bart_decoder_inputs`) if they are not passed. This is
-  different than some other modeling APIs.
-- Model predictions are intended to be identical to the original implementation when
-  :obj:`force_bos_token_to_be_generated=True`. This only works, however, if the string you pass to
-  :func:`fairseq.encode` starts with a space.
+  :func:`transformers.modeling_bart._prepare_bart_decoder_inputs`) if they are not passed. This is different than some
+  other modeling APIs.
+- Model predictions are intended to be identical to the original implementation. This only works, however, if the
+  string you pass to :func:`fairseq.encode` starts with a space.
 - :meth:`~transformers.BartForConditionalGeneration.generate` should be used for conditional generation tasks like
   summarization, see the example in that docstrings.
 - Models that load the `facebook/bart-large-cnn` weights will not have a :obj:`mask_token_id`, or be able to perform
   mask-filling tasks.
 - For training/forward passes that don't involve beam search, pass :obj:`use_cache=False`.
 
-Mask Filling
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The :obj:`facebook/bart-base` and :obj:`facebook/bart-large` checkpoints can be used to fill multi-token masks.
-
-.. code-block::
-
-    from transformers import BartForConditionalGeneration, BartTokenizer
-    model = BartForConditionalGeneration.from_pretrained("facebook/bart-large", force_bos_token_to_be_generated=True)
-    tok = BartTokenizer.from_pretrained("facebook/bart-large")
-    example_english_phrase = "UN Chief Says There Is No <mask> in Syria"
-    batch = tok(example_english_phrase, return_tensors='pt')
-    generated_ids = model.generate(batch['input_ids'])
-    assert tok.batch_decode(generated_ids, skip_special_tokens=True) == ['UN Chief Says There Is No Plan to Stop Chemical Weapons in Syria']
-
-
 
 BartConfig
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -93,7 +64,7 @@ BartModel
 .. autoclass:: transformers.BartModel
     :members: forward
 
-.. autofunction:: transformers.models.bart.modeling_bart._prepare_bart_decoder_inputs
+.. autofunction:: transformers.modeling_bart._prepare_bart_decoder_inputs
 
 
 BartForConditionalGeneration

docs/source/model_doc/bert.rst

@@ -57,10 +57,10 @@ BertTokenizerFast
 Bert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.bert.modeling_bert.BertForPreTrainingOutput
+.. autoclass:: transformers.modeling_bert.BertForPreTrainingOutput
     :members:
 
-.. autoclass:: transformers.models.bert.modeling_tf_bert.TFBertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_bert.TFBertForPreTrainingOutput
     :members:
 
 
@@ -188,10 +188,3 @@ TFBertForQuestionAnswering
 
 .. autoclass:: transformers.TFBertForQuestionAnswering
     :members: call
-
-
-FlaxBertModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxBertModel
-    :members: __call__

docs/source/model_doc/bertgeneration.rst

@@ -40,7 +40,7 @@ Usage:
   labels = tokenizer('This is a short summary', return_tensors="pt").input_ids
 
   # train...
-  loss = bert2bert(input_ids=input_ids, decoder_input_ids=labels, labels=labels).loss
+  loss = bert2bert(input_ids=input_ids, decoder_input_ids=labels, labels=labels, return_dict=True).loss
   loss.backward()
 
 

docs/source/model_doc/dpr.rst

@@ -71,13 +71,13 @@ DPRReaderTokenizerFast
 DPR specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.dpr.modeling_dpr.DPRContextEncoderOutput
+.. autoclass:: transformers.modeling_dpr.DPRContextEncoderOutput
     :members:
 
-.. autoclass:: transformers.models.dpr.modeling_dpr.DPRQuestionEncoderOutput
+.. autoclass:: transformers.modeling_dpr.DPRQuestionEncoderOutput
     :members:
 
-.. autoclass:: transformers.models.dpr.modeling_dpr.DPRReaderOutput
+.. autoclass:: transformers.modeling_dpr.DPRReaderOutput
     :members:
 
 
@@ -99,22 +99,3 @@ DPRReader
 
 .. autoclass:: transformers.DPRReader
     :members: forward
-
-TFDPRContextEncoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFDPRContextEncoder
-    :members: call
-
-TFDPRQuestionEncoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFDPRQuestionEncoder
-    :members: call
-
-
-TFDPRReader
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFDPRReader
-    :members: call

docs/source/model_doc/electra.rst

@@ -69,10 +69,10 @@ ElectraTokenizerFast
 Electra specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.electra.modeling_electra.ElectraForPreTrainingOutput
+.. autoclass:: transformers.modeling_electra.ElectraForPreTrainingOutput
     :members:
 
-.. autoclass:: transformers.models.electra.modeling_tf_electra.TFElectraForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_electra.TFElectraForPreTrainingOutput
     :members:
 
 

docs/source/model_doc/funnel.rst

@@ -65,10 +65,10 @@ FunnelTokenizerFast
 Funnel specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.funnel.modeling_funnel.FunnelForPreTrainingOutput
+.. autoclass:: transformers.modeling_funnel.FunnelForPreTrainingOutput
     :members:
 
-.. autoclass:: transformers.models.funnel.modeling_tf_funnel.TFFunnelForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_funnel.TFFunnelForPreTrainingOutput
     :members:
 
 

docs/source/model_doc/gpt.rst

@@ -72,10 +72,10 @@ OpenAIGPTTokenizerFast
 OpenAI specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.openai.modeling_openai.OpenAIGPTDoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_openai.OpenAIGPTDoubleHeadsModelOutput
     :members:
 
-.. autoclass:: transformers.models.openai.modeling_tf_openai.TFOpenAIGPTDoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_tf_openai.TFOpenAIGPTDoubleHeadsModelOutput
     :members:
 
 

docs/source/model_doc/gpt2.rst

@@ -60,10 +60,10 @@ GPT2TokenizerFast
 GPT2 specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.gpt2.modeling_gpt2.GPT2DoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_gpt2.GPT2DoubleHeadsModelOutput
     :members:
 
-.. autoclass:: transformers.models.gpt2.modeling_tf_gpt2.TFGPT2DoubleHeadsModelOutput
+.. autoclass:: transformers.modeling_tf_gpt2.TFGPT2DoubleHeadsModelOutput
     :members:
 
 

docs/source/model_doc/longformer.rst

@@ -90,30 +90,6 @@ LongformerTokenizerFast
 .. autoclass:: transformers.LongformerTokenizerFast
     :members: 
 
-Longformer specific outputs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerBaseModelOutput
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerBaseModelOutputWithPooling
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMultipleChoiceModelOutput
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerQuestionAnsweringModelOutput
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutput
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutputWithPooling
-    :members: 
-
-.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerQuestionAnsweringModelOutput
-    :members: 
-
 
 LongformerModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/source/model_doc/lxmert.rst

@@ -67,19 +67,19 @@ LxmertTokenizerFast
 Lxmert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.lxmert.modeling_lxmert.LxmertModelOutput
+.. autoclass:: transformers.modeling_lxmert.LxmertModelOutput
     :members:
 
-.. autoclass:: transformers.models.lxmert.modeling_lxmert.LxmertForPreTrainingOutput
+.. autoclass:: transformers.modeling_lxmert.LxmertForPreTrainingOutput
     :members:
 
-.. autoclass:: transformers.models.lxmert.modeling_lxmert.LxmertForQuestionAnsweringOutput
+.. autoclass:: transformers.modeling_lxmert.LxmertForQuestionAnsweringOutput
     :members:
 
-.. autoclass:: transformers.models.lxmert.modeling_tf_lxmert.TFLxmertModelOutput
+.. autoclass:: transformers.modeling_tf_lxmert.TFLxmertModelOutput
     :members:
 
-.. autoclass:: transformers.models.lxmert.modeling_tf_lxmert.TFLxmertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_lxmert.TFLxmertForPreTrainingOutput
     :members:
 
 

docs/source/model_doc/marian.rst

@@ -3,9 +3,9 @@ MarianMT
 
 **Bugs:** If you see something strange, file a `Github Issue
 <https://github.com/huggingface/transformers/issues/new?assignees=sshleifer&labels=&template=bug-report.md&title>`__
-and assign @patrickvonplaten.
+and assign @sshleifer.
 
-Translations should be similar, but not identical to output in the test set linked to in each model card.
+Translations should be similar, but not identical to, output in the test set linked to in each model card.
 
 Implementation Notes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -35,109 +35,24 @@ Naming
   <https://developers.google.com/admin-sdk/directory/v1/languages>`__, three digit codes require googling "language
   code {code}".
 - Codes formatted like :obj:`es_AR` are usually :obj:`code_{region}`. That one is Spanish from Argentina.
-- The models were converted in two stages. The first 1000 models use ISO-639-2 codes to identify languages, the second
-  group use a combination of ISO-639-5 codes and ISO-639-2 codes.
 
 
-Examples
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- Since Marian models are smaller than many other translation models available in the library, they can be useful for
-  fine-tuning experiments and integration tests.
-- `Fine-tune on TPU
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/builtin_trainer/train_distil_marian_enro_tpu.sh>`__
-- `Fine-tune on GPU
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/builtin_trainer/train_distil_marian_enro.sh>`__
-- `Fine-tune on GPU with pytorch-lightning
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/distil_marian_no_teacher.sh>`__
-
 Multilingual Models
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- All model names use the following format: :obj:`Helsinki-NLP/opus-mt-{src}-{tgt}`:
-- If a model can output multiple languages, and you should specify a language code by prepending the desired output
-  language to the :obj:`src_text`.
-- You can see a models's supported language codes in its model card, under target constituents, like in `opus-mt-en-roa
-  <https://huggingface.co/Helsinki-NLP/opus-mt-en-roa>`__.
-- Note that if a model is only multilingual on the source side, like :obj:`Helsinki-NLP/opus-mt-roa-en`, no language
-  codes are required.
+All model names use the following format: :obj:`Helsinki-NLP/opus-mt-{src}-{tgt}`:
 
-New multi-lingual models from the `Tatoeba-Challenge repo <https://github.com/Helsinki-NLP/Tatoeba-Challenge>`__
-require 3 character language codes:
+    - If :obj:`src` is in all caps, the model supports multiple input languages, you can figure out which ones by
+      looking at the model card, or the Group Members `mapping
+      <https://gist.github.com/sshleifer/6d20e7761931b08e73c3219027b97b8a>`_ .
+    - If :obj:`tgt` is in all caps, the model can output multiple languages, and you should specify a language code by
+      prepending the desired output language to the :obj:`src_text`.
+    - You can see a tokenizer's supported language codes in ``tokenizer.supported_language_codes``
+
+Example of translating english to many romance languages, using language codes:
 
 .. code-block:: python
 
-    from transformers import MarianMTModel, MarianTokenizer
-    src_text = [
-        '>>fra<< this is a sentence in english that we want to translate to french',
-        '>>por<< This should go to portuguese',
-        '>>esp<< And this to Spanish'
-    ]
-
-    model_name = 'Helsinki-NLP/opus-mt-en-roa'
-    tokenizer = MarianTokenizer.from_pretrained(model_name)
-    print(tokenizer.supported_language_codes)
-    model = MarianMTModel.from_pretrained(model_name)
-    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text, return_tensors="pt"))
-    tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
-    # ["c'est une phrase en anglais que nous voulons traduire en français",
-    # 'Isto deve ir para o português.',
-    # 'Y esto al español']
-
-
-
-
-Code to see available pretrained models:
-
-.. code-block:: python
-
-    from transformers.hf_api import HfApi
-    model_list = HfApi().model_list()
-    org = "Helsinki-NLP"
-    model_ids = [x.modelId for x in model_list if x.modelId.startswith(org)]
-    suffix = [x.split('/')[1] for x in model_ids]
-    old_style_multi_models = [f'{org}/{s}' for s in suffix if s != s.lower()]
-
-
-
-Old Style Multi-Lingual Models
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-These are the old style multi-lingual models ported from the OPUS-MT-Train repo: and the members of each language
-group:
-
-.. code-block:: python
-
-    ['Helsinki-NLP/opus-mt-NORTH_EU-NORTH_EU',
-     'Helsinki-NLP/opus-mt-ROMANCE-en',
-     'Helsinki-NLP/opus-mt-SCANDINAVIA-SCANDINAVIA',
-     'Helsinki-NLP/opus-mt-de-ZH',
-     'Helsinki-NLP/opus-mt-en-CELTIC',
-     'Helsinki-NLP/opus-mt-en-ROMANCE',
-     'Helsinki-NLP/opus-mt-es-NORWAY',
-     'Helsinki-NLP/opus-mt-fi-NORWAY',
-     'Helsinki-NLP/opus-mt-fi-ZH',
-     'Helsinki-NLP/opus-mt-fi_nb_no_nn_ru_sv_en-SAMI',
-     'Helsinki-NLP/opus-mt-sv-NORWAY',
-     'Helsinki-NLP/opus-mt-sv-ZH']
-    GROUP_MEMBERS = {
-     'ZH': ['cmn', 'cn', 'yue', 'ze_zh', 'zh_cn', 'zh_CN', 'zh_HK', 'zh_tw', 'zh_TW', 'zh_yue', 'zhs', 'zht', 'zh'],
-     'ROMANCE': ['fr', 'fr_BE', 'fr_CA', 'fr_FR', 'wa', 'frp', 'oc', 'ca', 'rm', 'lld', 'fur', 'lij', 'lmo', 'es', 'es_AR', 'es_CL', 'es_CO', 'es_CR', 'es_DO', 'es_EC', 'es_ES', 'es_GT', 'es_HN', 'es_MX', 'es_NI', 'es_PA', 'es_PE', 'es_PR', 'es_SV', 'es_UY', 'es_VE', 'pt', 'pt_br', 'pt_BR', 'pt_PT', 'gl', 'lad', 'an', 'mwl', 'it', 'it_IT', 'co', 'nap', 'scn', 'vec', 'sc', 'ro', 'la'],
-     'NORTH_EU': ['de', 'nl', 'fy', 'af', 'da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
-     'SCANDINAVIA': ['da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
-     'SAMI': ['se', 'sma', 'smj', 'smn', 'sms'],
-     'NORWAY': ['nb_NO', 'nb', 'nn_NO', 'nn', 'nog', 'no_nb', 'no'],
-     'CELTIC': ['ga', 'cy', 'br', 'gd', 'kw', 'gv']
-    }
-
-
-
-
-Example of translating english to many romance languages, using old-style 2 character language codes
-
-
-.. code-block::python
-
     from transformers import MarianMTModel, MarianTokenizer
     src_text = [
         '>>fr<< this is a sentence in english that we want to translate to french',
@@ -148,12 +63,52 @@ Example of translating english to many romance languages, using old-style 2 char
     model_name = 'Helsinki-NLP/opus-mt-en-ROMANCE'
     tokenizer = MarianTokenizer.from_pretrained(model_name)
     print(tokenizer.supported_language_codes)
-
     model = MarianMTModel.from_pretrained(model_name)
-    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text, return_tensors="pt"))
+    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text))
     tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
-    # ["c'est une phrase en anglais que nous voulons traduire en français", 'Isto deve ir para o português.',  'Y esto al español']
+    # ["c'est une phrase en anglais que nous voulons traduire en français",
+    # 'Isto deve ir para o português.',
+    # 'Y esto al español']
 
+Sometimes, models were trained on collections of languages that do not resolve to a group. In this case, _ is used as a
+separator for src or tgt, as in :obj:`Helsinki-NLP/opus-mt-en_el_es_fi-en_el_es_fi`. These still require language
+codes.
+
+There are many supported regional language codes, like :obj:`>>es_ES<<` (Spain) and :obj:`>>es_AR<<` (Argentina), that
+do not seem to change translations. I have not found these to provide different results than just using :obj:`>>es<<`.
+
+For example:
+
+    - `Helsinki-NLP/opus-mt-NORTH_EU-NORTH_EU`: translates from all NORTH_EU languages (see `mapping
+      <https://gist.github.com/sshleifer/6d20e7761931b08e73c3219027b97b8a>`_) to all NORTH_EU languages. Use a special
+      language code like :obj:`>>de<<` to specify output language.
+    - `Helsinki-NLP/opus-mt-ROMANCE-en`: translates from many romance languages to english, no codes needed since there
+      is only one target language.
+
+
+
+.. code-block:: python
+
+    GROUP_MEMBERS = {
+     'ZH': ['cmn', 'cn', 'yue', 'ze_zh', 'zh_cn', 'zh_CN', 'zh_HK', 'zh_tw', 'zh_TW', 'zh_yue', 'zhs', 'zht', 'zh'],
+     'ROMANCE': ['fr', 'fr_BE', 'fr_CA', 'fr_FR', 'wa', 'frp', 'oc', 'ca', 'rm', 'lld', 'fur', 'lij', 'lmo', 'es', 'es_AR', 'es_CL', 'es_CO', 'es_CR', 'es_DO', 'es_EC', 'es_ES', 'es_GT', 'es_HN', 'es_MX', 'es_NI', 'es_PA', 'es_PE', 'es_PR', 'es_SV', 'es_UY', 'es_VE', 'pt', 'pt_br', 'pt_BR', 'pt_PT', 'gl', 'lad', 'an', 'mwl', 'it', 'it_IT', 'co', 'nap', 'scn', 'vec', 'sc', 'ro', 'la'],
+     'NORTH_EU': ['de', 'nl', 'fy', 'af', 'da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
+     'SCANDINAVIA': ['da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
+     'SAMI': ['se', 'sma', 'smj', 'smn', 'sms'],
+     'NORWAY': ['nb_NO', 'nb', 'nn_NO', 'nn', 'nog', 'no_nb', 'no'],
+     'CELTIC': ['ga', 'cy', 'br', 'gd', 'kw', 'gv']
+    }
+
+Code to see available pretrained models:
+
+.. code-block:: python
+
+    from transformers.hf_api import HfApi
+    model_list = HfApi().model_list()
+    org = "Helsinki-NLP"
+    model_ids = [x.modelId for x in model_list if x.modelId.startswith(org)]
+    suffix = [x.split('/')[1] for x in model_ids]
+    multi_models = [f'{org}/{s}' for s in suffix if s != s.lower()]
 
 
 MarianConfig

docs/source/model_doc/mbart.rst

@@ -3,7 +3,7 @@ MBart
 
 **DISCLAIMER:** If you see something strange, file a `Github Issue
 <https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ and assign
-@patrickvonplaten
+@sshleifer
 
 Overview
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -19,13 +19,6 @@ on the encoder, decoder, or reconstructing parts of the text.
 
 The Authors' code can be found `here <https://github.com/pytorch/fairseq/tree/master/examples/mbart>`__
 
-Examples
-_______________________________________________________________________________________________________________________
-
-- Examples and scripts for fine-tuning mBART and other models for sequence to sequence tasks can be found in
-  `examples/seq2seq/ <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
-- Given the large embeddings table, mBART consumes a large amount of GPU RAM, especially for fine-tuning.
-  :class:`MarianMTModel` is usually a better choice for bilingual machine translation.
 
 Training
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -44,8 +37,12 @@ the sequences for sequence-to-sequence fine-tuning.
 
     example_english_phrase = "UN Chief Says There Is No Military Solution in Syria"
     expected_translation_romanian = "Şeful ONU declară că nu există o soluţie militară în Siria"
-    batch = tokenizer.prepare_seq2seq_batch(example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian, return_tensors="pt")
-    model(input_ids=batch['input_ids'], labels=batch['labels']) # forward pass
+    batch = tokenizer.prepare_seq2seq_batch(example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian)
+    input_ids = batch["input_ids"]
+    target_ids = batch["decoder_input_ids"]
+    decoder_input_ids = target_ids[:, :-1].contiguous()
+    labels = target_ids[:, 1:].clone()
+    model(input_ids=input_ids, decoder_input_ids=decoder_input_ids, labels=labels) #forward
 
 - Generation
 
@@ -58,7 +55,7 @@ the sequences for sequence-to-sequence fine-tuning.
     model = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-en-ro")
     tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-en-ro")
     article = "UN Chief Says There Is No Military Solution in Syria"
-    batch = tokenizer.prepare_seq2seq_batch(src_texts=[article], src_lang="en_XX", return_tensors="pt")
+    batch = tokenizer.prepare_seq2seq_batch(src_texts=[article], src_lang="en_XX")
     translated_tokens = model.generate(**batch, decoder_start_token_id=tokenizer.lang_code_to_id["ro_RO"])
     translation = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0]
     assert translation == "Şeful ONU declară că nu există o soluţie militară în Siria"

docs/source/model_doc/mobilebert.rst

@@ -58,10 +58,10 @@ MobileBertTokenizerFast
 MobileBert specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.mobilebert.modeling_mobilebert.MobileBertForPreTrainingOutput
+.. autoclass:: transformers.modeling_mobilebert.MobileBertForPreTrainingOutput
     :members:
 
-.. autoclass:: transformers.models.mobilebert.modeling_tf_mobilebert.TFMobileBertForPreTrainingOutput
+.. autoclass:: transformers.modeling_tf_mobilebert.TFMobileBertForPreTrainingOutput
     :members:
 
 

docs/source/model_doc/mt5.rst

@@ -1,53 +0,0 @@
-MT5
------------------------------------------------------------------------------------------------------------------------
-
-Overview
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The mT5 model was presented in `mT5: A massively multilingual pre-trained text-to-text transformer
-<https://arxiv.org/abs/2010.11934>`_ by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya
-Siddhant, Aditya Barua, Colin Raffel.
-
-The abstract from the paper is the following:
-
-*The recent "Text-to-Text Transfer Transformer" (T5) leveraged a unified text-to-text format and scale to attain
-state-of-the-art results on a wide variety of English-language NLP tasks. In this paper, we introduce mT5, a
-multilingual variant of T5 that was pre-trained on a new Common Crawl-based dataset covering 101 languages. We describe
-the design and modified training of mT5 and demonstrate its state-of-the-art performance on many multilingual
-benchmarks. All of the code and model checkpoints*
-
-The original code can be found `here <https://github.com/google-research/multilingual-t5>`__.
-
-MT5Config
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5Config
-    :members:
-
-
-MT5Model
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5Model
-    :members:
-
-
-MT5ForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MT5ForConditionalGeneration
-    :members:
-
-
-TFMT5Model
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFMT5Model
-    :members:
-
-
-TFMT5ForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFMT5ForConditionalGeneration
-    :members:

docs/source/model_doc/pegasus.rst

@@ -3,7 +3,7 @@ Pegasus
 
 **DISCLAIMER:** If you see something strange, file a `Github Issue
 <https://github.com/huggingface/transformers/issues/new?assignees=sshleifer&labels=&template=bug-report.md&title>`__
-and assign @patrickvonplaten.
+and assign @sshleifer.
 
 
 Overview
@@ -31,19 +31,10 @@ All the `checkpoints <https://huggingface.co/models?search=pegasus>`__ are fine-
 - Each checkpoint is 2.2 GB on disk and 568M parameters.
 - FP16 is not supported (help/ideas on this appreciated!).
 - Summarizing xsum in fp32 takes about 400ms/sample, with default parameters on a v100 GPU.
-- Full replication results and correctly pre-processed data can be found in this `Issue
-  <https://github.com/huggingface/transformers/issues/6844#issue-689259666>`__.
-- `Distilled checkpoints <https://huggingface.co/models?search=distill-pegasus>`__ are described in this `paper
-  <https://arxiv.org/abs/2010.13002>`__.
+- For XSUM, The paper reports rouge1,rouge2, rougeL of paper: 47.21/24.56/39.25. As of Aug 9, this port scores
+  46.91/24.34/39.1.
 
-Examples
-_______________________________________________________________________________________________________________________
-
-- `Script <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/finetune_pegasus_xsum.sh>`__ to
-  fine-tune pegasus on the XSUM dataset. Data download instructions at `examples/seq2seq/
-  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
-- FP16 is not supported (help/ideas on this appreciated!).
-- The adafactor optimizer is recommended for pegasus fine-tuning.
+The gap is likely because of different alpha/length_penalty implementations in beam search.
 
 
 Implementation Notes
@@ -54,7 +45,7 @@ Implementation Notes
 - Some key configuration differences:
 
     - static, sinusoidal position embeddings
-    - no :obj:`layernorm_embedding` (:obj:`PegasusConfig.normalize_embedding=False`)
+    - no :obj:`layernorm_embedding` (:obj`PegasusConfig.normalize_embedding=False`)
     - the model starts generating with pad_token_id (which has 0 token_embedding) as the prefix.
     - more beams are used (:obj:`num_beams=8`)
 - All pretrained pegasus checkpoints are the same besides three attributes: :obj:`tokenizer.model_max_length` (maximum
@@ -78,7 +69,7 @@ Usage Example
     torch_device = 'cuda' if torch.cuda.is_available() else 'cpu'
     tokenizer = PegasusTokenizer.from_pretrained(model_name)
     model = PegasusForConditionalGeneration.from_pretrained(model_name).to(torch_device)
-    batch = tokenizer.prepare_seq2seq_batch(src_text, truncation=True, padding='longest', return_tensors="pt").to(torch_device)
+    batch = tokenizer.prepare_seq2seq_batch(src_text, truncation=True, padding='longest').to(torch_device)
     translated = model.generate(**batch)
     tgt_text = tokenizer.batch_decode(translated, skip_special_tokens=True)
     assert tgt_text[0] == "California's largest electricity provider has turned off power to hundreds of thousands of customers."

docs/source/model_doc/prophetnet.rst

@@ -47,16 +47,16 @@ ProphetNetTokenizer
 ProphetNet specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetSeq2SeqLMOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqLMOutput
     :members:
 
-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetSeq2SeqModelOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetSeq2SeqModelOutput
     :members:
 
-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetDecoderModelOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderModelOutput
     :members:
 
-.. autoclass:: transformers.models.prophetnet.modeling_prophetnet.ProphetNetDecoderLMOutput
+.. autoclass:: transformers.modeling_prophetnet.ProphetNetDecoderLMOutput
     :members:
 
 ProphetNetModel

docs/source/model_doc/rag.rst

@@ -50,10 +50,10 @@ RagTokenizer
 Rag specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.rag.modeling_rag.RetrievAugLMMarginOutput
+.. autoclass:: transformers.modeling_rag.RetrievAugLMMarginOutput
     :members:
 
-.. autoclass:: transformers.models.rag.modeling_rag.RetrievAugLMOutput
+.. autoclass:: transformers.modeling_rag.RetrievAugLMOutput
     :members:
 
 RagRetriever

docs/source/model_doc/roberta.rst

@@ -146,10 +146,3 @@ TFRobertaForQuestionAnswering
 
 .. autoclass:: transformers.TFRobertaForQuestionAnswering
     :members: call
-
-
-FlaxRobertaModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.FlaxRobertaModel
-    :members: __call__

docs/source/model_doc/t5.rst

@@ -64,7 +64,7 @@ token. T5 can be trained / fine-tuned both in a supervised and unsupervised fash
   input_ids = tokenizer('The <extra_id_0> walks in <extra_id_1> park', return_tensors='pt').input_ids
   labels = tokenizer('<extra_id_0> cute dog <extra_id_1> the <extra_id_2>', return_tensors='pt').input_ids
   # the forward function automatically creates the correct decoder_input_ids
-  loss = model(input_ids=input_ids, labels=labels).loss
+  loss = model(input_ids=input_ids, labels=labels, return_dict=True).loss
 
 - Supervised training
 
@@ -77,7 +77,7 @@ token. T5 can be trained / fine-tuned both in a supervised and unsupervised fash
   input_ids = tokenizer('translate English to German: The house is wonderful.', return_tensors='pt').input_ids
   labels = tokenizer('Das Haus ist wunderbar.', return_tensors='pt').input_ids
   # the forward function automatically creates the correct decoder_input_ids
-  loss = model(input_ids=input_ids, labels=labels).loss
+  loss = model(input_ids=input_ids, labels=labels, return_dict=True).loss
 
 
 T5Config

docs/source/model_doc/transformerxl.rst

@@ -49,16 +49,16 @@ TransfoXLTokenizer
 TransfoXL specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.transfo_xl.modeling_transfo_xl.TransfoXLModelOutput
+.. autoclass:: transformers.modeling_transfo_xl.TransfoXLModelOutput
     :members:
 
-.. autoclass:: transformers.models.transfo_xl.modeling_transfo_xl.TransfoXLLMHeadModelOutput
+.. autoclass:: transformers.modeling_transfo_xl.TransfoXLLMHeadModelOutput
     :members:
 
-.. autoclass:: transformers.models.transfo_xl.modeling_tf_transfo_xl.TFTransfoXLModelOutput
+.. autoclass:: transformers.modeling_tf_transfo_xl.TFTransfoXLModelOutput
     :members:
 
-.. autoclass:: transformers.models.transfo_xl.modeling_tf_transfo_xl.TFTransfoXLLMHeadModelOutput
+.. autoclass:: transformers.modeling_tf_transfo_xl.TFTransfoXLLMHeadModelOutput
     :members:
 
 

docs/source/model_doc/xlm.rst

@@ -50,7 +50,7 @@ XLMTokenizer
 XLM specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.xlm.modeling_xlm.XLMForQuestionAnsweringOutput
+.. autoclass:: transformers.modeling_xlm.XLMForQuestionAnsweringOutput
     :members:
 
 

docs/source/model_doc/xlnet.rst

@@ -53,43 +53,43 @@ XLNetTokenizer
 XLNet specific outputs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetModelOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetModelOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetLMHeadModelOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetLMHeadModelOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForSequenceClassificationOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForSequenceClassificationOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForMultipleChoiceOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForMultipleChoiceOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForTokenClassificationOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForTokenClassificationOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForQuestionAnsweringSimpleOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForQuestionAnsweringSimpleOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_xlnet.XLNetForQuestionAnsweringOutput
+.. autoclass:: transformers.modeling_xlnet.XLNetForQuestionAnsweringOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetModelOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetModelOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetLMHeadModelOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetLMHeadModelOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForSequenceClassificationOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForSequenceClassificationOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForMultipleChoiceOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForMultipleChoiceOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForTokenClassificationOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForTokenClassificationOutput
     :members:
 
-.. autoclass:: transformers.models.xlnet.modeling_tf_xlnet.TFXLNetForQuestionAnsweringSimpleOutput
+.. autoclass:: transformers.modeling_tf_xlnet.TFXLNetForQuestionAnsweringSimpleOutput
     :members:
 
 

docs/source/model_sharing.rst

@@ -18,76 +18,39 @@ done something similar on your task, either using the model directly in your own
 :class:`~.transformers.Trainer`/:class:`~.transformers.TFTrainer` class. Let's see how you can share the result on the
 `model hub <https://huggingface.co/models>`__.
 
-Model versioning
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Since version v3.5.0, the model hub has built-in model versioning based on git and git-lfs. It is based on the paradigm
-that one model *is* one repo.
-
-This allows:
-
-- built-in versioning
-- access control
-- scalability
-
-This is built around *revisions*, which is a way to pin a specific version of a model, using a commit hash, tag or
-branch.
-
-For instance:
-
-.. code-block::
-
-    >>> model = AutoModel.from_pretrained(
-    >>>   "julien-c/EsperBERTo-small",
-    >>>   revision="v2.0.1" # tag name, or branch name, or commit hash
-    >>> )
-
 Basic steps
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In order to upload a model, you'll need to first create a git repo. This repo will live on the model hub, allowing
-users to clone it and you (and your organization members) to push to it.
+.. 
+    When #5258 is merged, we can remove the need to create the directory.
 
-You can create a model repo directly from the website, `here <https://huggingface.co/new>`.
+First, pick a directory with the name you want your model to have on the model hub (its full name will then be
+`username/awesome-name-you-picked` or `organization/awesome-name-you-picked`) and create it with either
 
-Alternatively, you can use the ``transformers-cli``. The next steps describe that process:
+.. code-block::
 
-Go to a terminal and run the following command. It should be in the virtual environment where you installed 🤗
-Transformers, since that command :obj:`transformers-cli` comes from the library.
+    mkdir path/to/awesome-name-you-picked
 
-.. code-block:: bash
+or in python
 
-    transformers-cli login
+.. code-block::
 
+    import os
+    os.makedirs("path/to/awesome-name-you-picked")
 
-Once you are logged in with your model hub credentials, you can start building your repositories. To create a repo:
+then you can save your model and tokenizer with:
 
-.. code-block:: bash
+.. code-block::
 
-    transformers-cli repo create your-model-name
+    model.save_pretrained("path/to/awesome-name-you-picked")
+    tokenizer.save_pretrained("path/to/awesome-name-you-picked")
 
-This creates a repo on the model hub, which can be cloned.
+Or, if you're using the Trainer API
 
-.. code-block:: bash
-
-    git clone https://huggingface.co/username/your-model-name
-
-    # Make sure you have git-lfs installed
-    # (https://git-lfs.github.com/)
-    git lfs install
-
-When you have your local clone of your repo and lfs installed, you can then add/remove from that clone as you would
-with any other git repo.
-
-.. code-block:: bash
-
-    # Commit as usual
-    cd your-model-name
-    echo "hello" >> README.md
-    git add . && git commit -m "Update from $USER"
-
-We are intentionally not wrapping git too much, so as to stay intuitive and easy-to-use.
+.. code-block::
 
+    trainer.save_model("path/to/awesome-name-you-picked")
+    tokenizer.save_pretrained("path/to/awesome-name-you-picked")
 
 Make your model work on all frameworks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -108,13 +71,13 @@ or removing TF. For instance, if you trained a :class:`~transformers.DistilBertF
 
 .. code-block::
 
-    >>> from transformers import TFDistilBertForSequenceClassification
+    from transformers import TFDistilBertForSequenceClassification
 
 and if you trained a :class:`~transformers.TFDistilBertForSequenceClassification`, try to type
 
 .. code-block::
 
-    >>> from transformers import DistilBertForSequenceClassification
+    from transformers import DistilBertForSequenceClassification
 
 This will give back an error if your model does not exist in the other framework (something that should be pretty rare
 since we're aiming for full parity between the two frameworks). In this case, skip this and go to the next step.
@@ -124,20 +87,20 @@ model class:
 
 .. code-block::
 
-    >>> tf_model = TFDistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_pt=True)
-    >>> tf_model.save_pretrained("path/to/awesome-name-you-picked")
+    tf_model = TFDistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_pt=True)
+    tf_model.save_pretrained("path/to/awesome-name-you-picked")
 
 and if you trained your model in TensorFlow and have to create a PyTorch version, adapt the following code to your
 model class:
 
 .. code-block::
 
-    >>> pt_model = DistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_tf=True)
-    >>> pt_model.save_pretrained("path/to/awesome-name-you-picked")
+    pt_model = DistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_tf=True)
+    pt_model.save_pretrained("path/to/awesome-name-you-picked")
 
 That's all there is to it!
 
-Check the directory before pushing to the model hub.
+Check the directory before uploading
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Make sure there are no garbage files in the directory you'll upload. It should only have:
@@ -153,47 +116,62 @@ Make sure there are no garbage files in the directory you'll upload. It should o
 
 Other files can safely be deleted.
 
-
-Uploading your files
+Upload your model with the CLI
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Once the repo is cloned, you can add the model, configuration and tokenizer files. For instance, saving the model and
-tokenizer files:
+Now go in a terminal and run the following command. It should be in the virtual environment where you installed 🤗
+Transformers, since that command :obj:`transformers-cli` comes from the library.
 
 .. code-block::
 
-    >>> model.save_pretrained("path/to/repo/clone/your-model-name")
-    >>> tokenizer.save_pretrained("path/to/repo/clone/your-model-name")
+    transformers-cli login
 
-Or, if you're using the Trainer API
+Then log in using the same credentials as on huggingface.co. To upload your model, just type
 
 .. code-block::
 
-    >>> trainer.save_model("path/to/awesome-name-you-picked")
-    >>> tokenizer.save_pretrained("path/to/repo/clone/your-model-name")
+    transformers-cli upload path/to/awesome-name-you-picked/
 
-You can then add these files to the staging environment and verify that they have been correctly staged with the ``git
-status`` command:
+This will upload the folder containing the weights, tokenizer and configuration we prepared in the previous section.
 
-.. code-block:: bash
+By default you will be prompted to confirm that you want these files to be uploaded. If you are uploading multiple
+models and need to script that process, you can add `-y` to bypass the prompt. For example:
 
-    git add --all
-    git status
+.. code-block::
 
-Finally, the files should be comitted:
+    transformers-cli upload -y path/to/awesome-name-you-picked/
 
-.. code-block:: bash
 
-    git commit -m "First version of the your-model-name model and tokenizer."
+If you want to upload a single file (a new version of your model, or the other framework checkpoint you want to add),
+just type:
 
-And pushed to the remote:
+.. code-block::
 
-.. code-block:: bash
+    transformers-cli upload path/to/awesome-name-you-picked/that-file 
 
-    git push
+or
 
-This will upload the folder containing the weights, tokenizer and configuration we have just prepared.
+.. code-block::
 
+   transformers-cli upload path/to/awesome-name-you-picked/that-file --filename awesome-name-you-picked/new_name
+
+if you want to change its filename.
+
+This uploads the model to your personal account. If you want your model to be namespaced by your organization name
+rather than your username, add the following flag to any command:
+
+.. code-block::
+
+    --organization organization_name
+
+so for instance:
+
+.. code-block::
+
+    transformers-cli upload path/to/awesome-name-you-picked/ --organization organization_name
+
+Your model will then be accessible through its identifier, which is, as we saw above,
+`username/awesome-name-you-picked` or `organization/awesome-name-you-picked`.
 
 Add a model card
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -211,7 +189,7 @@ don't forget to link to its model card so that people can fully trace how your m
 If you have never made a pull request to the 🤗 Transformers repo, look at the :doc:`contributing guide <contributing>`
 to see the steps to follow.
 
-.. note::
+.. Note::
 
     You can also send your model card in the folder you uploaded with the CLI by placing it in a `README.md` file
     inside `path/to/awesome-name-you-picked/`.
@@ -225,59 +203,20 @@ Anyone can load it from code:
 
 .. code-block::
 
-    >>> tokenizer = AutoTokenizer.from_pretrained("namespace/awesome-name-you-picked")
-    >>> model = AutoModel.from_pretrained("namespace/awesome-name-you-picked")
+    tokenizer = AutoTokenizer.from_pretrained("namespace/awesome-name-you-picked")
+    model = AutoModel.from_pretrained("namespace/awesome-name-you-picked")
 
+Additional commands
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-You may specify a revision by using the ``revision`` flag in the ``from_pretrained`` method:
+You can list all the files you uploaded on the hub like this:
 
 .. code-block::
 
-    >>> tokenizer = AutoTokenizer.from_pretrained(
-    >>>   "julien-c/EsperBERTo-small",
-    >>>   revision="v2.0.1" # tag name, or branch name, or commit hash
-    >>> )
+    transformers-cli s3 ls
 
-Workflow in a Colab notebook
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+You can also delete unneeded files with
 
-If you're in a Colab notebook (or similar) with no direct access to a terminal, here is the workflow you can use to
-upload your model. You can execute each one of them in a cell by adding a ! at the beginning.
+.. code-block::
 
-First you need to install `git-lfs` in the environment used by the notebook:
-
-.. code-block:: bash
-
-    sudo apt-get install git-lfs
-
-Then you can use the :obj:`transformers-cli` to create your new repo:
-
-
-.. code-block:: bash
-
-    transformers-cli login
-    transformers-cli repo create your-model-name
-
-Once it's created, you can clone it and configure it (replace username by your username on huggingface.co):
-
-.. code-block:: bash
-
-    git clone https://username:password@huggingface.co/username/your-model-name
-    # Alternatively if you have a token,
-    # you can use it instead of your password
-    git clone https://username:token@huggingface.co/username/your-model-name
-
-    cd your-model-name
-    git lfs install
-    git config --global user.email "email@example.com"
-    # Tip: using the same email than for your huggingface.co account will link your commits to your profile
-    git config --global user.name "Your name"
-
-Once you've saved your model inside, and your clone is setup with the right remote URL, you can add it and push it with
-usual git commands.
-
-.. code-block:: bash
-
-    git add .
-    git commit -m "Initial commit"
-    git push
+    transformers-cli s3 rm awesome-name-you-picked/filename

docs/source/model_summary.rst

@@ -560,7 +560,6 @@ A framework for translation models, using the same models as BART
 
 The library provides a version of this model for conditional generation.
 
-
 T5
 -----------------------------------------------------------------------------------------------------------------------
 
@@ -593,28 +592,6 @@ For instance, if we have the sentence “My dog is very cute .”, and we decide
 
 The library provides a version of this model for conditional generation.
 
-
-MT5
------------------------------------------------------------------------------------------------------------------------
-
-.. raw:: html
-
-   <a href="https://huggingface.co/models?filter=mt5">
-       <img alt="Models" src="https://img.shields.io/badge/All_model_pages-mt5-blueviolet">
-   </a>
-   <a href="model_doc/mt5.html">
-       <img alt="Doc" src="https://img.shields.io/badge/Model_documentation-mt5-blueviolet">
-   </a>
-
-`mT5: A massively multilingual pre-trained text-to-text transformer <https://arxiv.org/abs/2010.11934>`_, Linting Xue
-et al.
-
-The model architecture is same as T5. mT5's pre-training objective includes T5's self-supervised training, but not T5's
-supervised training. mT5 is trained on 101 languages.
-
-The library provides a version of this model for conditional generation.
-
-
 MBart
 -----------------------------------------------------------------------------------------------------------------------
 

docs/source/pretrained_models.rst

@@ -3,11 +3,11 @@ Pretrained models
 
 Here is the full list of the currently provided pretrained models together with a short presentation of each model.
 
-For a list that includes all community-uploaded models, refer to `https://huggingface.co/models
+For a list that includes community-uploaded models, refer to `https://huggingface.co/models
 <https://huggingface.co/models>`__.
 
 +--------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
-| Architecture       | Model id                                                   | Details of the model                                                                                                                  |
+| Architecture       | Shortcut name                                              | Details of the model                                                                                                                  |
 +====================+============================================================+=======================================================================================================================================+
 | BERT               | ``bert-base-uncased``                                      | | 12-layer, 768-hidden, 12-heads, 110M parameters.                                                                                    |
 |                    |                                                            | | Trained on lower-cased English text.                                                                                                |

docs/source/task_summary.rst

@@ -89,7 +89,7 @@ each other. The process is the following:
     >>> import torch
 
     >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased-finetuned-mrpc")
-    >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc")
+    >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc", return_dict=True)
 
     >>> classes = ["not paraphrase", "is paraphrase"]
 
@@ -122,7 +122,7 @@ each other. The process is the following:
     >>> import tensorflow as tf
 
     >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased-finetuned-mrpc")
-    >>> model = TFAutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc")
+    >>> model = TFAutoModelForSequenceClassification.from_pretrained("bert-base-cased-finetuned-mrpc", return_dict=True)
 
     >>> classes = ["not paraphrase", "is paraphrase"]
 
@@ -211,7 +211,7 @@ Here is an example of question answering using a model and a tokenizer. The proc
     >>> import torch
 
     >>> tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
-    >>> model = AutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
+    >>> model = AutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad", return_dict=True)
 
     >>> text = r"""
     ... 🤗 Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides general-purpose
@@ -253,7 +253,7 @@ Here is an example of question answering using a model and a tokenizer. The proc
     >>> import tensorflow as tf
 
     >>> tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
-    >>> model = TFAutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
+    >>> model = TFAutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad", return_dict=True)
 
     >>> text = r"""
     ... 🤗 Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides general-purpose
@@ -373,7 +373,7 @@ Here is an example of doing masked language modeling using a model and a tokeniz
     >>> import torch
 
     >>> tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased")
-    >>> model = AutoModelWithLMHead.from_pretrained("distilbert-base-cased")
+    >>> model = AutoModelWithLMHead.from_pretrained("distilbert-base-cased", return_dict=True)
 
     >>> sequence = f"Distilled models are smaller than the models they mimic. Using them instead of the large versions would help {tokenizer.mask_token} our carbon footprint."
 
@@ -389,7 +389,7 @@ Here is an example of doing masked language modeling using a model and a tokeniz
     >>> import tensorflow as tf
 
     >>> tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased")
-    >>> model = TFAutoModelWithLMHead.from_pretrained("distilbert-base-cased")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("distilbert-base-cased", return_dict=True)
 
     >>> sequence = f"Distilled models are smaller than the models they mimic. Using them instead of the large versions would help {tokenizer.mask_token} our carbon footprint."
 
@@ -437,7 +437,7 @@ of tokens.
     >>> from torch.nn import functional as F
 
     >>> tokenizer = AutoTokenizer.from_pretrained("gpt2")
-    >>> model = AutoModelWithLMHead.from_pretrained("gpt2")
+    >>> model = AutoModelWithLMHead.from_pretrained("gpt2", return_dict=True)
 
     >>> sequence = f"Hugging Face is based in DUMBO, New York City, and "
 
@@ -461,7 +461,7 @@ of tokens.
     >>> import tensorflow as tf
 
     >>> tokenizer = AutoTokenizer.from_pretrained("gpt2")
-    >>> model = TFAutoModelWithLMHead.from_pretrained("gpt2")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("gpt2", return_dict=True)
 
     >>> sequence = f"Hugging Face is based in DUMBO, New York City, and "
 
@@ -513,14 +513,14 @@ Here, the model generates a random text with a total maximal length of *50* toke
 concerned, I will"*. The default arguments of ``PreTrainedModel.generate()`` can be directly overridden in the
 pipeline, as is shown above for the argument ``max_length``.
 
-Here is an example of text generation using ``XLNet`` and its tokenizer.
+Here is an example of text generation using ``XLNet`` and its tokenzier.
 
 .. code-block::
 
     >>> ## PYTORCH CODE
     >>> from transformers import AutoModelWithLMHead, AutoTokenizer
 
-    >>> model = AutoModelWithLMHead.from_pretrained("xlnet-base-cased")
+    >>> model = AutoModelWithLMHead.from_pretrained("xlnet-base-cased", return_dict=True)
     >>> tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")
 
     >>> # Padding text helps XLNet with short prompts - proposed by Aman Rusia in https://github.com/rusiaaman/XLNet-gen#methodology
@@ -545,7 +545,7 @@ Here is an example of text generation using ``XLNet`` and its tokenizer.
     >>> ## TENSORFLOW CODE
     >>> from transformers import TFAutoModelWithLMHead, AutoTokenizer
 
-    >>> model = TFAutoModelWithLMHead.from_pretrained("xlnet-base-cased")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("xlnet-base-cased", return_dict=True)
     >>> tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")
 
     >>> # Padding text helps XLNet with short prompts - proposed by Aman Rusia in https://github.com/rusiaaman/XLNet-gen#methodology
@@ -664,7 +664,7 @@ Here is an example of doing named entity recognition, using a model and a tokeni
     >>> from transformers import AutoModelForTokenClassification, AutoTokenizer
     >>> import torch
 
-    >>> model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english")
+    >>> model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english", return_dict=True)
     >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
 
     >>> label_list = [
@@ -692,7 +692,7 @@ Here is an example of doing named entity recognition, using a model and a tokeni
     >>> from transformers import TFAutoModelForTokenClassification, AutoTokenizer
     >>> import tensorflow as tf
 
-    >>> model = TFAutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english")
+    >>> model = TFAutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english", return_dict=True)
     >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
 
     >>> label_list = [
@@ -790,7 +790,7 @@ CNN / Daily Mail), it yields very good results.
     >>> ## PYTORCH CODE
     >>> from transformers import AutoModelWithLMHead, AutoTokenizer
 
-    >>> model = AutoModelWithLMHead.from_pretrained("t5-base")
+    >>> model = AutoModelWithLMHead.from_pretrained("t5-base", return_dict=True)
     >>> tokenizer = AutoTokenizer.from_pretrained("t5-base")
 
     >>> # T5 uses a max_length of 512 so we cut the article to 512 tokens.
@@ -799,7 +799,7 @@ CNN / Daily Mail), it yields very good results.
     >>> ## TENSORFLOW CODE
     >>> from transformers import TFAutoModelWithLMHead, AutoTokenizer
 
-    >>> model = TFAutoModelWithLMHead.from_pretrained("t5-base")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("t5-base", return_dict=True)
     >>> tokenizer = AutoTokenizer.from_pretrained("t5-base")
 
     >>> # T5 uses a max_length of 512 so we cut the article to 512 tokens.
@@ -834,7 +834,7 @@ Here is an example of doing translation using a model and a tokenizer. The proce
 
 1. Instantiate a tokenizer and a model from the checkpoint name. Summarization is usually done using an encoder-decoder
    model, such as ``Bart`` or ``T5``.
-2. Define the article that should be summarized.
+2. Define the article that should be summarizaed.
 3. Add the T5 specific prefix "translate English to German: "
 4. Use the ``PreTrainedModel.generate()`` method to perform the translation.
 
@@ -843,7 +843,7 @@ Here is an example of doing translation using a model and a tokenizer. The proce
     >>> ## PYTORCH CODE
     >>> from transformers import AutoModelWithLMHead, AutoTokenizer
 
-    >>> model = AutoModelWithLMHead.from_pretrained("t5-base")
+    >>> model = AutoModelWithLMHead.from_pretrained("t5-base", return_dict=True)
     >>> tokenizer = AutoTokenizer.from_pretrained("t5-base")
 
     >>> inputs = tokenizer.encode("translate English to German: Hugging Face is a technology company based in New York and Paris", return_tensors="pt")
@@ -851,7 +851,7 @@ Here is an example of doing translation using a model and a tokenizer. The proce
     >>> ## TENSORFLOW CODE
     >>> from transformers import TFAutoModelWithLMHead, AutoTokenizer
 
-    >>> model = TFAutoModelWithLMHead.from_pretrained("t5-base")
+    >>> model = TFAutoModelWithLMHead.from_pretrained("t5-base", return_dict=True)
     >>> tokenizer = AutoTokenizer.from_pretrained("t5-base")
 
     >>> inputs = tokenizer.encode("translate English to German: Hugging Face is a technology company based in New York and Paris", return_tensors="tf")

docs/source/testing.rst

@@ -405,32 +405,16 @@ decorators are used to set the requirements of tests CPU/GPU/TPU-wise:
 
 * ``require_torch`` - this test will run only under torch
 * ``require_torch_gpu`` - as ``require_torch`` plus requires at least 1 GPU
-* ``require_torch_multi_gpu`` - as ``require_torch`` plus requires at least 2 GPUs
-* ``require_torch_non_multi_gpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
+* ``require_torch_multigpu`` - as ``require_torch`` plus requires at least 2 GPUs
+* ``require_torch_non_multigpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
 * ``require_torch_tpu`` - as ``require_torch`` plus requires at least 1 TPU
 
-Let's depict the GPU requirements in the following table:
-
-
-+----------+----------------------------------+
-| n gpus   |  decorator                       |
-+==========+==================================+
-| ``>= 0`` | ``@require_torch``               |
-+----------+----------------------------------+
-| ``>= 1`` | ``@require_torch_gpu``           |
-+----------+----------------------------------+
-| ``>= 2`` | ``@require_torch_multi_gpu``     |
-+----------+----------------------------------+
-| ``< 2``  | ``@require_torch_non_multi_gpu`` |
-+----------+----------------------------------+
-
-
 For example, here is a test that must be run only when there are 2 or more GPUs available and pytorch is installed:
 
 .. code-block:: python
 
-    @require_torch_multi_gpu
-    def test_example_with_multi_gpu():
+    @require_torch_multigpu
+    def test_example_with_multigpu():
 
 If a test requires ``tensorflow`` use the ``require_tf`` decorator. For example:
 
@@ -454,7 +438,7 @@ last for them to work correctly. Here is an example of the correct usage:
 .. code-block:: python
 
     @parameterized.expand(...)
-    @require_torch_multi_gpu
+    @require_torch_multigpu
     def test_integration_foo():
 
 This order problem doesn't exist with ``@pytest.mark.parametrize``, you can put it first or last and it will still
@@ -466,8 +450,7 @@ Inside tests:
 
 .. code-block:: bash
 
-   from transformers.testing_utils import get_gpu_count
-   n_gpu = get_gpu_count() # works with torch and tf
+   torch.cuda.device_count()
 
 
 
@@ -716,11 +699,11 @@ Temporary files and directories
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Using unique temporary files and directories are essential for parallel test running, so that the tests won't overwrite
-each other's data. Also we want to get the temporary files and directories removed at the end of each test that created
+each other's data. Also we want to get the temp files and directories removed at the end of each test that created
 them. Therefore, using packages like ``tempfile``, which address these needs is essential.
 
-However, when debugging tests, you need to be able to see what goes into the temporary file or directory and you want
-to know it's exact path and not having it randomized on every test re-run.
+However, when debugging tests, you need to be able to see what goes into the temp file or directory and you want to
+know it's exact path and not having it randomized on every test re-run.
 
 A helper class :obj:`transformers.test_utils.TestCasePlus` is best used for such purposes. It's a sub-class of
 :obj:`unittest.TestCase`, so we can easily inherit from it in the test modules.
@@ -736,33 +719,32 @@ Here is an example of its usage:
 
 This code creates a unique temporary directory, and sets :obj:`tmp_dir` to its location.
 
-* Create a unique temporary dir:
+In this and all the following scenarios the temporary directory will be auto-removed at the end of test, unless
+``after=False`` is passed to the helper function.
+
+* Create a temporary directory of my choice and delete it at the end - useful for debugging when you want to monitor a
+  specific directory:
 
 .. code-block:: python
 
     def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir()
+        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test")
 
-``tmp_dir`` will contain the path to the created temporary dir. It will be automatically removed at the end of the
-test.
-
-* Create a temporary dir of my choice, ensure it's empty before the test starts and don't empty it after the test.
+* Create a temporary directory of my choice and do not delete it at the end---useful for when you want to look at the
+  temp results:
 
 .. code-block:: python
 
     def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir("./xxx")
+        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", after=False)
 
-This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn't
-leave any data in there.
+* Create a temporary directory of my choice and ensure to delete it right away---useful for when you disabled deletion
+  in the previous test run and want to make sure the that temporary directory is empty before the new test is run:
 
-* You can override the default behavior by directly overriding the ``before`` and ``after`` args, leading to one of the
-  following behaviors:
+.. code-block:: python
 
-    - ``before=True``: the temporary dir will always be cleared at the beginning of the test.
-    - ``before=False``: if the temporary dir already existed, any existing files will remain there.
-    - ``after=True``: the temporary dir will always be deleted at the end of the test.
-    - ``after=False``: the temporary dir will always be left intact at the end of the test.
+   def test_whatever(self):
+        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", before=True)
 
 .. note::
    In order to run the equivalent of ``rm -r`` safely, only subdirs of the project repository checkout are allowed if
@@ -816,7 +798,7 @@ or the ``xfail`` way:
     @pytest.mark.xfail
     def test_feature_x():
 
-- Here is how to skip a test based on some internal check inside the test:
+Here is how to skip a test based on some internal check inside the test:
 
 .. code-block:: python
 
@@ -839,7 +821,7 @@ or the ``xfail`` way:
     def test_feature_x():
         pytest.xfail("expected to fail until bug XYZ is fixed")
 
-- Here is how to skip all tests in a module if some import is missing:
+Here is how to skip all tests in a module if some import is missing:
 
 .. code-block:: python
 

docs/source/tokenizer_summary.rst

@@ -1,243 +1,223 @@
-Summary of the tokenizers
+Tokenizer summary
 -----------------------------------------------------------------------------------------------------------------------
 
-On this page, we will have a closer look at tokenization. As we saw in :doc:`the preprocessing tutorial
-<preprocessing>`, tokenizing a text is splitting it into words or subwords, which then are converted to ids through a
-look-up table. Converting words or subwords to ids is straightforward, so in this summary, we will focus on splitting a
-text into words or subwords (i.e. tokenizing a text). More specifically, we will look at the three main types of
-tokenizers used in 🤗 Transformers: :ref:`Byte-Pair Encoding (BPE) <byte-pair-encoding>`, :ref:`WordPiece <wordpiece>`,
-and :ref:`SentencePiece <sentencepiece>`, and show exemplary which tokenizer type is used by which model.
+In this page, we will have a closer look at tokenization. As we saw in :doc:`the preprocessing tutorial
+<preprocessing>`, tokenizing a text is splitting it into words or subwords, which then are converted to ids. The second
+part is pretty straightforward, here we will focus on the first part. More specifically, we will look at the three main
+different kinds of tokenizers used in 🤗 Transformers: :ref:`Byte-Pair Encoding (BPE) <byte-pair-encoding>`,
+:ref:`WordPiece <wordpiece>` and :ref:`SentencePiece <sentencepiece>`, and provide examples of models using each of
+those.
 
-Note that on each model page, you can look at the documentation of the associated tokenizer to know which tokenizer
-type was used by the pretrained model. For instance, if we look at :class:`~transformers.BertTokenizer`, we can see
-that the model uses :ref:`WordPiece <wordpiece>`.
+Note that on each model page, you can look at the documentation of the associated tokenizer to know which of those
+algorithms the pretrained model used. For instance, if we look at :class:`~transformers.BertTokenizer`, we can see it's
+using :ref:`WordPiece <wordpiece>`.
 
-Introduction
+Introduction to tokenization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Splitting a text into smaller chunks is a task that is harder than it looks, and there are multiple ways of doing so.
-For instance, let's look at the sentence ``"Don't you love 🤗 Transformers? We sure do."`` A simple way of tokenizing
-this text is to split it by spaces, which would give:
+Splitting a text in smaller chunks is a task that's harder than it looks, and there are multiple ways of doing it. For
+instance, let's look at the sentence "Don't you love 🤗 Transformers? We sure do." A first simple way of tokenizing this
+text is just to split it by spaces, which would give:
 
 .. code-block::
 
     ["Don't", "you", "love", "🤗", "Transformers?", "We", "sure", "do."]
 
-This is a sensible first step, but if we look at the tokens ``"Transformers?"`` and ``"do."``, we notice that the
-punctuation is attached to the words ``"Transformer"`` and ``"do"``, which is suboptimal. We should take the
-punctuation into account so that a model does not have to learn a different representation of a word and every possible
-punctuation symbol that could follow it, which would explode the number of representations the model has to learn.
-Taking punctuation into account, tokenizing our exemplary text would give:
+This is a nice first step, but if we look at the tokens "Transformers?" or "do.", we can see we can do better. Those
+will be different than the tokens "Transformers" and "do" for our model, so we should probably take the punctuation
+into account. This would give:
 
 .. code-block::
 
     ["Don", "'", "t", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]
 
-Better. However, it is disadvantageous, how the tokenization dealt with the word ``"Don't"``. ``"Don't"`` stands for
-``"do not"``, so it would be better tokenized as ``["Do", "n't"]``. This is where things start getting complicated, and
-part of the reason each model has its own tokenizer type. Depending on the rules we apply for tokenizing a text, a
-different tokenized output is generated for the same text. A pretrained model only performs properly if you feed it an
-input that was tokenized with the same rules that were used to tokenize its training data.
+which is better already. One thing that is annoying though is how it dealt with "Don't". "Don't" stands for do not, so
+it should probably be better tokenized as ``["Do", "n't"]``. This is where things start getting more complicated, and
+part of the reason each kind of model has its own tokenizer class. Depending on the rules we apply to split our texts
+into tokens, we'll get different tokenized versions of the same text. And of course, a given pretrained model won't
+perform properly if you don't use the exact same rules as the persons who pretrained it.
 
 `spaCy <https://spacy.io/>`__ and `Moses <http://www.statmt.org/moses/?n=Development.GetStarted>`__ are two popular
-rule-based tokenizers. Applying them on our example, *spaCy* and *Moses* would output something like:
+rule-based tokenizers. On the text above, they'd output something like:
 
 .. code-block::
 
     ["Do", "n't", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]
 
-As can be seen space and punctuation tokenization, as well as rule-based tokenization, is used here. Space and
-punctuation tokenization and rule-based tokenization are both examples of word tokenization, which is loosely defined
-as splitting sentences into words. While it's the most intuitive way to split texts into smaller chunks, this
-tokenization method can lead to problems for massive text corpora. In this case, space and punctuation tokenization
-usually generates a very big vocabulary (the set of all unique words and tokens used). *E.g.*, :doc:`Transformer XL
-<model_doc/transformerxl>` uses space and punctuation tokenization, resulting in a vocabulary size of 267,735!
+Space/punctuation-tokenization and rule-based tokenization are both examples of word tokenization, which is splitting a
+sentence into words. While it's the most intuitive way to separate texts in smaller chunks, it can have a problem when
+you have a huge corpus: it usually yields a very big vocabulary (the set of all unique tokens used). :doc:`Transformer
+XL <model_doc/transformerxl>` for instance uses space/punctuation-tokenization, and has a vocabulary size of 267,735!
 
-Such a big vocabulary size forces the model to have an enormous embedding matrix as the input and output layer, which
-causes both an increased memory and time complexity. In general, transformers models rarely have a vocabulary size
-greater than 50,000, especially if they are pretrained only on a single language.
+A huge vocabulary size means a huge embedding matrix at the start of the model, which will cause memory problems.
+TransformerXL deals with it by using a special kind of embeddings called adaptive embeddings, but in general,
+transformers models rarely have a vocabulary size greater than 50,000, especially if they are trained on a single
+language.
 
-So if simple space and punctuation tokenization is unsatisfactory, why not simply tokenize on characters? While
-character tokenization is very simple and would greatly reduce memory and time complexity it makes it much harder for
-the model to learn meaningful input representations. *E.g.* learning a meaningful context-independent representation
-for the letter ``"t"`` is much harder as learning a context-independent representation for the word ``"today"``.
-Therefore, character tokenization is often accompanied by a loss of performance. So to get the best of both worlds,
-transformers models use a hybrid between word-level and character-level tokenization called **subword** tokenization.
+So if tokenizing on words is unsatisfactory, we could go on the opposite direction and simply tokenize on characters.
+While it's very simple and would save a lot of memory, this doesn't allow the model to learn representations of texts
+as meaningful as when using a word tokenization, leading to a loss of performance. So to get the best of both worlds,
+all transformers models use a hybrid between word-level and character-level tokenization called subword tokenization.
 
 Subword tokenization
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Subword tokenization algorithms rely on the principle that frequently used words should not be split into smaller
-subwords, but rare words should be decomposed into meaningful subwords. For instance ``"annoyingly"`` might be
-considered a rare word and could be decomposed into ``"annoying"`` and ``"ly"``. Both ``"annoying"`` and ``"ly"`` as
-stand-alone subwords would appear more frequently while at the same time the meaning of ``"annoyingly"`` is kept by the
-composite meaning of ``"annoying"`` and ``"ly"``. This is especially useful in agglutinative languages such as Turkish,
-where you can form (almost) arbitrarily long complex words by stringing together subwords.
+Subword tokenization algorithms rely on the principle that most common words should be left as is, but rare words
+should be decomposed in meaningful subword units. For instance "annoyingly" might be considered a rare word and
+decomposed as "annoying" and "ly". This is especially useful in agglutinative languages such as Turkish, where you can
+form (almost) arbitrarily long complex words by stringing together some subwords.
 
-Subword tokenization allows the model to have a reasonable vocabulary size while being able to learn meaningful
-context-independent representations. In addition, subword tokenization enables the model to process words it has never
-seen before, by decomposing them into known subwords. For instance, the :class:`~transformers.BertTokenizer` tokenizes
-``"I have a new GPU!"`` as follows:
+This allows the model to keep a reasonable vocabulary while still learning useful representations for common words or
+subwords. This also enables the model to process words it has never seen before, by decomposing them into subwords it
+knows. For instance, the base :class:`~transformers.BertTokenizer` will tokenize "I have a new GPU!" like this:
 
 .. code-block::
 
     >>> from transformers import BertTokenizer
-    >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+    >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
     >>> tokenizer.tokenize("I have a new GPU!")
-    ["i", "have", "a", "new", "gp", "##u", "!"]
+    ['i', 'have', 'a', 'new', 'gp', '##u', '!']
 
-Because we are considering the uncased model, the sentence was lowercased first. We can see that the words ``["i",
-"have", "a", "new"]`` are present in the tokenizer's vocabulary, but the word ``"gpu"`` is not. Consequently, the
-tokenizer splits ``"gpu"`` into known subwords: ``["gp" and "##u"]``. ``"##"`` means that the rest of the token should
-be attached to the previous one, without space (for decoding or reversal of the tokenization).
+Since we are considering the uncased model, the sentence was lowercased first. Then all the words were present in the
+vocabulary of the tokenizer, except for "gpu", so the tokenizer splits it in subwords it knows: "gp" and "##u". The
+"##" means that the rest of the token should be attached to the previous one, without space (for when we need to decode
+predictions and reverse the tokenization).
 
-As another example, :class:`~transformers.XLNetTokenizer` tokenizes our previously exemplary text as follows:
+Another example is when we use the base :class:`~transformers.XLNetTokenizer` to tokenize our previous text:
 
 .. code-block::
 
     >>> from transformers import XLNetTokenizer
-    >>> tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased")
+    >>> tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
     >>> tokenizer.tokenize("Don't you love 🤗 Transformers? We sure do.")
-    ["▁Don", "'", "t", "▁you", "▁love", "▁", "🤗", "▁", "Transform", "ers", "?", "▁We", "▁sure", "▁do", "."]
+    ['▁Don', "'", 't', '▁you', '▁love', '▁', '🤗', '▁', 'Transform', 'ers', '?', '▁We', '▁sure', '▁do', '.']
 
-We'll get back to the meaning of those ``"▁"`` when we look at :ref:`SentencePiece <sentencepiece>`. As one can see,
-the rare word ``"Transformers"`` has been split into the more frequent subwords ``"Transform"`` and ``"ers"``.
+We'll get back to the meaning of those '▁' when we look at :ref:`SentencePiece <sentencepiece>` but you can see
+Transformers has been split into "Transform" and "ers".
 
-Let's now look at how the different subword tokenization algorithms work. Note that all of those tokenization
-algorithms rely on some form of training which is usually done on the corpus the corresponding model will be trained
-on.
+Let's now look at how the different subword tokenization algorithms work. Note that they all rely on some form of
+training which is usually done on the corpus the corresponding model will be trained on.
 
 .. _byte-pair-encoding:
 
-Byte-Pair Encoding (BPE)
+Byte-Pair Encoding
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Byte-Pair Encoding (BPE) was introduced in `Neural Machine Translation of Rare Words with Subword Units (Sennrich et
-al., 2015) <https://arxiv.org/abs/1508.07909>`__. BPE relies on a pre-tokenizer that splits the training data into
-words. Pretokenization can be as simple as space tokenization, e.g. :doc:`GPT-2 <model_doc/gpt2>`, :doc:`Roberta
-<model_doc/roberta>`. More advanced pre-tokenization include rule-based tokenization, e.g. :doc:`XLM <model_doc/xlm>`,
-:doc:`FlauBERT <model_doc/flaubert>` which uses Moses for most languages, or :doc:`GPT <model_doc/gpt>` which uses
-Spacy and ftfy, to count the frequency of each word in the training corpus.
+Byte-Pair Encoding was introduced in `this paper <https://arxiv.org/abs/1508.07909>`__. It relies on a pretokenizer
+splitting the training data into words, which can be a simple space tokenization (:doc:`GPT-2 <model_doc/gpt2>` and
+:doc:`Roberta <model_doc/roberta>` uses this for instance) or a rule-based tokenizer (:doc:`XLM <model_doc/xlm>` use
+Moses for most languages, as does :doc:`FlauBERT <model_doc/flaubert>`),
 
-After pre-tokenization, a set of unique words has been created and the frequency of each word it occurred in the
-training data has been determined. Next, BPE creates a base vocabulary consisting of all symbols that occur in the set
-of unique words and learns merge rules to form a new symbol from two symbols of the base vocabulary. It does so until
-the vocabulary has attained the desired vocabulary size. Note that the desired vocabulary size is a hyperparameter to
-define before training the tokenizer.
+:doc:`GPT <model_doc/gpt>` uses Spacy and ftfy, and counts the frequency of each word in the training corpus.
 
-As an example, let's assume that after pre-tokenization, the following set of words including their frequency has been
-determined:
+It then begins from the list of all characters and will learn merge rules to form a new token from two symbols in the
+vocabulary until it has learned a vocabulary of the desired size (this is a hyperparameter to pick).
+
+Let's say that after the pre-tokenization we have the following words (the number indicating the frequency of each
+word):
 
 .. code-block::
 
-    ("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)
+    ('hug', 10), ('pug', 5), ('pun', 12), ('bun', 4), ('hugs', 5)
 
-Consequently, the base vocabulary is ``["b", "g", "h", "n", "p", "s", "u"]``. Splitting all words into symbols of the
-base vocabulary, we obtain:
+Then the base vocabulary is ['b', 'g', 'h', 'n', 'p', 's', 'u'] and all our words are first split by character:
 
 .. code-block::
 
-    ("h" "u" "g", 10), ("p" "u" "g", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "u" "g" "s", 5)
+    ('h' 'u' 'g', 10), ('p' 'u' 'g', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'u' 'g' 's', 5)
 
-BPE then counts the frequency of each possible symbol pair and picks the symbol pair that occurs most frequently. In
-the example above ``"h"`` followed by ``"u"`` is present `10 + 5 = 15` times (10 times in the 10 occurrences of
-``"hug"``, 5 times in the 5 occurrences of "hugs"). However, the most frequent symbol pair is ``"u"`` followed by "g",
-occurring `10 + 5 + 5 = 20` times in total. Thus, the first merge rule the tokenizer learns is to group all ``"u"``
-symbols followed by a ``"g"`` symbol together. Next, "ug" is added to the vocabulary. The set of words then becomes
+We then take each pair of symbols and look at the most frequent. For instance 'hu' is present `10 + 5 = 15` times (10
+times in the 10 occurrences of 'hug', 5 times in the 5 occurrences of 'hugs'). The most frequent here is 'ug', present
+`10 + 5 + 5 = 20` times in total. So the first merge rule the tokenizer learns is to group all 'u' and 'g' together
+then it adds 'ug' to the vocabulary. Our corpus then becomes
 
 .. code-block::
 
-    ("h" "ug", 10), ("p" "ug", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "ug" "s", 5)
+    ('h' 'ug', 10), ('p' 'ug', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'ug' 's', 5)
 
-BPE then identifies the next most common symbol pair. It's ``"u"`` followed by ``"n"``, which occurs 16 times. ``"u"``,
-``"n"`` is merged to ``"un"`` and added to the vocabulary. The next most frequent symbol pair is ``"h"`` followed by
-``"ug"``, occurring 15 times. Again the pair is merged and ``"hug"`` can be added to the vocabulary.
+and we continue by looking at the next most common pair of symbols. It's 'un', present 16 times, so we merge those two
+and add 'un' to the vocabulary. Then it's 'hug' (as 'h' + 'ug'), present 15 times, so we merge those two and add 'hug'
+to the vocabulary.
 
-At this stage, the vocabulary is ``["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"]`` and our set of unique words
-is represented as
+At this stage, the vocabulary is ``['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']`` and our corpus is
+represented as
 
 .. code-block::
 
-    ("hug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("hug" "s", 5)
+    ('hug', 10), ('p' 'ug', 5), ('p' 'un', 12), ('b' 'un', 4), ('hug' 's', 5)
 
-Assuming, that the Byte-Pair Encoding training would stop at this point, the learned merge rules would then be applied
-to new words (as long as those new words do not include symbols that were not in the base vocabulary). For instance,
-the word ``"bug"`` would be tokenized to ``["b", "ug"]`` but ``"mug"`` would be tokenized as ``["<unk>", "ug"]`` since
-the symbol ``"m"`` is not in the base vocabulary. In general, single letters such as ``"m"`` are not replaced by the
-``"<unk>"`` symbol because the training data usually includes at least one occurrence of each letter, but it is likely
-to happen for very special characters like emojis.
+If we stop there, the tokenizer can apply the rules it learned to new words (as long as they don't contain characters
+that were not in the base vocabulary). For instance 'bug' would be tokenized as ``['b', 'ug']`` but mug would be
+tokenized as ``['<unk>', 'ug']`` since the 'm' is not in the base vocabulary. This doesn't happen to letters in general
+(since the base corpus uses all of them), but to special characters like emojis.
 
-As mentioned earlier, the vocabulary size, *i.e.* the base vocabulary size + the number of merges, is a hyperparameter
+As we said before, the vocabulary size (which is the base vocabulary size + the number of merges) is a hyperparameter
 to choose. For instance :doc:`GPT <model_doc/gpt>` has a vocabulary size of 40,478 since they have 478 base characters
-and chose to stop training after 40,000 merges.
+and chose to stop the training of the tokenizer at 40,000 merges.
 
 Byte-level BPE
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-A base vocabulary that includes all possible base characters can be quite large if *e.g.* all unicode characters are
-considered as base characters. To have a better base vocabulary, `GPT-2
-<https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf>`__ uses bytes
-as the base vocabulary, which is a clever trick to force the base vocabulary to be of size 256 while ensuring that
-every base character is included in the vocabulary. With some additional rules to deal with punctuation, the GPT2's
-tokenizer can tokenize every text without the need for the <unk> symbol. :doc:`GPT-2 <model_doc/gpt>` has a vocabulary
-size of 50,257, which corresponds to the 256 bytes base tokens, a special end-of-text token and the symbols learned
-with 50,000 merges.
+To deal with the fact the base vocabulary needs to get all base characters, which can be quite big if one allows for
+all unicode characters, the `GPT-2 paper
+<https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf>`__ introduces a
+clever trick, which is to use bytes as the base vocabulary (which gives a size of 256). With some additional rules to
+deal with punctuation, this manages to be able to tokenize every text without needing an unknown token. For instance,
+the :doc:`GPT-2 model <model_doc/gpt>` has a vocabulary size of 50,257, which corresponds to the 256 bytes base tokens,
+a special end-of-text token and the symbols learned with 50,000 merges.
 
 .. _wordpiece:
 
 WordPiece
 =======================================================================================================================
 
-WordPiece is the subword tokenization algorithm used for :doc:`BERT <model_doc/bert>`, :doc:`DistilBERT
-<model_doc/distilbert>`, and :doc:`Electra <model_doc/electra>`. The algorithm was outlined in `Japanese and Korean
-Voice Seach (Schuster et al., 2012)
-<https://static.googleusercontent.com/media/research.google.com/ja//pubs/archive/37842.pdf>`__ and is very similar to
-BPE. WordPiece first initializes the vocabulary to include every character present in the training data and
-progressively learn a given number of merge rules. In contrast to BPE, WordPiece does not choose the most frequent
-symbol pair, but the one that maximizes the likelihood of the training data once added to the vocabulary.
+WordPiece is the subword tokenization algorithm used for :doc:`BERT <model_doc/bert>` (as well as :doc:`DistilBERT
+<model_doc/distilbert>` and :doc:`Electra <model_doc/electra>`) and was outlined in `this paper
+<https://static.googleusercontent.com/media/research.google.com/ja//pubs/archive/37842.pdf>`__. It relies on the same
+base as BPE, which is to initialize the vocabulary to every character present in the corpus and progressively learn a
+given number of merge rules, the difference is that it doesn't choose the pair that is the most frequent but the one
+that will maximize the likelihood on the corpus once merged.
 
-So what does this mean exactly? Referring to the previous example, maximizing the likelihood of the training data is
-equivalent to finding the symbol pair, whose probability divided by the probabilities of its first symbol followed by
-its second symbol is the greatest among all symbol pairs. *E.g.* ``"u"``, followed by ``"g"`` would have only been
-merged if the probability of ``"ug"`` divided by ``"u"``, ``"g"`` would have been greater than for any other symbol
-pair. Intuitively, WordPiece is slightly different to BPE in that it evaluates what it `loses` by merging two symbols
-to make ensure it's `worth it`.
+What does this mean? Well, in the previous example, it means we would only merge 'u' and 'g' if the probability of
+having 'ug' divided by the probability of having 'u' then 'g' is greater than for any other pair of symbols. It's
+subtly different from what BPE does in the sense that it evaluates what it "loses" by merging two symbols and makes
+sure it's `worth it`.
 
 .. _unigram:
 
 Unigram
 =======================================================================================================================
 
-Unigram is a subword tokenization algorithm introduced in `Subword Regularization: Improving Neural Network Translation
-Models with Multiple Subword Candidates (Kudo, 2018) <https://arxiv.org/pdf/1804.10959.pdf>`__. In contrast to BPE or
-WordPiece, Unigram initializes its base vocabulary to a large number of symbols and progressively trims down each
-symbol to obtain a smaller vocabulary. The base vocabulary could for instance correspond to all pre-tokenized words and
-the most common substrings. Unigram is not used directly for any of the models in the transformers, but it's used in
-conjunction with :ref:`SentencePiece <sentencepiece>`.
+Unigram is a subword tokenization algorithm introduced in `this paper <https://arxiv.org/pdf/1804.10959.pdf>`__.
+Instead of starting with a group of base symbols and learning merges with some rule, like BPE or WordPiece, it starts
+from a large vocabulary (for instance, all pretokenized words and the most common substrings) that it will trim down
+progressively. It's not used directly for any of the pretrained models in the library, but it's used in conjunction
+with :ref:`SentencePiece <sentencepiece>`.
 
-At each training step, the Unigram algorithm defines a loss (often defined as the log-likelihood) over the training
-data given the current vocabulary and a unigram language model. Then, for each symbol in the vocabulary, the algorithm
-computes how much the overall loss would increase if the symbol was to be removed from the vocabulary. Unigram then
-removes p (with p usually being 10% or 20%) percent of the symbols whose loss increase is the lowest, *i.e.* those
-symbols that least affect the overall loss over the training data. This process is repeated until the vocabulary has
-reached the desired size. The Unigram algorithm always keeps the base characters so that any word can be tokenized.
+More specifically, at a given step, unigram computes a loss from the corpus we have and the current vocabulary, then,
+for each subword, evaluate how much the loss would increase if the subword was removed from the vocabulary. It then
+sorts the subwords by this quantity (that represents how much worse the loss becomes if the token is removed) and
+removes all the worst p tokens (for instance p could be 10% or 20%). It then repeats the process until the vocabulary
+has reached the desired size, always keeping the base characters (to be able to tokenize any word written with them,
+like BPE or WordPiece).
 
-Because Unigram is not based on merge rules (in contrast to BPE and WordPiece), the algorithm has several ways of
-tokenizing new text after training. As an example, if a trained Unigram tokenizer exhibits the vocabulary:
+Contrary to BPE and WordPiece that work out rules in a certain order that you can then apply in the same order when
+tokenizing new text, Unigram will have several ways of tokenizing a new text. For instance, if it ends up with the
+vocabulary
 
 .. code-block::
 
-    ["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"],
+    ['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']
 
-``"hugs"`` could be tokenized both as ``["hug", "s"]``, ``["h", "ug", "s"]`` or ``["h", "u", "g", "s"]``. So which one
-to choose? Unigram saves the probability of each token in the training corpus on top of saving the vocabulary so that
-the probability of each possible tokenization can be computed after training. The algorithm simply picks the most
-likely tokenization in practice, but also offers the possibility to sample a possible tokenization according to their
-probabilities.
+we had before, it could tokenize "hugs" as ``['hug', 's']``, ``['h', 'ug', 's']`` or ``['h', 'u', 'g', 's']``. So which
+one choose? On top of saving the vocabulary, the trained tokenizer will save the probability of each token in the
+training corpus. You can then give a probability to each tokenization (which is the product of the probabilities of the
+tokens forming it) and pick the most likely one (or if you want to apply some data augmentation, you could sample one
+of the tokenization according to their probabilities).
 
-Those probabilities are defined by the loss the tokenizer is trained on. Assuming that the training data consists of
-the words :math:`x_{1}, \dots, x_{N}` and that the set of all possible tokenizations for a word :math:`x_{i}` is
-defined as :math:`S(x_{i})`, then the overall loss is defined as
+Those probabilities define the loss that trains the tokenizer: if our corpus consists of the words :math:`x_{1}, \dots,
+x_{N}` and if for the word :math:`x_{i}` we note :math:`S(x_{i})` the set of all possible tokenizations of
+:math:`x_{i}` (with the current vocabulary), then the loss is defined as
 
 .. math::
     \mathcal{L} = -\sum_{i=1}^{N} \log \left ( \sum_{x \in S(x_{i})} p(x) \right )
@@ -247,18 +227,15 @@ defined as :math:`S(x_{i})`, then the overall loss is defined as
 SentencePiece
 =======================================================================================================================
 
-All tokenization algorithms described so far have the same problem: It is assumed that the input text uses spaces to
-separate words. However, not all languages use spaces to separate words. One possible solution is to use language
-specific pre-tokenizers, *e.g.* :doc:`XLM <model_doc/xlm>` uses a specific Chinese, Japanese, and Thai pre-tokenizer).
-To solve this problem more generally, `SentencePiece: A simple and language independent subword tokenizer and
-detokenizer for Neural Text Processing (Kudo et al., 2018) <https://arxiv.org/pdf/1808.06226.pdf>`__ treats the input
-as a raw input stream, thus including the space in the set of characters to use. It then uses the BPE or unigram
-algorithm to construct the appropriate vocabulary.
+All the methods we have been looking at so far required some form of pretokenization, which has a central problem: not
+all languages use spaces to separate words. This is a problem :doc:`XLM <model_doc/xlm>` solves by using specific
+pretokenizers for each of those languages (in this case, Chinese, Japanese and Thai). To solve this problem,
+SentencePiece (introduced in `this paper <https://arxiv.org/pdf/1808.06226.pdf>`__) treats the input as a raw stream,
+includes the space in the set of characters to use, then uses BPE or unigram to construct the appropriate vocabulary.
 
-The :class:`~transformers.XLNetTokenizer` uses SentencePiece for example, which is also why in the example earlier the
-``"▁"`` character was included in the vocabulary. Decoding with SentencePiece is very easy since all tokens can just be
-concatenated and ``"▁"`` is replaced by a space.
+That's why in the example we saw before using :class:`~transformers.XLNetTokenizer` (which uses SentencePiece), we had
+the '▁' character, that represents space. Decoding a tokenized text is then super easy: we just have to concatenate all
+of them together and replace '▁' with space.
 
-All transformers models in the library that use SentencePiece use it in combination with unigram. Examples of models
-using SentencePiece are :doc:`ALBERT <model_doc/albert>`, :doc:`XLNet <model_doc/xlnet>`, :doc:`Marian
-<model_doc/marian>`, and :doc:`T5 <model_doc/t5>`.
+All transformers models in the library that use SentencePiece use it with unigram. Examples of models using it are
+:doc:`ALBERT <model_doc/albert>`, :doc:`XLNet <model_doc/xlnet>` or the :doc:`Marian framework <model_doc/marian>`.

docs/source/training.rst

@@ -39,7 +39,7 @@ head on top of the encoder with an output size of 2. Models are initialized in `
 .. code-block:: python
 
     from transformers import BertForSequenceClassification
-    model = BertForSequenceClassification.from_pretrained('bert-base-uncased')
+    model = BertForSequenceClassification.from_pretrained('bert-base-uncased', return_dict=True)
     model.train()
 
 This is useful because it allows us to make use of the pre-trained BERT encoder and easily train it on whatever

examples/README.md

@@ -37,7 +37,7 @@ git checkout tags/v3.4.0
 |---|---|:---:|:---:|:---:|:---:|
 | [**`language-modeling`**](https://github.com/huggingface/transformers/tree/master/examples/language-modeling)       | Raw text        | ✅ | -  | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb)
 | [**`text-classification`**](https://github.com/huggingface/transformers/tree/master/examples/text-classification)   | GLUE, XNLI      | ✅ | ✅ | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://github.com/huggingface/notebooks/blob/master/examples/text_classification.ipynb)
-| [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER       | ✅ | ✅ | ✅ | -
+| [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER       | ✅ | ✅ | - | -
 | [**`multiple-choice`**](https://github.com/huggingface/transformers/tree/master/examples/multiple-choice)           | SWAG, RACE, ARC | ✅ | ✅ | - | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ViktorAlm/notebooks/blob/master/MPC_GPU_Demo_for_TF_and_PT.ipynb)
 | [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering)     | SQuAD           | ✅ | ✅ | - | -
 | [**`text-generation`**](https://github.com/huggingface/transformers/tree/master/examples/text-generation)           | -               | n/a | n/a | - | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)
@@ -62,14 +62,13 @@ When using PyTorch, we support TPUs thanks to `pytorch/xla`. For more context an
 very detailed [pytorch/xla README](https://github.com/pytorch/xla/blob/master/README.md).
 
 In this repo, we provide a very simple launcher script named [xla_spawn.py](https://github.com/huggingface/transformers/tree/master/examples/xla_spawn.py) that lets you run our example scripts on multiple TPU cores without any boilerplate.
-Just pass a `--num_cores` flag to this script, then your regular training script with its arguments (this is similar to the `torch.distributed.launch` helper for torch.distributed). 
-Note that this approach does not work for examples that use `pytorch-lightning`.
+Just pass a `--num_cores` flag to this script, then your regular training script with its arguments (this is similar to the `torch.distributed.launch` helper for torch.distributed).
 
 For example for `run_glue`:
 
 ```bash
 python examples/xla_spawn.py --num_cores 8 \
-	examples/text-classification/run_glue.py \
+	examples/text-classification/run_glue.py
 	--model_name_or_path bert-base-cased \
 	--task_name mnli \
 	--data_dir ./data/glue_data/MNLI \

examples/adversarial/run_hans.py

@@ -23,7 +23,6 @@ from typing import Dict, List, Optional
 import numpy as np
 import torch
 
-import transformers
 from transformers import (
     AutoConfig,
     AutoModelForSequenceClassification,
@@ -34,7 +33,6 @@ from transformers import (
     default_data_collator,
     set_seed,
 )
-from transformers.trainer_utils import is_main_process
 from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels
 
 
@@ -57,8 +55,7 @@ class ModelArguments:
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
 
 
@@ -127,11 +124,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/bert-loses-patience/pabee/modeling_pabee_albert.py

@@ -21,7 +21,7 @@ import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss
 
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.albert.modeling_albert import (
+from transformers.modeling_albert import (
     ALBERT_INPUTS_DOCSTRING,
     ALBERT_START_DOCSTRING,
     AlbertModel,

examples/bert-loses-patience/pabee/modeling_pabee_bert.py

@@ -23,7 +23,7 @@ from torch import nn
 from torch.nn import CrossEntropyLoss, MSELoss
 
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.bert.modeling_bert import (
+from transformers.modeling_bert import (
     BERT_INPUTS_DOCSTRING,
     BERT_START_DOCSTRING,
     BertEncoder,

examples/bert-loses-patience/run_glue_with_pabee.py

@@ -29,7 +29,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from pabee.modeling_pabee_albert import AlbertForSequenceClassificationWithPabee
 from pabee.modeling_pabee_bert import BertForSequenceClassificationWithPabee
 from transformers import (
@@ -45,7 +44,6 @@ from transformers import glue_compute_metrics as compute_metrics
 from transformers import glue_convert_examples_to_features as convert_examples_to_features
 from transformers import glue_output_modes as output_modes
 from transformers import glue_processors as processors
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -476,7 +474,7 @@ def main():
         "--cache_dir",
         default="",
         type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
     )
     parser.add_argument(
         "--max_seq_length",
@@ -632,11 +630,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 

examples/bert-loses-patience/test_run_glue_with_pabee.py

@@ -4,7 +4,7 @@ import sys
 from unittest.mock import patch
 
 import run_glue_with_pabee
-from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me
+from transformers.testing_utils import TestCasePlus
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -20,7 +20,6 @@ def get_setup_file():
 
 
 class PabeeTests(TestCasePlus):
-    @require_torch_non_multi_gpu_but_fix_me
     def test_run_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)

examples/bertology/run_bertology.py

@@ -30,7 +30,6 @@ from torch.utils.data import DataLoader, SequentialSampler, Subset
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm
 
-import transformers
 from transformers import (
     AutoConfig,
     AutoModelForSequenceClassification,
@@ -42,7 +41,6 @@ from transformers import (
     glue_processors,
     set_seed,
 )
-from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -298,7 +296,7 @@ def main():
         "--cache_dir",
         default=None,
         type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
     )
     parser.add_argument(
         "--data_subset", type=int, default=-1, help="If > 0: limit the data to a subset of data_subset instances."
@@ -370,11 +368,6 @@ def main():
     # Setup logging
     logging.basicConfig(level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
     logger.info("device: {} n_gpu: {}, distributed: {}".format(args.device, args.n_gpu, bool(args.local_rank != -1)))
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
 
     # Set seeds
     set_seed(args.seed)

examples/conftest.py

@@ -17,14 +17,17 @@ warnings.simplefilter(action="ignore", category=FutureWarning)
 
 
 def pytest_addoption(parser):
-    from transformers.testing_utils import pytest_addoption_shared
-
-    pytest_addoption_shared(parser)
+    parser.addoption(
+        "--make_reports",
+        action="store",
+        default=False,
+        help="generate report files - the value will be used as a `report_`+val+`reportname.txt`",
+    )
 
 
 def pytest_terminal_summary(terminalreporter):
     from transformers.testing_utils import pytest_terminal_summary_main
 
-    make_reports = terminalreporter.config.getoption("--make-reports")
+    make_reports = terminalreporter.config.getoption("--make_reports")
     if make_reports:
         pytest_terminal_summary_main(terminalreporter, id=make_reports)

examples/contrib/legacy/run_language_modeling.py

@@ -29,7 +29,6 @@ from typing import Optional
 
 from torch.utils.data import ConcatDataset
 
-import transformers
 from transformers import (
     CONFIG_MAPPING,
     MODEL_WITH_LM_HEAD_MAPPING,
@@ -48,7 +47,6 @@ from transformers import (
     TrainingArguments,
     set_seed,
 )
-from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -81,8 +79,7 @@ class ModelArguments:
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
 
 
@@ -222,11 +219,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/contrib/mm-imdb/run_mmimdb.py

@@ -31,7 +31,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from transformers import (
     WEIGHTS_NAME,
     AdamW,
@@ -42,7 +41,6 @@ from transformers import (
     MMBTForClassification,
     get_linear_schedule_with_warmup,
 )
-from transformers.trainer_utils import is_main_process
 from utils_mmimdb import ImageEncoder, JsonlDataset, collate_fn, get_image_transforms, get_mmimdb_labels
 
 
@@ -350,7 +348,7 @@ def main():
         "--cache_dir",
         default=None,
         type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
     )
     parser.add_argument(
         "--max_seq_length",
@@ -478,11 +476,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 

examples/contrib/run_camembert.py

@@ -1,6 +1,7 @@
 import torch
 
-from transformers import CamembertForMaskedLM, CamembertTokenizer
+from transformers.modeling_camembert import CamembertForMaskedLM
+from transformers.tokenization_camembert import CamembertTokenizer
 
 
 def fill_mask(masked_input, model, tokenizer, topk=5):

examples/contrib/run_swag.py

@@ -31,16 +31,8 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
-from transformers import (
-    WEIGHTS_NAME,
-    AdamW,
-    AutoConfig,
-    AutoModelForMultipleChoice,
-    AutoTokenizer,
-    get_linear_schedule_with_warmup,
-)
-from transformers.trainer_utils import is_main_process
+from transformers import WEIGHTS_NAME, AdamW, AutoConfig, AutoTokenizer, get_linear_schedule_with_warmup
+from transformers.modeling_auto import AutoModelForMultipleChoice
 
 
 try:
@@ -628,11 +620,6 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
 
     # Set seed
     set_seed(args)

examples/deebert/run_glue_deebert.py

@@ -13,7 +13,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from src.modeling_highway_bert import DeeBertForSequenceClassification
 from src.modeling_highway_roberta import DeeRobertaForSequenceClassification
 from transformers import (
@@ -29,7 +28,6 @@ from transformers import glue_compute_metrics as compute_metrics
 from transformers import glue_convert_examples_to_features as convert_examples_to_features
 from transformers import glue_output_modes as output_modes
 from transformers import glue_processors as processors
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -452,7 +450,7 @@ def main():
         "--cache_dir",
         default="",
         type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
     )
     parser.add_argument(
         "--max_seq_length",
@@ -582,11 +580,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 

examples/deebert/src/modeling_highway_bert.py

@@ -3,7 +3,7 @@ from torch import nn
 from torch.nn import CrossEntropyLoss, MSELoss
 
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.bert.modeling_bert import (
+from transformers.modeling_bert import (
     BERT_INPUTS_DOCSTRING,
     BERT_START_DOCSTRING,
     BertEmbeddings,

examples/deebert/src/modeling_highway_roberta.py

@@ -3,13 +3,9 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss
 
-from transformers import RobertaConfig
+from transformers.configuration_roberta import RobertaConfig
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
-from transformers.models.roberta.modeling_roberta import (
-    ROBERTA_INPUTS_DOCSTRING,
-    ROBERTA_START_DOCSTRING,
-    RobertaEmbeddings,
-)
+from transformers.modeling_roberta import ROBERTA_INPUTS_DOCSTRING, ROBERTA_START_DOCSTRING, RobertaEmbeddings
 
 from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy
 

examples/deebert/test_glue_deebert.py

@@ -5,7 +5,7 @@ import unittest
 from unittest.mock import patch
 
 import run_glue_deebert
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow
+from transformers.testing_utils import slow
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -26,7 +26,6 @@ class DeeBertTests(unittest.TestCase):
         logger.addHandler(stream_handler)
 
     @slow
-    @require_torch_non_multi_gpu_but_fix_me
     def test_glue_deebert_train(self):
 
         train_args = """

examples/distillation/README.md

@@ -12,7 +12,7 @@ This folder contains the original code used to train Distil* as well as examples
 
 **October 3, 2019 - Update** We release our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108) explaining our approach on **DistilBERT**. It includes updated results and further experiments. We applied the same method to GPT2 and release the weights of **DistilGPT2**. DistilGPT2 is two times faster and 33% smaller than GPT2. **The paper supersedes our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. Please use the paper as a reference when comparing/reporting results on DistilBERT.**
 
-**September 19, 2019 - Update:** We fixed bugs in the code and released an updated version of the weights trained with a modification of the distillation loss. DistilBERT now reaches 99% of `BERT-base`'s performance on GLUE, and 86.9 F1 score on SQuAD v1.1 dev set (compared to 88.5 for `BERT-base`). We will publish a formal write-up of our approach in the near future!
+**September 19, 2019 - Update:** We fixed bugs in the code and released an upadted version of the weights trained with a modification of the distillation loss. DistilBERT now reaches 99% of `BERT-base`'s performance on GLUE, and 86.9 F1 score on SQuAD v1.1 dev set (compared to 88.5 for `BERT-base`). We will publish a formal write-up of our approach in the near future!
 
 
 ## What is Distil*

examples/distillation/run_squad_w_distillation.py

@@ -30,7 +30,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from transformers import (
     WEIGHTS_NAME,
     AdamW,
@@ -58,7 +57,6 @@ from transformers.data.metrics.squad_metrics import (
     squad_evaluate,
 )
 from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -578,7 +576,7 @@ def main():
         "--cache_dir",
         default="",
         type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
     )
 
     parser.add_argument(
@@ -747,11 +745,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 

examples/language-modeling/README.md

@@ -10,7 +10,7 @@ These scripts leverage the 🤗 Datasets library and the Trainer API. You can ea
 need extra processing on your datasets.
 
 **Note:** The old script `run_language_modeling.py` is still available
-[here](https://github.com/huggingface/transformers/blob/master/examples/contrib/legacy/run_language_modeling.py).
+[here](https://github.com/huggingface/transformers/blob/master/examples/contrib/legacy/language-modeling/run_language_modeling.py).
 
 The following examples, will run on a datasets hosted on our [hub](https://huggingface.co/datasets) or with your own
 text files for training and validation. We give examples of both below.
@@ -68,25 +68,19 @@ python run_mlm.py \
 To run on your own training and validation files, use the following command:
 
 ```bash
-python run_mlm.py \
+python run_clm.py \
     --model_name_or_path roberta-base \
     --train_file path_to_train_file \
     --validation_file path_to_validation_file \
     --do_train \
     --do_eval \
-    --output_dir /tmp/test-mlm
+    --output_dir /tmp/test-clm
 ```
 
-If your dataset is organized with one sample per line, you can use the `--line_by_line` flag (otherwise the script
-concatenates all texts and then splits them in blocks of the same length).
-
-**Note:** On TPU, you should use the flag `--pad_to_max_length` in conjunction with the `--line_by_line` flag to make
-sure all your batches have the same length.
-
 ### Whole word masking
 
 The BERT authors released a new version of BERT using Whole Word Masking in May 2019. Instead of masking randomly
-selected tokens (which may be part of words), they mask randomly selected words (masking all the tokens corresponding
+selected tokens (which may be aprt of words), they mask randomly selected words (masking all the tokens corresponding
 to that word). This technique has been refined for Chinese in [this paper](https://arxiv.org/abs/1906.08101).
 
 To fine-tune a model using whole word masking, use the following script:
@@ -117,8 +111,8 @@ It works well on so many Chines Task like CLUE (Chinese GLUE). They use LTP, so
 we need LTP.
 
 Now LTP only only works well on `transformers==3.2.0`. So we don't add it to requirements.txt.
-You need to create a separate environment with this version of Transformers to run the `run_chinese_ref.py` script that
-will create the reference files. The script is in `examples/contrib`. Once in the proper environment, run the
+You need to create a separate enviromnent with this version of Transformers to run the `run_chinese_ref.py` script that
+will create the reference files. The script is in `examples/contrib`. Once in the proper enviromnent, run the
 following:
 
 
@@ -150,8 +144,6 @@ python run_mlm_wwm.py \
     --output_dir /tmp/test-mlm-wwm
 ```
 
-**Note:** On TPU, you should the flag `--pad_to_max_length` to make sure all your batches have the same length.
-
 ### XLNet and permutation language modeling
 
 XLNet uses a different training objective, which is permutation language modeling. It is an autoregressive method 
@@ -187,9 +179,3 @@ python run_plm.py \
     --do_eval \
     --output_dir /tmp/test-plm
 ```
-
-If your dataset is organized with one sample per line, you can use the `--line_by_line` flag (otherwise the script
-concatenates all texts and then splits them in blocks of the same length).
-
-**Note:** On TPU, you should use the flag `--pad_to_max_length` in conjunction with the `--line_by_line` flag to make
-sure all your batches have the same length.

examples/language-modeling/run_clm.py

@@ -76,8 +76,7 @@ class ModelArguments:
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
     use_fast_tokenizer: bool = field(
         default=True,
@@ -169,8 +168,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -193,7 +190,7 @@ def main():
         if data_args.train_file is not None:
             data_files["train"] = data_args.train_file
         if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
+            data_files["validation"] = data_args.train_file
         extension = data_args.train_file.split(".")[-1]
         if extension == "txt":
             extension = "text"
@@ -257,7 +254,7 @@ def main():
         tokenize_function,
         batched=True,
         num_proc=data_args.preprocessing_num_workers,
-        remove_columns=column_names,
+        remove_columns=[text_column_name],
         load_from_cache_file=not data_args.overwrite_cache,
     )
 
@@ -313,12 +310,9 @@ def main():
 
     # Training
     if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
         )
-        trainer.train(model_path=model_path)
         trainer.save_model()  # Saves the tokenizer too for easy upload
 
     # Evaluation

examples/language-modeling/run_mlm.py

@@ -74,8 +74,7 @@ class ModelArguments:
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
     use_fast_tokenizer: bool = field(
         default=True,
@@ -117,17 +116,6 @@ class DataTrainingArguments:
     mlm_probability: float = field(
         default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
     )
-    line_by_line: bool = field(
-        default=False,
-        metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."},
-    )
-    pad_to_max_length: bool = field(
-        default=False,
-        metadata={
-            "help": "Whether to pad all samples to `max_seq_length`. "
-            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
-        },
-    )
 
     def __post_init__(self):
         if self.dataset_name is None and self.train_file is None and self.validation_file is None:
@@ -180,8 +168,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -204,7 +190,7 @@ def main():
         if data_args.train_file is not None:
             data_files["train"] = data_args.train_file
         if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
+            data_files["validation"] = data_args.train_file
         extension = data_args.train_file.split(".")[-1]
         if extension == "txt":
             extension = "text"
@@ -260,83 +246,18 @@ def main():
         column_names = datasets["validation"].column_names
     text_column_name = "text" if "text" in column_names else column_names[0]
 
-    if data_args.line_by_line:
-        # When using line_by_line, we just tokenize each nonempty line.
-        padding = "max_length" if data_args.pad_to_max_length else False
+    def tokenize_function(examples):
+        # Remove empty lines
+        examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
+        return tokenizer(examples["text"], truncation=True, max_length=data_args.max_seq_length)
 
-        def tokenize_function(examples):
-            # Remove empty lines
-            examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
-            return tokenizer(
-                examples["text"],
-                padding=padding,
-                truncation=True,
-                max_length=data_args.max_seq_length,
-                # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it
-                # receives the `special_tokens_mask`.
-                return_special_tokens_mask=True,
-            )
-
-        tokenized_datasets = datasets.map(
-            tokenize_function,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
-    else:
-        # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts.
-        # We use `return_special_tokens_mask=True` because DataCollatorForLanguageModeling (see below) is more
-        # efficient when it receives the `special_tokens_mask`.
-        def tokenize_function(examples):
-            return tokenizer(examples[text_column_name], return_special_tokens_mask=True)
-
-        tokenized_datasets = datasets.map(
-            tokenize_function,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=column_names,
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
-
-        if data_args.max_seq_length is None:
-            max_seq_length = tokenizer.model_max_length
-        else:
-            if data_args.max_seq_length > tokenizer.model_max_length:
-                logger.warn(
-                    f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the"
-                    f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."
-                )
-            max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)
-
-        # Main data processing function that will concatenate all texts from our dataset and generate chunks of
-        # max_seq_length.
-        def group_texts(examples):
-            # Concatenate all texts.
-            concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
-            total_length = len(concatenated_examples[list(examples.keys())[0]])
-            # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
-            # customize this part to your needs.
-            total_length = (total_length // max_seq_length) * max_seq_length
-            # Split by chunks of max_len.
-            result = {
-                k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)]
-                for k, t in concatenated_examples.items()
-            }
-            return result
-
-        # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a
-        # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value
-        # might be slower to preprocess.
-        #
-        # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
-        # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map
-        tokenized_datasets = tokenized_datasets.map(
-            group_texts,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
+    tokenized_datasets = datasets.map(
+        tokenize_function,
+        batched=True,
+        num_proc=data_args.preprocessing_num_workers,
+        remove_columns=[text_column_name],
+        load_from_cache_file=not data_args.overwrite_cache,
+    )
 
     # Data collator
     # This one will take care of randomly masking the tokens.
@@ -354,12 +275,9 @@ def main():
 
     # Training
     if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
         )
-        trainer.train(model_path=model_path)
         trainer.save_model()  # Saves the tokenizer too for easy upload
 
     # Evaluation

examples/language-modeling/run_mlm_wwm.py

@@ -76,8 +76,7 @@ class ModelArguments:
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
     use_fast_tokenizer: bool = field(
         default=True,
@@ -121,13 +120,6 @@ class DataTrainingArguments:
     mlm_probability: float = field(
         default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
     )
-    pad_to_max_length: bool = field(
-        default=False,
-        metadata={
-            "help": "Whether to pad all samples to `max_seq_length`. "
-            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
-        },
-    )
 
     def __post_init__(self):
         if self.train_file is not None:
@@ -187,8 +179,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -207,7 +197,7 @@ def main():
     if data_args.train_file is not None:
         data_files["train"] = data_args.train_file
     if data_args.validation_file is not None:
-        data_files["validation"] = data_args.validation_file
+        data_files["validation"] = data_args.train_file
     extension = data_args.train_file.split(".")[-1]
     if extension == "txt":
         extension = "text"
@@ -263,12 +253,10 @@ def main():
         column_names = datasets["validation"].column_names
     text_column_name = "text" if "text" in column_names else column_names[0]
 
-    padding = "max_length" if data_args.pad_to_max_length else False
-
     def tokenize_function(examples):
         # Remove empty lines
         examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
-        return tokenizer(examples["text"], padding=padding, truncation=True, max_length=data_args.max_seq_length)
+        return tokenizer(examples["text"], truncation=True, max_length=data_args.max_seq_length)
 
     tokenized_datasets = datasets.map(
         tokenize_function,
@@ -302,12 +290,9 @@ def main():
 
     # Training
     if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
         )
-        trainer.train(model_path=model_path)
         trainer.save_model()  # Saves the tokenizer too for easy upload
 
     # Evaluation

examples/language-modeling/run_plm.py

@@ -64,8 +64,7 @@ class ModelArguments:
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
     use_fast_tokenizer: bool = field(
         default=True,
@@ -114,17 +113,6 @@ class DataTrainingArguments:
     max_span_length: int = field(
         default=5, metadata={"help": "Maximum length of a span of masked tokens for permutation language modeling."}
     )
-    line_by_line: bool = field(
-        default=False,
-        metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."},
-    )
-    pad_to_max_length: bool = field(
-        default=False,
-        metadata={
-            "help": "Whether to pad all samples to `max_seq_length`. "
-            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
-        },
-    )
 
     def __post_init__(self):
         if self.dataset_name is None and self.train_file is None and self.validation_file is None:
@@ -177,8 +165,6 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -201,7 +187,7 @@ def main():
         if data_args.train_file is not None:
             data_files["train"] = data_args.train_file
         if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
+            data_files["validation"] = data_args.train_file
         extension = data_args.train_file.split(".")[-1]
         if extension == "txt":
             extension = "text"
@@ -257,73 +243,18 @@ def main():
         column_names = datasets["validation"].column_names
     text_column_name = "text" if "text" in column_names else column_names[0]
 
-    if data_args.line_by_line:
-        # When using line_by_line, we just tokenize each nonempty line.
-        padding = "max_length" if data_args.pad_to_max_length else False
+    def tokenize_function(examples):
+        # Remove empty lines
+        examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
+        return tokenizer(examples["text"], truncation=True, max_length=data_args.max_seq_length)
 
-        def tokenize_function(examples):
-            # Remove empty lines
-            examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
-            return tokenizer(examples["text"], padding=padding, truncation=True, max_length=data_args.max_seq_length)
-
-        tokenized_datasets = datasets.map(
-            tokenize_function,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
-    else:
-        # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts.
-        def tokenize_function(examples):
-            return tokenizer(examples[text_column_name])
-
-        tokenized_datasets = datasets.map(
-            tokenize_function,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=column_names,
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
-
-        if data_args.max_seq_length is None:
-            max_seq_length = tokenizer.model_max_length
-        else:
-            if data_args.max_seq_length > tokenizer.model_max_length:
-                logger.warn(
-                    f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the"
-                    f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."
-                )
-            max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)
-
-        # Main data processing function that will concatenate all texts from our dataset and generate chunks of
-        # max_seq_length.
-        def group_texts(examples):
-            # Concatenate all texts.
-            concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
-            total_length = len(concatenated_examples[list(examples.keys())[0]])
-            # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
-            # customize this part to your needs.
-            total_length = (total_length // max_seq_length) * max_seq_length
-            # Split by chunks of max_len.
-            result = {
-                k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)]
-                for k, t in concatenated_examples.items()
-            }
-            return result
-
-        # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a
-        # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value
-        # might be slower to preprocess.
-        #
-        # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
-        # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map
-        tokenized_datasets = tokenized_datasets.map(
-            group_texts,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
+    tokenized_datasets = datasets.map(
+        tokenize_function,
+        batched=True,
+        num_proc=data_args.preprocessing_num_workers,
+        remove_columns=[text_column_name],
+        load_from_cache_file=not data_args.overwrite_cache,
+    )
 
     # Data collator
     data_collator = DataCollatorForPermutationLanguageModeling(
@@ -344,12 +275,9 @@ def main():
 
     # Training
     if training_args.do_train:
-        model_path = (
-            model_args.model_name_or_path
-            if (model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path))
-            else None
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
         )
-        trainer.train(model_path=model_path)
         trainer.save_model()  # Saves the tokenizer too for easy upload
 
     # Evaluation

examples/lightning_base.py

@@ -4,11 +4,9 @@ import os
 from pathlib import Path
 from typing import Any, Dict
 
-import packaging
 import pytorch_lightning as pl
 from pytorch_lightning.utilities import rank_zero_info
 
-import pkg_resources
 from transformers import (
     AdamW,
     AutoConfig,
@@ -35,17 +33,6 @@ from transformers.optimization import (
 logger = logging.getLogger(__name__)
 
 
-def require_min_ver(pkg, min_ver):
-    got_ver = pkg_resources.get_distribution(pkg).version
-    if packaging.version.parse(got_ver) < packaging.version.parse(min_ver):
-        logger.warning(
-            f"{pkg}>={min_ver} is required for a normal functioning of this module, but found {pkg}=={got_ver}. "
-            "Try: pip install -r examples/requirements.txt"
-        )
-
-
-require_min_ver("pytorch_lightning", "1.0.4")
-
 MODEL_MODES = {
     "base": AutoModel,
     "sequence-classification": AutoModelForSequenceClassification,
@@ -236,7 +223,7 @@ class BaseTransformer(pl.LightningModule):
             "--cache_dir",
             default="",
             type=str,
-            help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+            help="Where do you want to store the pre-trained models downloaded from s3",
         )
         parser.add_argument(
             "--encoder_layerdrop",

examples/lxmert/demo.ipynb

@@ -210,6 +210,7 @@
     "        visual_feats=features,\n",
     "        visual_pos=normalized_boxes,\n",
     "        token_type_ids=inputs.token_type_ids,\n",
+    "        return_dict=True,\n",
     "        output_attentions=False,\n",
     "    )\n",
     "    output_vqa = lxmert_vqa(\n",
@@ -218,6 +219,7 @@
     "        visual_feats=features,\n",
     "        visual_pos=normalized_boxes,\n",
     "        token_type_ids=inputs.token_type_ids,\n",
+    "        return_dict=True,\n",
     "        output_attentions=False,\n",
     "    )\n",
     "    # get prediction\n",
@@ -264,4 +266,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

examples/lxmert/modeling_frcnn.py

@@ -1801,7 +1801,7 @@ class GeneralizedRCNN(nn.Module):
                 f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when "
                 f"initializing {model.__class__.__name__}: {unexpected_keys}\n"
                 f"- This IS expected if you are initializing {model.__class__.__name__} from the checkpoint of a model trained on another task "
-                f"or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n"
+                f"or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n"
                 f"- This IS NOT expected if you are initializing {model.__class__.__name__} from the checkpoint of a model that you expect "
                 f"to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model)."
             )

examples/lxmert/requirements.txt

@@ -47,7 +47,7 @@ nbclient==0.5.0
 nbconvert==6.0.1
 nbformat==5.0.7
 nest-asyncio==1.4.0
-notebook==6.1.5
+notebook==6.1.4
 numpy==1.19.2
 opencv-python==4.4.0.42
 packaging==20.3

examples/movement-pruning/emmental/modeling_bert_masked.py

@@ -16,7 +16,7 @@
 """Masked Version of BERT. It replaces the `torch.nn.Linear` layers with
 :class:`~emmental.MaskedLinear` and add an additional parameters in the forward pass to
 compute the adaptive mask.
-Built on top of `transformers.models.bert.modeling_bert`"""
+Built on top of `transformers.modeling_bert`"""
 
 
 import logging
@@ -29,8 +29,8 @@ from torch.nn import CrossEntropyLoss, MSELoss
 from emmental import MaskedBertConfig
 from emmental.modules import MaskedLinear
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
+from transformers.modeling_bert import ACT2FN, BertLayerNorm, load_tf_weights_in_bert
 from transformers.modeling_utils import PreTrainedModel, prune_linear_layer
-from transformers.models.bert.modeling_bert import ACT2FN, BertLayerNorm, load_tf_weights_in_bert
 
 
 logger = logging.getLogger(__name__)

examples/movement-pruning/masked_run_glue.py

@@ -620,7 +620,7 @@ def main():
         "--cache_dir",
         default="",
         type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
     )
     parser.add_argument(
         "--max_seq_length",

examples/movement-pruning/masked_run_squad.py

@@ -725,7 +725,7 @@ def main():
         "--cache_dir",
         default="",
         type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
     )
 
     parser.add_argument(

examples/multiple-choice/run_multiple_choice.py

@@ -23,7 +23,6 @@ from typing import Dict, Optional
 
 import numpy as np
 
-import transformers
 from transformers import (
     AutoConfig,
     AutoModelForMultipleChoice,
@@ -34,7 +33,6 @@ from transformers import (
     TrainingArguments,
     set_seed,
 )
-from transformers.trainer_utils import is_main_process
 from utils_multiple_choice import MultipleChoiceDataset, Split, processors
 
 
@@ -61,8 +59,7 @@ class ModelArguments:
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
 
 
@@ -118,11 +115,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/multiple-choice/run_tf_multiple_choice.py

@@ -33,15 +33,9 @@ from transformers import (
     TFTrainingArguments,
     set_seed,
 )
-from transformers.utils import logging as hf_logging
 from utils_multiple_choice import Split, TFMultipleChoiceDataset, processors
 
 
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -65,8 +59,7 @@ class ModelArguments:
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
 
 

examples/question-answering/run_squad.py

@@ -29,7 +29,6 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
-import transformers
 from transformers import (
     MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     WEIGHTS_NAME,
@@ -46,7 +45,6 @@ from transformers.data.metrics.squad_metrics import (
     squad_evaluate,
 )
 from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor
-from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -321,7 +319,7 @@ def evaluate(args, model, tokenizer, prefix=""):
             eval_feature = features[feature_index.item()]
             unique_id = int(eval_feature.unique_id)
 
-            output = [to_list(output[i]) for output in outputs.to_tuple()]
+            output = [to_list(output[i]) for output in outputs]
 
             # Some models (XLNet, XLM) use 5 arguments for their predictions, while the other "simpler"
             # models only use two.
@@ -532,7 +530,7 @@ def main():
         "--cache_dir",
         default="",
         type=str,
-        help="Where do you want to store the pre-trained models downloaded from huggingface.co",
+        help="Where do you want to store the pre-trained models downloaded from s3",
     )
 
     parser.add_argument(
@@ -714,11 +712,7 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
+
     # Set seed
     set_seed(args)
 
@@ -736,7 +730,6 @@ def main():
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,
         cache_dir=args.cache_dir if args.cache_dir else None,
-        use_fast=False,  # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
     )
     model = AutoModelForQuestionAnswering.from_pretrained(
         args.model_name_or_path,
@@ -785,10 +778,7 @@ def main():
 
         # Load a trained model and vocabulary that you have fine-tuned
         model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)  # , force_download=True)
-
-        # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
-        # So we use use_fast=False here for now until Fast-tokenizer-compatible-examples are out
-        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case, use_fast=False)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
         model.to(args.device)
 
     # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory

examples/question-answering/run_squad_trainer.py

@@ -22,11 +22,9 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional
 
-import transformers
 from transformers import AutoConfig, AutoModelForQuestionAnswering, AutoTokenizer, HfArgumentParser, SquadDataset
 from transformers import SquadDataTrainingArguments as DataTrainingArguments
 from transformers import Trainer, TrainingArguments
-from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -51,8 +49,7 @@ class ModelArguments:
     # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
     # or just modify its tokenizer_config.json.
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
 
 
@@ -94,11 +91,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Prepare Question-Answering task
@@ -115,7 +107,6 @@ def main():
     tokenizer = AutoTokenizer.from_pretrained(
         model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
         cache_dir=model_args.cache_dir,
-        use_fast=False,  # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
     )
     model = AutoModelForQuestionAnswering.from_pretrained(
         model_args.model_name_or_path,

examples/question-answering/run_tf_squad.py

@@ -33,12 +33,6 @@ from transformers import (
     squad_convert_examples_to_features,
 )
 from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor
-from transformers.utils import logging as hf_logging
-
-
-hf_logging.set_verbosity_info()
-hf_logging.enable_default_handler()
-hf_logging.enable_explicit_format()
 
 
 logger = logging.getLogger(__name__)
@@ -63,8 +57,7 @@ class ModelArguments:
     # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
     # or just modify its tokenizer_config.json.
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
 
 

examples/rag/distributed_retriever.py

@@ -27,7 +27,7 @@ class RagPyTorchDistributedRetriever(RagRetriever):
             It is used to decode the question and then use the generator_tokenizer.
         generator_tokenizer (:class:`~transformers.PretrainedTokenizer`):
             The tokenizer used for the generator part of the RagModel.
-        index (:class:`~transformers.models.rag.retrieval_rag.Index`, optional, defaults to the one defined by the configuration):
+        index (:class:`~transformers.retrieval_rag.Index`, optional, defaults to the one defined by the configuration):
             If specified, use this index instead of the one built using the configuration
     """
 

examples/rag/eval_rag.py

@@ -95,7 +95,7 @@ def evaluate_batch_retrieval(args, rag_model, questions):
         truncation=True,
     )["input_ids"].to(args.device)
 
-    question_enc_outputs = rag_model.rag.question_encoder(retriever_input_ids)
+    question_enc_outputs = rag_model.rag.question_encoder(retriever_input_ids, return_dict=True)
     question_enc_pool_output = question_enc_outputs.pooler_output
 
     result = rag_model.retriever(

examples/rag/finetune.py

@@ -204,6 +204,7 @@ class GenerativeQAModule(BaseTransformer):
             decoder_input_ids=decoder_input_ids,
             use_cache=False,
             labels=lm_labels,
+            return_dict=True,
             **rag_kwargs,
         )
 

examples/rag/finetune.sh

@@ -7,7 +7,7 @@ export PYTHONPATH="../":"${PYTHONPATH}"
 python examples/rag/finetune.py \
     --data_dir $DATA_DIR \
     --output_dir $OUTPUT_DIR \
-    --model_name_or_path $MODEL_NAME_OR_PATH \
+    --model_name_or_path $MODLE_NAME_OR_PATH \
     --model_type rag_sequence \
     --fp16 \
     --gpus 8 \

examples/rag/test_distributed_retriever.py

@@ -11,12 +11,15 @@ import numpy as np
 from datasets import Dataset
 
 import faiss
-from transformers import BartConfig, BartTokenizer, DPRConfig, DPRQuestionEncoderTokenizer, RagConfig
+from transformers.configuration_bart import BartConfig
+from transformers.configuration_dpr import DPRConfig
+from transformers.configuration_rag import RagConfig
 from transformers.file_utils import is_datasets_available, is_faiss_available, is_psutil_available, is_torch_available
-from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
-from transformers.models.rag.retrieval_rag import CustomHFIndex
-from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
-from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me
+from transformers.retrieval_rag import CustomHFIndex
+from transformers.tokenization_bart import BartTokenizer
+from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
+from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer
+from transformers.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
 
 
 sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # noqa: E402 # isort:skip
@@ -133,7 +136,7 @@ class RagRetrieverTest(TestCase):
             question_encoder=DPRConfig().to_dict(),
             generator=BartConfig().to_dict(),
         )
-        with patch("transformers.models.rag.retrieval_rag.load_dataset") as mock_load_dataset:
+        with patch("transformers.retrieval_rag.load_dataset") as mock_load_dataset:
             mock_load_dataset.return_value = dataset
             retriever = RagPyTorchDistributedRetriever(
                 config,
@@ -175,7 +178,6 @@ class RagRetrieverTest(TestCase):
             retriever.init_retrieval(port)
         return retriever
 
-    @require_torch_non_multi_gpu_but_fix_me
     def test_pytorch_distributed_retriever_retrieve(self):
         n_docs = 1
         retriever = self.get_dummy_pytorch_distributed_retriever(init_retrieval=True)
@@ -191,7 +193,6 @@ class RagRetrieverTest(TestCase):
         self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
         self.assertListEqual(doc_ids.tolist(), [[1], [0]])
 
-    @require_torch_non_multi_gpu_but_fix_me
     def test_custom_hf_index_retriever_retrieve(self):
         n_docs = 1
         retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=False)
@@ -207,7 +208,6 @@ class RagRetrieverTest(TestCase):
         self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
         self.assertListEqual(doc_ids.tolist(), [[1], [0]])
 
-    @require_torch_non_multi_gpu_but_fix_me
     def test_custom_pytorch_distributed_retriever_retrieve_from_disk(self):
         n_docs = 1
         retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=True)

examples/rag/use_own_knowledge_dataset.py

@@ -47,7 +47,7 @@ def embed(documents: dict, ctx_encoder: DPRContextEncoder, ctx_tokenizer: DPRCon
     input_ids = ctx_tokenizer(
         documents["title"], documents["text"], truncation=True, padding="longest", return_tensors="pt"
     )["input_ids"]
-    embeddings = ctx_encoder(input_ids.to(device=device)).pooler_output
+    embeddings = ctx_encoder(input_ids.to(device=device), return_dict=True).pooler_output
     return {"embeddings": embeddings.detach().cpu().numpy()}
 
 

examples/requirements.txt

@@ -18,4 +18,3 @@ fire
 pytest
 conllu
 sentencepiece != 0.1.92
-protobuf

examples/seq2seq/README.md

@@ -1,7 +1,7 @@
 ## Sequence to Sequence Training and Evaluation
 
 This directory contains examples for finetuning and evaluating transformers on summarization and translation tasks.
-Please tag @patil-suraj with any issues/unexpected behaviors, or send a PR!
+Please tag @sshleifer with any issues/unexpected behaviors, or send a PR!
 For deprecated `bertabs` instructions, see [`bertabs/README.md`](bertabs/README.md).
 
 ### Supported Architectures
@@ -380,7 +380,7 @@ cp xsum/test* all_pl
 then use `all_pl` as DATA in the command above.
 
 #### Direct Knowledge Distillation (KD)
-+ In this method, we use try to enforce that the student and teacher produce similar encoder_outputs, logits, and hidden_states using `SummarizationDistiller`.
++ In this method, we use try to enforce that the student and teacher produce similar encoder_outputs, logits, and hidden_states using `BartSummarizationDistiller`.
 + This method was used for `sshleifer/distilbart-xsum-12-6`, `6-6`, and `9-6` checkpoints were produced.
 + You must use [`distillation.py`](./distillation.py). Note that this command initializes the student for you.
 

examples/seq2seq/bertabs/configuration_bertabs.py

@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 BERTABS_FINETUNED_CONFIG_MAP = {
-    "bertabs-finetuned-cnndm": "https://huggingface.co/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization/resolve/main/config.json",
+    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization/config.json",
 }
 
 

examples/seq2seq/distillation.py

@@ -16,7 +16,7 @@ from finetune import SummarizationModule, TranslationModule
 from finetune import main as ft_main
 from make_student import create_student_by_copying_alternating_layers, get_layers_to_supervise
 from transformers import AutoModelForSeq2SeqLM, MBartTokenizer, T5ForConditionalGeneration
-from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.modeling_bart import shift_tokens_right
 from utils import calculate_bleu, check_output_dir, freeze_params, label_smoothed_nll_loss, use_task_specific_params
 
 
@@ -25,8 +25,8 @@ sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
 from lightning_base import generic_train  # noqa
 
 
-class SummarizationDistiller(SummarizationModule):
-    """Supports T5, Bart, Pegasus and other models that inherit from Bart."""
+class BartSummarizationDistiller(SummarizationModule):
+    """Supports Bart, Pegasus and other models that inherit from Bart."""
 
     loss_names = ["loss", "ce_loss", "mlm_loss", "hid_loss_enc", "hid_loss_dec"]
 
@@ -40,38 +40,25 @@ class SummarizationDistiller(SummarizationModule):
         hparams.model_name_or_path = str(save_dir)  # Tell lightning we are training the student
         teacher = AutoModelForSeq2SeqLM.from_pretrained(hparams.teacher).eval()
         use_task_specific_params(teacher, hparams.task)  # We copy good generation parameters to student by default
-        if hparams.student is not None:
-            student = AutoModelForSeq2SeqLM.from_pretrained(hparams.student)
-            use_task_specific_params(student, hparams.task)
-            e_layer_ids, d_layer_ids = None, None
-        else:
-            student, e_layer_ids, d_layer_ids = create_student_by_copying_alternating_layers(
-                teacher, e=hparams.student_encoder_layers, d=hparams.student_decoder_layers, save_path=save_dir
-            )
-
+        student, e_layer_ids, d_layer_ids = create_student_by_copying_alternating_layers(
+            teacher, e=hparams.student_encoder_layers, d=hparams.student_decoder_layers, save_path=save_dir
+        )
         if hparams.length_penalty != -1:
             student.config.length_penalty = hparams.length_penalty
-        hparams.tokenizer_name = hparams.teacher  # Use teacher's tokenizer
         super().__init__(hparams, model=student, config=student.config)
-        assert (
-            student.config.model_type == teacher.config.model_type
-        ), f"teacher, student model types should be the same, got {student.config.model_type} != {teacher.config.model_type}"
+        model_type = student.config.model_type
+        self.e_layer_ids, self.d_layer_ids = e_layer_ids, d_layer_ids  # type: List[int], List[int]
 
-        if student.config.model_type == "t5":
-            student_encoder_layers = len(student.get_encoder().block)
-            student_decoder_layers = len(student.get_decoder().block)
+        if model_type == "t5":
             teacher_encoder_layers = len(teacher.get_encoder().block)
             teacher_decoder_layers = len(teacher.get_decoder().block)
         else:
-            student_encoder_layers = student.config.encoder_layers
-            student_decoder_layers = student.config.decoder_layers
             teacher_encoder_layers = teacher.config.encoder_layers
             teacher_decoder_layers = teacher.config.decoder_layers
 
-        self.different_base_models = not (hparams.student is None or hparams.teacher == hparams.student)
-        self.do_calc_hidden_loss = (not self.different_base_models) and hparams.alpha_hid > 0
-        self.different_encoder = self.different_base_models or (student_encoder_layers != teacher_encoder_layers)
-        # self.different_encoder determines whether we need to run the teacher encoder
+        self.different_encoder = hparams.student_encoder_layers != teacher_encoder_layers
+        self.different_decoder = hparams.student_decoder_layers != teacher_decoder_layers
+
         self.teacher = teacher
         freeze_params(self.teacher)
 
@@ -80,28 +67,13 @@ class SummarizationDistiller(SummarizationModule):
                 del self.teacher.model.encoder
             except AttributeError:  # T5
                 del self.teacher.encoder
-
-        if e_layer_ids is None:
-            e_layer_ids = list(range(student_encoder_layers))
-        if d_layer_ids is None:
-            d_layer_ids = list(range(student_decoder_layers))
-
-        self.e_layer_ids, self.d_layer_ids = e_layer_ids, d_layer_ids  # type: List[int], List[int]
-
-        if self.do_calc_hidden_loss:  # Intermediate supervision: Decide which layers to supervise
-            if hparams.supervise_forward:
-                self.e_matches = get_layers_to_supervise(
-                    n_student=len(self.e_layer_ids), n_teacher=teacher_encoder_layers
-                )
-                self.d_matches = get_layers_to_supervise(
-                    n_student=len(self.d_layer_ids), n_teacher=teacher_decoder_layers
-                )
-            else:  # student layer should emulate hidden states of the teacher layer it was copied from
-                self.e_matches = self.e_layer_ids
-                self.d_matches = self.d_layer_ids
-        else:
-            self.e_matches = None
-            self.d_matches = None
+        # Intermediate supervision: Decide which layers to supervise
+        if hparams.supervise_forward:
+            self.e_matches = get_layers_to_supervise(n_student=len(self.e_layer_ids), n_teacher=teacher_encoder_layers)
+            self.d_matches = get_layers_to_supervise(n_student=len(self.d_layer_ids), n_teacher=teacher_decoder_layers)
+        else:  # student layer should emulate hidden states of the teacher layer it was copied from
+            self.e_matches = self.e_layer_ids
+            self.d_matches = self.d_layer_ids
 
         self.ce_loss_fct = nn.KLDivLoss(reduction="batchmean")
         self.temperature = 2.0
@@ -111,8 +83,22 @@ class SummarizationDistiller(SummarizationModule):
         gc.collect()
         torch.cuda.empty_cache()
 
+    def calc_mse_loss(self, teacher_outputs: torch.Tensor, student_outputs: torch.Tensor, mask) -> torch.FloatTensor:
+        """Supervise MSE(teacher.encoder_outputs, student.encoder_outputs)."""
+        # raise NotImplementedError()
+        if mask is not None:
+            # mask has False at padding_idx
+            sel_mask = mask[:, :, None].expand_as(student_outputs).bool()
+            s_logits_slct = torch.masked_select(student_outputs, sel_mask)
+            t_logits_slct = torch.masked_select(teacher_outputs, sel_mask)
+        else:
+            t_logits_slct = teacher_outputs
+            s_logits_slct = student_outputs
+        return F.mse_loss(s_logits_slct, t_logits_slct)
+
     def calc_ce_loss(self, mask, s_logits, t_logits):
         """Copy pasted from distillbert (transformers/examples/distillation/)"""
+
         # mask has False at padding_idx
         sel_mask = mask[:, :, None].expand_as(s_logits)
         vocab_size = s_logits.size(-1)
@@ -136,8 +122,8 @@ class SummarizationDistiller(SummarizationModule):
         add_distill_args(parser)
         return parser
 
-    def _step(self, batch: dict) -> tuple:
-        """Compute the loss for a batch"""
+    def _step(self, batch):
+        # assert is_frozen(self.teacher) copied_decoder_layers
         pad_token_id = self.tokenizer.pad_token_id
         input_ids, src_mask, labels = batch["input_ids"], batch["attention_mask"], batch["labels"]
         if isinstance(self.model, T5ForConditionalGeneration):
@@ -146,15 +132,14 @@ class SummarizationDistiller(SummarizationModule):
             decoder_input_ids = shift_tokens_right(labels, pad_token_id)
 
         # noinspection PyCallingNonCallable
-        student_outputs = self(
+        lm_logits, dec_hidden, enc_outputs, enc_hidden_state = self(
             input_ids,
             attention_mask=src_mask,
             decoder_input_ids=decoder_input_ids,
-            output_hidden_states=self.do_calc_hidden_loss,
+            output_hidden_states=True,
             output_attentions=False,
             use_cache=False,
         )
-        lm_logits = student_outputs["logits"]
 
         # Same cross entropy vs. label smoothing logic as finetune.py
         assert lm_logits.shape[-1] == self.model.config.vocab_size
@@ -163,7 +148,7 @@ class SummarizationDistiller(SummarizationModule):
             loss_fct = torch.nn.CrossEntropyLoss(ignore_index=pad_token_id)
             student_lm_loss = loss_fct(lm_logits.view(-1, lm_logits.shape[-1]), labels.view(-1))
         else:
-            lprobs = F.log_softmax(lm_logits, dim=-1)
+            lprobs = torch.nn.functional.log_softmax(lm_logits, dim=-1)
             student_lm_loss, _ = label_smoothed_nll_loss(
                 lprobs, labels, self.hparams.label_smoothing, ignore_index=pad_token_id
             )
@@ -171,44 +156,37 @@ class SummarizationDistiller(SummarizationModule):
         def zero_tensor():
             return torch.tensor(0.0).type_as(student_lm_loss)
 
-        teacher_enc_outputs = student_outputs[
-            "encoder_last_hidden_state"
-        ]  # use this unless self.different_base_models
         hid_loss_enc, hid_loss_dec = zero_tensor(), zero_tensor()
         if self.different_encoder:  # compute encoder hidden state loss
-            all_teacher_encoder_outputs = self.teacher.get_encoder()(
+            with torch.no_grad():
+                teacher_enc_hid = self.teacher.get_encoder()(
+                    input_ids, attention_mask=src_mask, output_hidden_states=True, return_dict=True
+                ).hidden_states
+
+            hid_loss_enc = self.calc_hidden_loss(
+                src_mask,
+                enc_hidden_state,
+                teacher_enc_hid,
+                self.e_matches,
+                normalize_hidden=self.hparams.normalize_hidden,
+            )
+
+        with torch.no_grad():
+            outputs = self.teacher(
                 input_ids,
                 attention_mask=src_mask,
-                output_hidden_states=self.do_calc_hidden_loss,
+                encoder_outputs=(enc_outputs,),
+                decoder_input_ids=decoder_input_ids,
+                lm_labels=labels,
+                output_hidden_states=True,
+                return_dict=True,
             )
-            if self.different_base_models:
-                teacher_enc_outputs = all_teacher_encoder_outputs["last_hidden_state"]
-            elif self.do_calc_hidden_loss:
-                hid_loss_enc = self.calc_hidden_loss(
-                    src_mask,
-                    student_outputs["encoder_hidden_states"],
-                    all_teacher_encoder_outputs["hidden_states"],
-                    self.e_matches,
-                    normalize_hidden=self.hparams.normalize_hidden,
-                )
-
-        teacher_outputs = self.teacher(
-            input_ids,
-            attention_mask=src_mask,
-            encoder_outputs=(teacher_enc_outputs,),
-            decoder_input_ids=decoder_input_ids,
-            output_hidden_states=self.do_calc_hidden_loss,
-            use_cache=False,  # since we are not passing labels, never let this default to True
-        )
+            tlogits, tdec_hidden = outputs.logits, outputs.decoder_hidden_states
         dec_mask = decoder_input_ids.ne(pad_token_id)
-        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, teacher_outputs["logits"])
-        if self.do_calc_hidden_loss:  # Intermediate supervision of decoder hidden states
+        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, tlogits)
+        if self.alpha_hid > 0:  # Intermediate supervision of decoder hidden states
             hid_loss_dec = self.calc_hidden_loss(
-                dec_mask,
-                student_outputs["decoder_hidden_states"],
-                teacher_outputs["decoder_hidden_states"],
-                self.d_matches,
-                normalize_hidden=self.hparams.normalize_hidden,
+                dec_mask, dec_hidden, tdec_hidden, self.d_matches, normalize_hidden=self.hparams.normalize_hidden
             )
 
         blended_loss = (
@@ -228,7 +206,6 @@ class SummarizationDistiller(SummarizationModule):
         valid_count = mask.sum() * hidden_states[0].size(-1)
         student_states = torch.stack([hidden_states[i] for i in range(len(matches))])
         teacher_states = torch.stack([hidden_states_T[j] for j in matches])
-        assert student_states.shape == teacher_states.shape, f"{student_states.shape} != {teacher_states.shape}"
         if normalize_hidden:
             student_states = F.layer_norm(student_states, student_states.shape[1:])
             teacher_states = F.layer_norm(teacher_states, teacher_states.shape[1:])
@@ -238,16 +215,10 @@ class SummarizationDistiller(SummarizationModule):
 
 
 def add_distill_args(parser):
-    # NOTE: if --student argument was specified and the teacher and student base models
-    # are different, the models still have to have the same tokenizer, specified by
-    # --tokenizer_name. So, for example, you can distill from t5_large to t5_small but not
-    # from bart to t5. This s because if the tokenizers are different, the output space
-    # for the two models is also different and their logits are not comparable.
     parser.add_argument("--teacher", type=str)
     parser.add_argument("--alpha_ce", default=0.8, type=float)
     parser.add_argument("--alpha_mlm", default=0.2, type=float)
     parser.add_argument("--alpha_hid", default=0.0, type=float, required=False)
-    parser.add_argument("--student", type=str, required=False)
     parser.add_argument("--student_decoder_layers", default=12, type=int, required=False)
     parser.add_argument("--student_encoder_layers", default=12, type=int, required=False)
     parser.add_argument("--no_teacher", action="store_true", default=False)
@@ -256,8 +227,8 @@ def add_distill_args(parser):
     parser.add_argument("--normalize_hidden", action="store_true", default=False)
 
 
-class TranslationDistiller(SummarizationDistiller):
-    """Supports T5, mBART, Marian, other models that inherit from Bart."""
+class BartTranslationDistiller(BartSummarizationDistiller):
+    """Supports Mbart, Marian, other models that inherit from Bart."""
 
     mode = "translation"
     metric_names = ["bleu"]
@@ -286,7 +257,7 @@ def create_module(args):
     if args.no_teacher:
         module_cls = TranslationModule if "translation" in args.task else SummarizationModule
     else:  # DISTILL WITH TEACHER
-        module_cls = TranslationDistiller if "translation" in args.task else SummarizationDistiller
+        module_cls = BartTranslationDistiller if "translation" in args.task else BartSummarizationDistiller
     args.setup_cls: str = module_cls.__name__
     print(f"using module {args.setup_cls}")
     model = module_cls(args)
@@ -304,7 +275,7 @@ def distill_main(args):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser = pl.Trainer.add_argparse_args(parser)
-    parser = SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+    parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
     args = parser.parse_args()
 
     distill_main(args)

examples/seq2seq/finetune.py

@@ -17,7 +17,7 @@ from torch.utils.data import DataLoader
 
 from callbacks import Seq2SeqLoggingCallback, get_checkpoint_callback, get_early_stopping_callback
 from transformers import MBartTokenizer, T5ForConditionalGeneration
-from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.modeling_bart import shift_tokens_right
 from utils import (
     ROUGE_KEYS,
     LegacySeq2SeqDataset,
@@ -148,7 +148,7 @@ class SummarizationModule(BaseTransformer):
             self.save_readable_batch(batch)
 
         outputs = self(src_ids, attention_mask=src_mask, decoder_input_ids=decoder_input_ids, use_cache=False)
-        lm_logits = outputs["logits"]
+        lm_logits = outputs[0]
         if self.hparams.label_smoothing == 0:
             # Same behavior as modeling_bart.py, besides ignoring pad_token_id
             ce_loss_fct = torch.nn.CrossEntropyLoss(ignore_index=pad_token_id)

examples/seq2seq/finetune_trainer.py

@@ -4,11 +4,17 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional
 
-import transformers
-from seq2seq_trainer import Seq2SeqTrainer
-from seq2seq_training_args import Seq2SeqTrainingArguments
-from transformers import AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer, HfArgumentParser, MBartTokenizer, set_seed
-from transformers.trainer_utils import EvaluationStrategy, is_main_process
+from seq2seq_trainer import Seq2SeqTrainer, arg_to_scheduler_choices
+from transformers import (
+    AutoConfig,
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    HfArgumentParser,
+    MBartTokenizer,
+    TrainingArguments,
+    set_seed,
+)
+from transformers.trainer_utils import EvaluationStrategy
 from utils import (
     Seq2SeqDataCollator,
     Seq2SeqDataset,
@@ -27,6 +33,41 @@ from utils import (
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class Seq2SeqTrainingArguments(TrainingArguments):
+    """
+    Parameters:
+        label_smoothing (:obj:`float`, `optional`, defaults to 0):
+            The label smoothing epsilon to apply (if not zero).
+        sortish_sampler (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether to SortishSamler or not. It sorts the inputs according to lenghts in-order to minimizing the padding size.
+        predict_with_generate (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether to use generate to calculate generative metrics (ROUGE, BLEU).
+    """
+
+    label_smoothing: Optional[float] = field(
+        default=0.0, metadata={"help": "The label smoothing epsilon to apply (if not zero)."}
+    )
+    sortish_sampler: bool = field(default=False, metadata={"help": "Whether to SortishSamler or not."})
+    predict_with_generate: bool = field(
+        default=False, metadata={"help": "Whether to use generate to calculate generative metrics (ROUGE, BLEU)."}
+    )
+    adafactor: bool = field(default=False, metadata={"help": "whether to use adafactor"})
+    encoder_layerdrop: Optional[float] = field(
+        default=None, metadata={"help": "Encoder layer dropout probability. Goes into model.config."}
+    )
+    decoder_layerdrop: Optional[float] = field(
+        default=None, metadata={"help": "Decoder layer dropout probability. Goes into model.config."}
+    )
+    dropout: Optional[float] = field(default=None, metadata={"help": "Dropout probability. Goes into model.config."})
+    attention_dropout: Optional[float] = field(
+        default=None, metadata={"help": "Attention dropout probability. Goes into model.config."}
+    )
+    lr_scheduler: Optional[str] = field(
+        default="linear", metadata={"help": f"Which lr scheduler to use. Selected in {arg_to_scheduler_choices}"}
+    )
+
+
 @dataclass
 class ModelArguments:
     """
@@ -43,8 +84,7 @@ class ModelArguments:
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
     cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
     freeze_encoder: bool = field(default=False, metadata={"help": "Whether tp freeze the encoder."})
     freeze_embeds: bool = field(default=False, metadata={"help": "Whether  to freeze the embeddings."})
@@ -133,11 +173,6 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
-    # Set the verbosity to info of the Transformers logger (on main process only):
-    if is_main_process(training_args.local_rank):
-        transformers.utils.logging.set_verbosity_info()
-        transformers.utils.logging.enable_default_handler()
-        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/seq2seq/run_distiller.sh

@@ -0,0 +1,10 @@
+# Add parent directory to python path to access lightning_base.py
+export PYTHONPATH="../":"${PYTHONPATH}"
+
+python distillation.py \
+--learning_rate=3e-4 \
+--do_train \
+--do_predict \
+--fp16 \
+--val_check_interval 0.1 \
+"$@"

examples/seq2seq/seq2seq_trainer.py

@@ -5,8 +5,8 @@ from torch import nn
 from torch.utils.data import DistributedSampler, RandomSampler
 
 from transformers import PreTrainedModel, Trainer, logging
+from transformers.configuration_fsmt import FSMTConfig
 from transformers.file_utils import is_torch_tpu_available
-from transformers.models.fsmt.configuration_fsmt import FSMTConfig
 from transformers.optimization import (
     Adafactor,
     AdamW,
@@ -30,6 +30,7 @@ arg_to_scheduler = {
     "constant": get_constant_schedule,
     "constant_w_warmup": get_constant_schedule_with_warmup,
 }
+arg_to_scheduler_choices = sorted(arg_to_scheduler.keys())
 
 
 class Seq2SeqTrainer(Trainer):
@@ -61,7 +62,10 @@ class Seq2SeqTrainer(Trainer):
             self.loss_fn = torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id)
         else:
             # dynamically import label_smoothed_nll_loss
-            from utils import label_smoothed_nll_loss
+            try:
+                from .utils import label_smoothed_nll_loss
+            except ImportError:
+                from utils import label_smoothed_nll_loss
 
             self.loss_fn = label_smoothed_nll_loss
 

examples/seq2seq/seq2seq_training_args.py

@@ -1,45 +0,0 @@
-import logging
-from dataclasses import dataclass, field
-from typing import Optional
-
-from seq2seq_trainer import arg_to_scheduler
-from transformers import TrainingArguments
-
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class Seq2SeqTrainingArguments(TrainingArguments):
-    """
-    Parameters:
-        label_smoothing (:obj:`float`, `optional`, defaults to 0):
-            The label smoothing epsilon to apply (if not zero).
-        sortish_sampler (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether to SortishSamler or not. It sorts the inputs according to lenghts in-order to minimizing the padding size.
-        predict_with_generate (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether to use generate to calculate generative metrics (ROUGE, BLEU).
-    """
-
-    label_smoothing: Optional[float] = field(
-        default=0.0, metadata={"help": "The label smoothing epsilon to apply (if not zero)."}
-    )
-    sortish_sampler: bool = field(default=False, metadata={"help": "Whether to SortishSamler or not."})
-    predict_with_generate: bool = field(
-        default=False, metadata={"help": "Whether to use generate to calculate generative metrics (ROUGE, BLEU)."}
-    )
-    adafactor: bool = field(default=False, metadata={"help": "whether to use adafactor"})
-    encoder_layerdrop: Optional[float] = field(
-        default=None, metadata={"help": "Encoder layer dropout probability. Goes into model.config."}
-    )
-    decoder_layerdrop: Optional[float] = field(
-        default=None, metadata={"help": "Decoder layer dropout probability. Goes into model.config."}
-    )
-    dropout: Optional[float] = field(default=None, metadata={"help": "Dropout probability. Goes into model.config."})
-    attention_dropout: Optional[float] = field(
-        default=None, metadata={"help": "Attention dropout probability. Goes into model.config."}
-    )
-    lr_scheduler: Optional[str] = field(
-        default="linear",
-        metadata={"help": f"Which lr scheduler to use. Selected in {sorted(arg_to_scheduler.keys())}"},
-    )