style

using Mixin
removed self.parameters() call in TransfoXL
2020-11-09 13:52:39 +01:00 · 2020-11-02 16:28:08 +01:00 · 2020-11-02 16:17:18 +01:00 · 2020-11-02 14:04:28 +01:00
94 changed files with 1823 additions and 4861 deletions
@@ -77,17 +77,16 @@ jobs:
                      - v0.4-torch_and_tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
            - run: pip install .[sklearn,tf-cpu,torch,testing]
            - save_cache:
                key: v0.4-{{ checksum "setup.py" }}
                paths:
                    - '~/.cache/pip'
-            - run: RUN_PT_TF_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf ./tests/ -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt
+            - run: RUN_PT_TF_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pt_tf_cross_test --durations=0 | tee output.txt
            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
    run_tests_torch:
        working_directory: ~/transformers
        docker:
@@ -103,126 +102,18 @@ jobs:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
            - run: pip install .[sklearn,torch,testing]
            - save_cache:
                  key: v0.4-torch-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -s --make_reports=tests ./tests/ | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

-    run_tests_torch_1_3:
-        working_directory: ~/transformers
-        docker:
-            - image: circleci/python:3.7
-        environment:
-            OMP_NUM_THREADS: 1
-        resource_class: xlarge
-        parallelism: 1
-        steps:
-            - checkout
-            - restore_cache:
-                  keys:
-                      - v0.4-torch-{{ checksum "setup.py" }}
-                      - v0.4-{{ checksum "setup.py" }}
-            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,testing]
-            - run: pip install torch==1.3.0
-            - save_cache:
-                  key: v0.4-torch-{{ checksum "setup.py" }}
-                  paths:
-                      - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
-    run_tests_torch_1_4:
-        working_directory: ~/transformers
-        docker:
-            - image: circleci/python:3.7
-        environment:
-            OMP_NUM_THREADS: 1
-        resource_class: xlarge
-        parallelism: 1
-        steps:
-            - checkout
-            - restore_cache:
-                  keys:
-                      - v0.4-torch-{{ checksum "setup.py" }}
-                      - v0.4-{{ checksum "setup.py" }}
-            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,testing]
-            - run: pip install torch==1.4.0
-            - save_cache:
-                  key: v0.4-torch-{{ checksum "setup.py" }}
-                  paths:
-                      - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
-    run_tests_torch_1_5:
-        working_directory: ~/transformers
-        docker:
-            - image: circleci/python:3.7
-        environment:
-            OMP_NUM_THREADS: 1
-        resource_class: xlarge
-        parallelism: 1
-        steps:
-            - checkout
-            - restore_cache:
-                  keys:
-                      - v0.4-torch-{{ checksum "setup.py" }}
-                      - v0.4-{{ checksum "setup.py" }}
-            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,testing]
-            - run: pip install torch==1.5.1
-            - save_cache:
-                  key: v0.4-torch-{{ checksum "setup.py" }}
-                  paths:
-                      - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
-    run_tests_torch_1_6:
-        working_directory: ~/transformers
-        docker:
-            - image: circleci/python:3.7
-        environment:
-            OMP_NUM_THREADS: 1
-        resource_class: xlarge
-        parallelism: 1
-        steps:
-            - checkout
-            - restore_cache:
-                  keys:
-                      - v0.4-torch-{{ checksum "setup.py" }}
-                      - v0.4-{{ checksum "setup.py" }}
-            - run: pip install --upgrade pip
-            - run: pip install .[sklearn,testing]
-            - run: pip install torch==1.6.0
-            - save_cache:
-                  key: v0.4-torch-{{ checksum "setup.py" }}
-                  paths:
-                      - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
-
    run_tests_tf:
        working_directory: ~/transformers
        docker:
@@ -238,17 +129,16 @@ jobs:
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
            - run: pip install .[sklearn,tf-cpu,testing]
            - save_cache:
                  key: v0.4-tf-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_tf ./tests/ | tee tests_output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+               path: ~/transformers/output.txt
+               destination: test_output.txt
    run_tests_flax:
        working_directory: ~/transformers
        docker:
@@ -264,17 +154,16 @@ jobs:
                    - v0.4-flax-{{ checksum "setup.py" }}
                    - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
            - run: sudo pip install .[flax,sklearn,torch,testing]
            - save_cache:
                  key: v0.4-flax-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_flax ./tests/ | tee tests_output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ | tee output.txt
            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
    run_tests_pipelines_torch:
        working_directory: ~/transformers
        docker:
@@ -290,17 +179,16 @@ jobs:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
            - run: pip install .[sklearn,torch,testing]
            - save_cache:
                  key: v0.4-torch-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
-            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test ./tests/ | tee tests_output.txt
+            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt
            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
    run_tests_pipelines_tf:
        working_directory: ~/transformers
        docker:
@@ -316,17 +204,16 @@ jobs:
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
+            - run: pip install git+https://github.com/huggingface/datasets
            - run: pip install .[sklearn,tf-cpu,testing]
            - save_cache:
                  key: v0.4-tf-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
-            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf ./tests/ -m is_pipeline_test | tee tests_output.txt
+            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s ./tests/ -m is_pipeline_test | tee output.txt
            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+               path: ~/transformers/output.txt
+               destination: test_output.txt
    run_tests_custom_tokenizers:
        working_directory: ~/transformers
        docker:
@@ -346,12 +233,10 @@ jobs:
                  key: v0.4-custom_tokenizers-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
-            - run: python -m pytest -s --make-reports=tests_custom_tokenizers ./tests/test_tokenization_bert_japanese.py | tee tests_output.txt
+            - run: python -m pytest -s ./tests/test_tokenization_bert_japanese.py | tee output.txt
            - store_artifacts:
-                  path: ~/transformers/tests_output.txt
-            - store_artifacts:
-                  path: ~/transformers/reports
-
+                path: ~/transformers/output.txt
+                destination: test_output.txt
    run_examples_torch:
        working_directory: ~/transformers
        docker:
@@ -373,12 +258,11 @@ jobs:
                  key: v0.4-torch_examples-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
-            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_torch ./examples/ | tee examples_output.txt
+            - run: python -m pytest -n 8 --dist=loadfile -s --make_reports=examples ./examples/ | tee examples_output.txt
            - store_artifacts:
                  path: ~/transformers/examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
-
    build_doc:
        working_directory: ~/transformers
        docker:
@@ -398,7 +282,6 @@ jobs:
            - run: cd docs && make html SPHINXOPTS="-W"
            - store_artifacts:
                path: ./docs/_build
-
    deploy_doc:
        working_directory: ~/transformers
        docker:
@@ -418,7 +301,6 @@ jobs:
                  paths:
                      - '~/.cache/pip'
            - run: ./.circleci/deploy.sh
-
    check_code_quality:
        working_directory: ~/transformers
        docker:
@@ -445,7 +327,6 @@ jobs:
            - run: python utils/check_copies.py
            - run: python utils/check_dummies.py
            - run: python utils/check_repo.py
-
    check_repository_consistency:
        working_directory: ~/transformers
        docker:
@@ -476,7 +357,6 @@ jobs:
            - setup_remote_docker
            - *build_push_docker
            - *deploy_cluster
-
    cleanup-gke-jobs:
        docker:
            - image: circleci/python:3.6
@@ -486,7 +366,6 @@ jobs:
                  cluster: $GKE_CLUSTER
                  perform-login: true
            - *delete_gke_jobs
-
 workflow_filters: &workflow_filters
    filters:
        branches:
@@ -502,10 +381,6 @@ workflows:
            - run_tests_custom_tokenizers
            - run_tests_torch_and_tf
            - run_tests_torch
-            - run_tests_torch_1_3
-            - run_tests_torch_1_4
-            - run_tests_torch_1_5
-            - run_tests_torch_1_6
            - run_tests_tf
            - run_tests_flax
            - run_tests_pipelines_torch
@@ -36,15 +36,13 @@ assignees: ''
 examples/distillation: @VictorSanh
 nlp datasets: [different repo](https://github.com/huggingface/nlp)
 rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)
- Text Generation: @patrickvonplaten @TevenLeScao
+ Text Generation: @TevenLeScao
 blenderbot: @mariamabarham
 Bart: @sshleifer
 Marian: @sshleifer
 T5: @patrickvonplaten
 Longformer/Reformer: @patrickvonplaten
- TransfoXL/XLNet: @TevenLeScao
- RAG: @patrickvonplaten, @lhoestq
- FSTM: @stas00
+ TransfoXL/XLNet: @TevenLeScao 
 examples/seq2seq: @sshleifer
 examples/bert-loses-patience: @JetRunner
 tensorflow: @jplu
@@ -60,5 +60,4 @@ members/contributors which may be interested in your PR.
 tensorflow: @jplu
 examples/token-classification: @stefan-it
 documentation: @sgugger
- FSTM: @stas00
 -->
@@ -61,19 +61,7 @@ jobs:
          CUDA_VISIBLE_DEVICES: 0
        run: |
          source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_torch_gpu_failures_short.txt
-        
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_torch_gpu_test_reports
-          path: reports
-                  
+          python -m pytest -n 2 --dist=loadfile -s ./tests/

  run_tests_tf_gpu:
    runs-on: [self-hosted, single-gpu]
@@ -122,18 +110,7 @@ jobs:
          CUDA_VISIBLE_DEVICES: 0
        run: |
          source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_gpu_failures_short.txt
-        
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_tf_gpu_test_reports
-          path: reports
+          python -m pytest -n 2 --dist=loadfile -s ./tests/

  run_tests_torch_multiple_gpu:
    runs-on: [self-hosted, multi-gpu]
@@ -181,18 +158,7 @@ jobs:
          OMP_NUM_THREADS: 1
        run: |
          source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_multiple_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_torch_multiple_gpu_failures_short.txt          
-
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_torch_multi_gpu_test_reports
-          path: reports
+          python -m pytest -n 2 --dist=loadfile -s ./tests/

  run_tests_tf_multiple_gpu:
    runs-on: [self-hosted, multi-gpu]
@@ -240,16 +206,4 @@ jobs:
          OMP_NUM_THREADS: 1
        run: |
          source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_multiple_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_multiple_gpu_failures_short.txt
-
-      - name: Test suite reports artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: run_all_tests_tf_multi_gpu_test_reports
-          path: reports
-          
+          python -m pytest -n 2 --dist=loadfile -s ./tests/
@@ -60,11 +60,11 @@ jobs:
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=tests_torch tests

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_torch_gpu_failures_short.txt
+        run: cat reports/report_test_torch_failures_short.txt
        
      - name: Run examples tests on GPU
        if: ${{ always() }}
@@ -74,11 +74,11 @@ jobs:
        run: |
          source .env/bin/activate
          pip install -r examples/requirements.txt
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_gpu examples
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=examples_torch examples

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/examples_torch_gpu_failures_short.txt
+        run: cat reports/report_examples_torch_failures_short.txt

      - name: Run all pipeline tests on GPU
        if: ${{ always() }}
@@ -89,11 +89,11 @@ jobs:
          RUN_PIPELINE_TESTS: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_torch_pipeline tests

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_torch_pipeline_gpu_failures_short.txt
+        run: cat reports/report_tests_torch_pipeline_failures_short.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
@@ -154,26 +154,11 @@ jobs:
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=tests_tf tests
          
      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_tf_gpu_failures_short.txt
-
-      - name: Run all pipeline tests on GPU
-        if: ${{ always() }}
-        env:
-          TF_FORCE_GPU_ALLOW_GROWTH: "true"
-          OMP_NUM_THREADS: 1
-          RUN_SLOW: yes
-          RUN_PIPELINE_TESTS: yes
-        run: |
-          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_pipelines_gpu_failures_short.txt
+        run: cat reports/report_test_tf_failures_short.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
@@ -192,7 +177,7 @@ jobs:
        id: cache
        with:
          path: .env
-          key: v1.1-slow_tests_torch_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v0.1-slow_tests_torch_multi_gpu-${{ hashFiles('setup.py') }}

      - name: Python version
        run: |
@@ -233,12 +218,26 @@ jobs:
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_multiple_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=tests_torch tests

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_torch_multiple_gpu_failures_short.txt
+        run: cat reports/report_test_torch_failures_short.txt
        
+      - name: Run examples tests on GPU
+        if: ${{ always() }}
+        env:
+          OMP_NUM_THREADS: 1
+          RUN_SLOW: yes
+        run: |
+          source .env/bin/activate
+          pip install -r examples/requirements.txt
+          python -m pytest -n 1 --dist=loadfile -s --make_reports=examples_torch examples
+
+      - name: Failure short reports
+        if: ${{ always() }}
+        run: cat reports/report_examples_torch_failures_short.txt
+
      - name: Run all pipeline tests on GPU
        if: ${{ always() }}
        env:
@@ -248,11 +247,11 @@ jobs:
          RUN_PIPELINE_TESTS: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_multiple_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make_reports=tests_torch_pipeline tests

      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_torch_pipeline_multiple_gpu_failures_short.txt
+        run: cat reports/report_tests_torch_pipeline_failures_short.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
@@ -271,7 +270,7 @@ jobs:
        id: cache
        with:
          path: .env
-          key: v1.1-slow_tests_tf_multi_gpu-${{ hashFiles('setup.py') }}
+          key: v0.1-slow_tests_tf_multi_gpu-${{ hashFiles('setup.py') }}

      - name: Python version
        run: |
@@ -312,14 +311,9 @@ jobs:
          RUN_SLOW: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_multiple_gpu tests
-
-      - name: Failure short reports
-        if: ${{ always() }}
-        run: cat reports/tests_tf_multiple_gpu_failures_short.txt
+          python -m pytest -n 1 --dist=loadfile -s ./tests/ --durations=0

      - name: Run all pipeline tests on GPU
-        if: ${{ always() }}
        env:
          TF_FORCE_GPU_ALLOW_GROWTH: "true"
          OMP_NUM_THREADS: 1
@@ -327,11 +321,11 @@ jobs:
          RUN_PIPELINE_TESTS: yes
        run: |
          source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multiple_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s ./tests/ -m is_pipeline_test --durations=50
          
      - name: Failure short reports
        if: ${{ always() }}
-        run: cat reports/tests_tf_multiple_gpu_pipelines_failures_short.txt
+        run: cat reports/report_test_tf_failures_short.txt

      - name: Test suite reports artifacts
        if: ${{ always() }}
@@ -339,4 +333,4 @@ jobs:
        with:
          name: run_all_tests_tf_multi_gpu_test_reports
          path: reports
-          
+          
@@ -272,4 +272,3 @@ conversion utilities for the following models:
    internal/pipelines_utils
    internal/tokenization_utils
    internal/trainer_utils
-    internal/generation_utils
@@ -1,50 +0,0 @@
-Utilities for Generation
-----------------------------------------------------------------------------------------------------------------------
-
-This page lists all the utility functions used by :meth:`~transformers.PretrainedModel.generate`,
-:meth:`~transformers.PretrainedModel.greedy_search`, :meth:`~transformers.PretrainedModel.sample`,
-:meth:`~transformers.PretrainedModel.beam_search`, and :meth:`~transformers.PretrainedModel.beam_sample`.
-
-Most of those are only useful if you are studying the code of the generate methods in the library.
-
-LogitsProcessor
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-A :class:`~transformers.LogitsProcessor` can be used to modify the prediction scores of a language model head for
-generation.
-
-.. autoclass:: transformers.LogitsProcessor
-    :members: __call__
-
-.. autoclass:: transformers.LogitsProcessorList
-    :members: __call__
-
-.. autoclass:: transformers.MinLengthLogitsProcessor
-    :members: __call__
-
-.. autoclass:: transformers.TemperatureLogitsWarper
-    :members: __call__
-
-.. autoclass:: transformers.RepetitionPenaltyLogitsProcessor
-    :members: __call__
-
-.. autoclass:: transformers.TopPLogitsWarper
-    :members: __call__
-
-.. autoclass:: transformers.TopKLogitsWarper
-    :members: __call__
-
-.. autoclass:: transformers.NoRepeatNGramLogitsProcessor
-    :members: __call__
-
-.. autoclass:: transformers.NoBadWordsLogitsProcessor
-    :members: __call__
-
-BeamSearch
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.BeamScorer
-    :members: process, finalize
-
-.. autoclass:: transformers.BeamSearchScorer
-    :members: process, finalize
@@ -45,7 +45,7 @@ TFModelUtilsMixin
    :members:


-Generation
+Generative models
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.generation_utils.GenerationMixin
@@ -68,7 +68,7 @@ For example for `run_glue`:

 ```bash
 python examples/xla_spawn.py --num_cores 8 \
-	examples/text-classification/run_glue.py \
+	examples/text-classification/run_glue.py
 	--model_name_or_path bert-base-cased \
 	--task_name mnli \
 	--data_dir ./data/glue_data/MNLI \
@@ -17,14 +17,17 @@ warnings.simplefilter(action="ignore", category=FutureWarning)


 def pytest_addoption(parser):
-    from transformers.testing_utils import pytest_addoption_shared
-
-    pytest_addoption_shared(parser)
+    parser.addoption(
+        "--make_reports",
+        action="store",
+        default=False,
+        help="generate report files - the value will be used as a `report_`+val+`reportname.txt`",
+    )


 def pytest_terminal_summary(terminalreporter):
    from transformers.testing_utils import pytest_terminal_summary_main

-    make_reports = terminalreporter.config.getoption("--make-reports")
+    make_reports = terminalreporter.config.getoption("--make_reports")
    if make_reports:
        pytest_terminal_summary_main(terminalreporter, id=make_reports)
@@ -10,7 +10,7 @@ These scripts leverage the 🤗 Datasets library and the Trainer API. You can ea
 need extra processing on your datasets.

 **Note:** The old script `run_language_modeling.py` is still available
-[here](https://github.com/huggingface/transformers/blob/master/examples/contrib/legacy/run_language_modeling.py).
+[here](https://github.com/huggingface/transformers/blob/master/examples/contrib/legacy/language-modeling/run_language_modeling.py).

 The following examples, will run on a datasets hosted on our [hub](https://huggingface.co/datasets) or with your own
 text files for training and validation. We give examples of both below.
@@ -68,25 +68,19 @@ python run_mlm.py \
 To run on your own training and validation files, use the following command:

 ```bash
-python run_mlm.py \
+python run_clm.py \
    --model_name_or_path roberta-base \
    --train_file path_to_train_file \
    --validation_file path_to_validation_file \
    --do_train \
    --do_eval \
-    --output_dir /tmp/test-mlm
+    --output_dir /tmp/test-clm
 ```

-If your dataset is organized with one sample per line, you can use the `--line_by_line` flag (otherwise the script
-concatenates all texts and then splits them in blocks of the same length).
-
-**Note:** On TPU, you should use the flag `--pad_to_max_length` in conjunction with the `--line_by_line` flag to make
-sure all your batches have the same length.
-
 ### Whole word masking

 The BERT authors released a new version of BERT using Whole Word Masking in May 2019. Instead of masking randomly
-selected tokens (which may be part of words), they mask randomly selected words (masking all the tokens corresponding
+selected tokens (which may be aprt of words), they mask randomly selected words (masking all the tokens corresponding
 to that word). This technique has been refined for Chinese in [this paper](https://arxiv.org/abs/1906.08101).

 To fine-tune a model using whole word masking, use the following script:
@@ -117,8 +111,8 @@ It works well on so many Chines Task like CLUE (Chinese GLUE). They use LTP, so
 we need LTP.

 Now LTP only only works well on `transformers==3.2.0`. So we don't add it to requirements.txt.
-You need to create a separate environment with this version of Transformers to run the `run_chinese_ref.py` script that
-will create the reference files. The script is in `examples/contrib`. Once in the proper environment, run the
+You need to create a separate enviromnent with this version of Transformers to run the `run_chinese_ref.py` script that
+will create the reference files. The script is in `examples/contrib`. Once in the proper enviromnent, run the
 following:


@@ -150,8 +144,6 @@ python run_mlm_wwm.py \
    --output_dir /tmp/test-mlm-wwm
 ```

-**Note:** On TPU, you should the flag `--pad_to_max_length` to make sure all your batches have the same length.
-
 ### XLNet and permutation language modeling

 XLNet uses a different training objective, which is permutation language modeling. It is an autoregressive method 
@@ -187,9 +179,3 @@ python run_plm.py \
    --do_eval \
    --output_dir /tmp/test-plm
 ```
-
-If your dataset is organized with one sample per line, you can use the `--line_by_line` flag (otherwise the script
-concatenates all texts and then splits them in blocks of the same length).
-
-**Note:** On TPU, you should use the flag `--pad_to_max_length` in conjunction with the `--line_by_line` flag to make
-sure all your batches have the same length.
@@ -190,7 +190,7 @@ def main():
        if data_args.train_file is not None:
            data_files["train"] = data_args.train_file
        if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
+            data_files["validation"] = data_args.train_file
        extension = data_args.train_file.split(".")[-1]
        if extension == "txt":
            extension = "text"
@@ -116,17 +116,6 @@ class DataTrainingArguments:
    mlm_probability: float = field(
        default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
    )
-    line_by_line: bool = field(
-        default=False,
-        metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."},
-    )
-    pad_to_max_length: bool = field(
-        default=False,
-        metadata={
-            "help": "Whether to pad all samples to `max_seq_length`. "
-            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
-        },
-    )

    def __post_init__(self):
        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
@@ -201,7 +190,7 @@ def main():
        if data_args.train_file is not None:
            data_files["train"] = data_args.train_file
        if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
+            data_files["validation"] = data_args.train_file
        extension = data_args.train_file.split(".")[-1]
        if extension == "txt":
            extension = "text"
@@ -257,83 +246,18 @@ def main():
        column_names = datasets["validation"].column_names
    text_column_name = "text" if "text" in column_names else column_names[0]

-    if data_args.line_by_line:
-        # When using line_by_line, we just tokenize each nonempty line.
-        padding = "max_length" if data_args.pad_to_max_length else False
+    def tokenize_function(examples):
+        # Remove empty lines
+        examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
+        return tokenizer(examples["text"], truncation=True, max_length=data_args.max_seq_length)

-        def tokenize_function(examples):
-            # Remove empty lines
-            examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
-            return tokenizer(
-                examples["text"],
-                padding=padding,
-                truncation=True,
-                max_length=data_args.max_seq_length,
-                # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it
-                # receives the `special_tokens_mask`.
-                return_special_tokens_mask=True,
-            )
-
-        tokenized_datasets = datasets.map(
-            tokenize_function,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
-    else:
-        # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts.
-        # We use `return_special_tokens_mask=True` because DataCollatorForLanguageModeling (see below) is more
-        # efficient when it receives the `special_tokens_mask`.
-        def tokenize_function(examples):
-            return tokenizer(examples[text_column_name], return_special_tokens_mask=True)
-
-        tokenized_datasets = datasets.map(
-            tokenize_function,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
-
-        if data_args.max_seq_length is None:
-            max_seq_length = tokenizer.model_max_length
-        else:
-            if data_args.max_seq_length > tokenizer.model_max_length:
-                logger.warn(
-                    f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the"
-                    f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."
-                )
-            max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)
-
-        # Main data processing function that will concatenate all texts from our dataset and generate chunks of
-        # max_seq_length.
-        def group_texts(examples):
-            # Concatenate all texts.
-            concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
-            total_length = len(concatenated_examples[list(examples.keys())[0]])
-            # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
-            # customize this part to your needs.
-            total_length = (total_length // max_seq_length) * max_seq_length
-            # Split by chunks of max_len.
-            result = {
-                k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)]
-                for k, t in concatenated_examples.items()
-            }
-            return result
-
-        # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a
-        # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value
-        # might be slower to preprocess.
-        #
-        # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
-        # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map
-        tokenized_datasets = tokenized_datasets.map(
-            group_texts,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
+    tokenized_datasets = datasets.map(
+        tokenize_function,
+        batched=True,
+        num_proc=data_args.preprocessing_num_workers,
+        remove_columns=[text_column_name],
+        load_from_cache_file=not data_args.overwrite_cache,
+    )

    # Data collator
    # This one will take care of randomly masking the tokens.
@@ -120,13 +120,6 @@ class DataTrainingArguments:
    mlm_probability: float = field(
        default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
    )
-    pad_to_max_length: bool = field(
-        default=False,
-        metadata={
-            "help": "Whether to pad all samples to `max_seq_length`. "
-            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
-        },
-    )

    def __post_init__(self):
        if self.train_file is not None:
@@ -204,7 +197,7 @@ def main():
    if data_args.train_file is not None:
        data_files["train"] = data_args.train_file
    if data_args.validation_file is not None:
-        data_files["validation"] = data_args.validation_file
+        data_files["validation"] = data_args.train_file
    extension = data_args.train_file.split(".")[-1]
    if extension == "txt":
        extension = "text"
@@ -260,12 +253,10 @@ def main():
        column_names = datasets["validation"].column_names
    text_column_name = "text" if "text" in column_names else column_names[0]

-    padding = "max_length" if data_args.pad_to_max_length else False
-
    def tokenize_function(examples):
        # Remove empty lines
        examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
-        return tokenizer(examples["text"], padding=padding, truncation=True, max_length=data_args.max_seq_length)
+        return tokenizer(examples["text"], truncation=True, max_length=data_args.max_seq_length)

    tokenized_datasets = datasets.map(
        tokenize_function,
@@ -113,17 +113,6 @@ class DataTrainingArguments:
    max_span_length: int = field(
        default=5, metadata={"help": "Maximum length of a span of masked tokens for permutation language modeling."}
    )
-    line_by_line: bool = field(
-        default=False,
-        metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."},
-    )
-    pad_to_max_length: bool = field(
-        default=False,
-        metadata={
-            "help": "Whether to pad all samples to `max_seq_length`. "
-            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
-        },
-    )

    def __post_init__(self):
        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
@@ -198,7 +187,7 @@ def main():
        if data_args.train_file is not None:
            data_files["train"] = data_args.train_file
        if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
+            data_files["validation"] = data_args.train_file
        extension = data_args.train_file.split(".")[-1]
        if extension == "txt":
            extension = "text"
@@ -254,73 +243,18 @@ def main():
        column_names = datasets["validation"].column_names
    text_column_name = "text" if "text" in column_names else column_names[0]

-    if data_args.line_by_line:
-        # When using line_by_line, we just tokenize each nonempty line.
-        padding = "max_length" if data_args.pad_to_max_length else False
+    def tokenize_function(examples):
+        # Remove empty lines
+        examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
+        return tokenizer(examples["text"], truncation=True, max_length=data_args.max_seq_length)

-        def tokenize_function(examples):
-            # Remove empty lines
-            examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
-            return tokenizer(examples["text"], padding=padding, truncation=True, max_length=data_args.max_seq_length)
-
-        tokenized_datasets = datasets.map(
-            tokenize_function,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
-    else:
-        # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts.
-        def tokenize_function(examples):
-            return tokenizer(examples[text_column_name])
-
-        tokenized_datasets = datasets.map(
-            tokenize_function,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
-
-        if data_args.max_seq_length is None:
-            max_seq_length = tokenizer.model_max_length
-        else:
-            if data_args.max_seq_length > tokenizer.model_max_length:
-                logger.warn(
-                    f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the"
-                    f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."
-                )
-            max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)
-
-        # Main data processing function that will concatenate all texts from our dataset and generate chunks of
-        # max_seq_length.
-        def group_texts(examples):
-            # Concatenate all texts.
-            concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
-            total_length = len(concatenated_examples[list(examples.keys())[0]])
-            # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
-            # customize this part to your needs.
-            total_length = (total_length // max_seq_length) * max_seq_length
-            # Split by chunks of max_len.
-            result = {
-                k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)]
-                for k, t in concatenated_examples.items()
-            }
-            return result
-
-        # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a
-        # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value
-        # might be slower to preprocess.
-        #
-        # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
-        # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map
-        tokenized_datasets = tokenized_datasets.map(
-            group_texts,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            load_from_cache_file=not data_args.overwrite_cache,
-        )
+    tokenized_datasets = datasets.map(
+        tokenize_function,
+        batched=True,
+        num_proc=data_args.preprocessing_num_workers,
+        remove_columns=[text_column_name],
+        load_from_cache_file=not data_args.overwrite_cache,
+    )

    # Data collator
    data_collator = DataCollatorForPermutationLanguageModeling(
@@ -7,7 +7,6 @@ from typing import Any, Dict
 import pytorch_lightning as pl
 from pytorch_lightning.utilities import rank_zero_info

-import pkg_resources
 from transformers import (
    AdamW,
    AutoConfig,
@@ -33,15 +32,6 @@ from transformers.optimization import (

 logger = logging.getLogger(__name__)

-try:
-    pkg = "pytorch_lightning"
-    min_ver = "1.0.4"
-    pkg_resources.require(f"{pkg}>={min_ver}")
-except pkg_resources.VersionConflict:
-    logger.warning(
-        f"{pkg}>={min_ver} is required for a normal functioning of this module, but found {pkg}=={pkg_resources.get_distribution(pkg).version}. Try pip install -r examples/requirements.txt"
-    )
-

 MODEL_MODES = {
    "base": AutoModel,
@@ -4,9 +4,16 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional

-from seq2seq_trainer import Seq2SeqTrainer
-from seq2seq_training_args import Seq2SeqTrainingArguments
-from transformers import AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer, HfArgumentParser, MBartTokenizer, set_seed
+from seq2seq_trainer import Seq2SeqTrainer, arg_to_scheduler_choices
+from transformers import (
+    AutoConfig,
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    HfArgumentParser,
+    MBartTokenizer,
+    TrainingArguments,
+    set_seed,
+)
 from transformers.trainer_utils import EvaluationStrategy
 from utils import (
    Seq2SeqDataCollator,
@@ -26,6 +33,41 @@ from utils import (
 logger = logging.getLogger(__name__)


+@dataclass
+class Seq2SeqTrainingArguments(TrainingArguments):
+    """
+    Parameters:
+        label_smoothing (:obj:`float`, `optional`, defaults to 0):
+            The label smoothing epsilon to apply (if not zero).
+        sortish_sampler (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether to SortishSamler or not. It sorts the inputs according to lenghts in-order to minimizing the padding size.
+        predict_with_generate (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether to use generate to calculate generative metrics (ROUGE, BLEU).
+    """
+
+    label_smoothing: Optional[float] = field(
+        default=0.0, metadata={"help": "The label smoothing epsilon to apply (if not zero)."}
+    )
+    sortish_sampler: bool = field(default=False, metadata={"help": "Whether to SortishSamler or not."})
+    predict_with_generate: bool = field(
+        default=False, metadata={"help": "Whether to use generate to calculate generative metrics (ROUGE, BLEU)."}
+    )
+    adafactor: bool = field(default=False, metadata={"help": "whether to use adafactor"})
+    encoder_layerdrop: Optional[float] = field(
+        default=None, metadata={"help": "Encoder layer dropout probability. Goes into model.config."}
+    )
+    decoder_layerdrop: Optional[float] = field(
+        default=None, metadata={"help": "Decoder layer dropout probability. Goes into model.config."}
+    )
+    dropout: Optional[float] = field(default=None, metadata={"help": "Dropout probability. Goes into model.config."})
+    attention_dropout: Optional[float] = field(
+        default=None, metadata={"help": "Attention dropout probability. Goes into model.config."}
+    )
+    lr_scheduler: Optional[str] = field(
+        default="linear", metadata={"help": f"Which lr scheduler to use. Selected in {arg_to_scheduler_choices}"}
+    )
+
+
@dataclass
 class ModelArguments:
    """
@@ -30,6 +30,7 @@ arg_to_scheduler = {
    "constant": get_constant_schedule,
    "constant_w_warmup": get_constant_schedule_with_warmup,
 }
+arg_to_scheduler_choices = sorted(arg_to_scheduler.keys())


 class Seq2SeqTrainer(Trainer):
@@ -61,7 +62,10 @@ class Seq2SeqTrainer(Trainer):
            self.loss_fn = torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id)
        else:
            # dynamically import label_smoothed_nll_loss
-            from utils import label_smoothed_nll_loss
+            try:
+                from .utils import label_smoothed_nll_loss
+            except ImportError:
+                from utils import label_smoothed_nll_loss

            self.loss_fn = label_smoothed_nll_loss

@@ -1,45 +0,0 @@
-import logging
-from dataclasses import dataclass, field
-from typing import Optional
-
-from seq2seq_trainer import arg_to_scheduler
-from transformers import TrainingArguments
-
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class Seq2SeqTrainingArguments(TrainingArguments):
-    """
-    Parameters:
-        label_smoothing (:obj:`float`, `optional`, defaults to 0):
-            The label smoothing epsilon to apply (if not zero).
-        sortish_sampler (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether to SortishSamler or not. It sorts the inputs according to lenghts in-order to minimizing the padding size.
-        predict_with_generate (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether to use generate to calculate generative metrics (ROUGE, BLEU).
-    """
-
-    label_smoothing: Optional[float] = field(
-        default=0.0, metadata={"help": "The label smoothing epsilon to apply (if not zero)."}
-    )
-    sortish_sampler: bool = field(default=False, metadata={"help": "Whether to SortishSamler or not."})
-    predict_with_generate: bool = field(
-        default=False, metadata={"help": "Whether to use generate to calculate generative metrics (ROUGE, BLEU)."}
-    )
-    adafactor: bool = field(default=False, metadata={"help": "whether to use adafactor"})
-    encoder_layerdrop: Optional[float] = field(
-        default=None, metadata={"help": "Encoder layer dropout probability. Goes into model.config."}
-    )
-    decoder_layerdrop: Optional[float] = field(
-        default=None, metadata={"help": "Decoder layer dropout probability. Goes into model.config."}
-    )
-    dropout: Optional[float] = field(default=None, metadata={"help": "Dropout probability. Goes into model.config."})
-    attention_dropout: Optional[float] = field(
-        default=None, metadata={"help": "Attention dropout probability. Goes into model.config."}
-    )
-    lr_scheduler: Optional[str] = field(
-        default="linear",
-        metadata={"help": f"Which lr scheduler to use. Selected in {sorted(arg_to_scheduler.keys())}"},
-    )
@@ -1,13 +1,11 @@
-import os
 import tempfile
 import unittest

-from transformers.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
+from transformers.convert_marian_tatoeba_to_pytorch import TatoebaConverter
 from transformers.file_utils import cached_property
 from transformers.testing_utils import slow


-@unittest.skipUnless(os.path.exists(DEFAULT_REPO), "Tatoeba directory does not exist.")
 class TatoebaConversionTester(unittest.TestCase):
    @cached_property
    def resolver(self):
@@ -66,16 +66,14 @@ class Split(Enum):


 class TokenClassificationTask:
-    @staticmethod
-    def read_examples_from_file(data_dir, mode: Union[Split, str]) -> List[InputExample]:
+    def read_examples_from_file(self, data_dir, mode: Union[Split, str]) -> List[InputExample]:
        raise NotImplementedError

-    @staticmethod
-    def get_labels(path: str) -> List[str]:
+    def get_labels(self, path: str) -> List[str]:
        raise NotImplementedError

-    @staticmethod
    def convert_examples_to_features(
+        self,
        examples: List[InputExample],
        label_list: List[str],
        max_seq_length: int,
@@ -1,60 +0,0 @@
---
-language: 
- en
-tags:
- bluebert
-license: 
- PUBLIC DOMAIN NOTICE
-datasets:
- pubmed
-
---
-
-# BlueBert-Base, Uncased, PubMed
-
-## Model description
-
-A BERT model pre-trained on PubMed abstracts
-
-## Intended uses & limitations
-
-#### How to use
-
-Please see https://github.com/ncbi-nlp/bluebert
-
-## Training data
-
-We provide [preprocessed PubMed texts](https://ftp.ncbi.nlm.nih.gov/pub/lu/Suppl/NCBI-BERT/pubmed_uncased_sentence_nltk.txt.tar.gz) that were used to pre-train the BlueBERT models. 
-The corpus contains ~4000M words extracted from the [PubMed ASCII code version](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PubMed/). 
-
-Pre-trained model: https://huggingface.co/bert-base-uncased
-
-## Training procedure
-
-*  lowercasing the text
-*  removing speical chars `\x00`-`\x7F`
-*  tokenizing the text using the [NLTK Treebank tokenizer](https://www.nltk.org/_modules/nltk/tokenize/treebank.html)
-
-Below is a code snippet for more details.
-
-```python
-value = value.lower()
-value = re.sub(r'[\r\n]+', ' ', value)
-value = re.sub(r'[^\x00-\x7F]+', ' ', value)
-
-tokenized = TreebankWordTokenizer().tokenize(value)
-sentence = ' '.join(tokenized)
-sentence = re.sub(r"\s's\b", "'s", sentence)
-```
-
-### BibTeX entry and citation info
-
-```bibtex
-@InProceedings{peng2019transfer,
-  author    = {Yifan Peng and Shankai Yan and Zhiyong Lu},
-  title     = {Transfer Learning in Biomedical Natural Language Processing: An Evaluation of BERT and ELMo on Ten Benchmarking Datasets},
-  booktitle = {Proceedings of the 2019 Workshop on Biomedical Natural Language Processing (BioNLP 2019)},
-  year      = {2019},
-  pages     = {58--65},
-}
-```
@@ -1,117 +0,0 @@
---
-datasets:
- squad_v2
---
-
-# roberta-base for QA 
-
-## Overview
-**Language model:** roberta-base  
-**Language:** English  
-**Downstream-task:** Extractive QA  
-**Training data:** SQuAD 2.0  
-**Eval data:** SQuAD 2.0  
-**Code:**  See [example](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) in [FARM](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py)  
-**Infrastructure**: 4x Tesla v100
-
-## Hyperparameters
-
-```
-batch_size = 96
-n_epochs = 2
-base_LM_model = "roberta-base"
-max_seq_len = 386
-learning_rate = 3e-5
-lr_schedule = LinearWarmup
-warmup_proportion = 0.2
-doc_stride=128
-max_query_length=64
-``` 
-
-## Performance
-Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/).
-
-```
-"exact": 79.97136359807968
-"f1": 83.00449234495325
-
-"total": 11873
-"HasAns_exact": 78.03643724696356
-"HasAns_f1": 84.11139298441825
-"HasAns_total": 5928
-"NoAns_exact": 81.90075693860386
-"NoAns_f1": 81.90075693860386
-"NoAns_total": 5945
-```
-
-## Usage
-
-### In Transformers
-```python
-from transformers.pipelines import pipeline
-from transformers.modeling_auto import AutoModelForQuestionAnswering
-from transformers.tokenization_auto import AutoTokenizer
-
-model_name = "deepset/roberta-base-squad2-v2"
-
-# a) Get predictions
-nlp = pipeline('question-answering', model=model_name, tokenizer=model_name)
-QA_input = {
-    'question': 'Why is model conversion important?',
-    'context': 'The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks.'
-}
-res = nlp(QA_input)
-
-# b) Load model & tokenizer
-model = AutoModelForQuestionAnswering.from_pretrained(model_name)
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-```
-
-### In FARM
-
-```python
-from farm.modeling.adaptive_model import AdaptiveModel
-from farm.modeling.tokenization import Tokenizer
-from farm.infer import Inferencer
-
-model_name = "deepset/roberta-base-squad2-v2"
-
-# a) Get predictions
-nlp = Inferencer.load(model_name, task_type="question_answering")
-QA_input = [{"questions": ["Why is model conversion important?"],
-             "text": "The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."}]
-res = nlp.inference_from_dicts(dicts=QA_input, rest_api_schema=True)
-
-# b) Load model & tokenizer
-model = AdaptiveModel.convert_from_transformers(model_name, device="cpu", task_type="question_answering")
-tokenizer = Tokenizer.load(model_name)
-```
-
-### In haystack
-For doing QA at scale (i.e. many docs instead of single paragraph), you can load the model also in [haystack](https://github.com/deepset-ai/haystack/):
-```python
-reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2")
-# or 
-reader = TransformersReader(model="deepset/roberta-base-squad2",tokenizer="deepset/roberta-base-squad2")
-```
-
-
-## Authors
-Branden Chan: `branden.chan [at] deepset.ai`
-Timo Möller: `timo.moeller [at] deepset.ai`
-Malte Pietsch: `malte.pietsch [at] deepset.ai`
-Tanay Soni: `tanay.soni [at] deepset.ai`
-
-## About us
-![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png)
-
-We bring NLP to the industry via open source!  
-Our focus: Industry specific language models & large scale QA systems.  
-  
-Some of our work: 
- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert)
- [FARM](https://github.com/deepset-ai/FARM)
- [Haystack](https://github.com/deepset-ai/haystack/)
-
-Get in touch:
-[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai)
@@ -5,8 +5,6 @@ datasets:

 # roberta-base for QA 

-NOTE: This model has been superseded by deepset/roberta-base-squad2-v2. For an explanation of why, see [this github issue](https://github.com/deepset-ai/FARM/issues/552) from the FARM repository.
-
 ## Overview
 **Language model:** roberta-base  
 **Language:** English  
@@ -108,3 +106,4 @@ Some of our work:

 Get in touch:
 [Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai)
+
@@ -1,40 +1,3 @@
 ---
-language: en
 license: apache-2.0
-datasets:
- bookcorpus
- wikipedia
 ---
-
-# DistilBERT base model (cased)
-
-This model is a distilled version of the [BERT base model](https://huggingface.co/bert-base-cased).
-It was introduced in [this paper](https://arxiv.org/abs/1910.01108).
-The code for the distillation process can be found
-[here](https://github.com/huggingface/transformers/tree/master/examples/distillation).
-This model is cased: it does make a difference between english and English.
-
-All the training details on the pre-training, the uses, limitations and potential biases are the same as for [DistilBERT-base-uncased](https://huggingface.co/distilbert-base-uncased).
-We highly encourage to check it if you want to know more.
-
-## Evaluation results
-
-When fine-tuned on downstream tasks, this model achieves the following results:
-
-Glue test results:
-
-| Task | MNLI | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  |
-|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|
-|      | 81.5 | 87.8 | 88.2 | 90.4  | 47.2 | 85.5  | 85.6 | 60.6 |
-
-### BibTeX entry and citation info
-
-```bibtex
-@article{Sanh2019DistilBERTAD,
-  title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter},
-  author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf},
-  journal={ArXiv},
-  year={2019},
-  volume={abs/1910.01108}
-}
-```
@@ -6,8 +6,3 @@ metrics:
 - squad
 license: apache-2.0
 ---
-
-# DistilBERT base cased distilled SQuAD
-
-This model is a fine-tune checkpoint of [DistilBERT-base-cased](https://huggingface.co/distilbert-base-cased), fine-tuned using (a second step of) knowledge distillation on SQuAD v1.1.
-This model reaches a F1 score of 87.1 on the dev set (for comparison, BERT bert-base-cased version reaches a F1 score of 88.7).
@@ -1,35 +1,4 @@
 ---
 language: multilingual
 license: apache-2.0
-datasets:
- wikipedia
 ---
-
-# DistilBERT base multilingual model (cased)
-
-This model is a distilled version of the [BERT base multilingual model](bert-base-multilingual-cased). The code for the distillation process can be found
-[here](https://github.com/huggingface/transformers/tree/master/examples/distillation). This model is cased: it does make a difference between english and English.
-
-The model is trained on the concatenation of Wikipedia in 104 different languages listed [here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages).
-The model has 6 layers, 768 dimension and 12 heads, totalizing 134M parameters (compared to 177M parameters for mBERT-base).
-On average DistilmBERT is twice as fast as mBERT-base.
-
-We encourage to check [BERT base multilingual model](bert-base-multilingual-cased) to know more about usage, limitations and potential biases.
-
-| Model                        | English | Spanish | Chinese | German | Arabic  | Urdu |
-| :---:                        | :---:   | :---:   | :---:   | :---:  | :---:   | :---:|
-| mBERT base cased (computed)  | 82.1    | 74.6    | 69.1    | 72.3   | 66.4    | 58.5 |
-| mBERT base uncased (reported)| 81.4    | 74.3    | 63.8    | 70.5   | 62.1    | 58.3 |
-| DistilmBERT                  | 78.2    | 69.1    | 64.0    | 66.3   | 59.1    | 54.7 |
-
-### BibTeX entry and citation info
-
-```bibtex
-@article{Sanh2019DistilBERTAD,
-  title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter},
-  author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf},
-  journal={ArXiv},
-  year={2019},
-  volume={abs/1910.01108}
-}
-```
@@ -10,7 +10,7 @@ datasets:

 # DistilBERT base model (uncased)

-This model is a distilled version of the [BERT base model](https://huggingface.co/bert-base-uncased). It was
+This model is a distilled version of the [BERT base mode](https://huggingface.co/distilbert-base-uncased). It was
 introduced in [this paper](https://arxiv.org/abs/1910.01108). The code for the distillation process can be found
 [here](https://github.com/huggingface/transformers/tree/master/examples/distillation). This model is uncased: it does
 not make a difference between english and English.
@@ -102,7 +102,7 @@ output = model(encoded_input)

 Even if the training data used for this model could be characterized as fairly neutral, this model can have biased
 predictions. It also inherits some of
-[the bias of its teacher model](https://huggingface.co/bert-base-uncased#limitations-and-bias).
+[the bias of its teacher model](https://huggingface.co/bert-base-uncased#limitations-and-bias). 

 ```python
 >>> from transformers import pipeline
@@ -196,9 +196,9 @@ When fine-tuned on downstream tasks, this model achieves the following results:

 Glue test results:

-| Task | MNLI | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  |
-|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|
-|      | 82.2 | 88.5 | 89.2 | 91.3  | 51.3 | 85.8  | 87.5 | 59.9 |
+| Task | MNLI | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  | Average |
+|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|:-------:|
+|      | 82.2 | 88.5 | 89.2 | 91.3  | 51.3 | 85.8  | 87.5 | 59.9 | 77.0    |


 ### BibTeX entry and citation info
@@ -1,5 +1,4 @@
 ---
-language: en
 datasets:
 - squad
 widget:
@@ -9,8 +8,3 @@ widget:
  context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
 license: apache-2.0
 ---
-
-# DistilBERT base uncased distilled SQuAD
-
-This model is a fine-tune checkpoint of [DistilBERT-base-uncased](https://huggingface.co/distilbert-base-uncased), fine-tuned using (a second step of) knowledge distillation on SQuAD v1.1.
-This model reaches a F1 score of 86.9 on the dev set (for comparison, Bert bert-base-uncased version reaches a F1 score of 88.5).
@@ -1,19 +1,3 @@
 ---
-language: en
 license: apache-2.0
-datasets:
- sst-2
 ---
-
-# DistilBERT base uncased finetuned SST-2
-
-This model is a fine-tune checkpoint of [DistilBERT-base-uncased](https://huggingface.co/distilbert-base-uncased), fine-tuned on SST-2.
-This model reaches an accuracy of 91.3 on the dev set (for comparison, Bert bert-base-uncased version reaches an accuracy of 92.7).
-
-# Fine-tuning hyper-parameters
-
- learning_rate = 1e-5
- batch_size = 32
- warmup = 600
- max_seq_length = 128
- num_train_epochs = 3.0
@@ -1,21 +1,10 @@
 ---
-language: en
 tags:
 - exbert

 license: apache-2.0
-datasets:
- openwebtext
 ---

-# DistilGPT2
-
-DistilGPT2 English language model pretrained with the supervision of [GPT2](https://huggingface.co/gpt2) (the smallest version of GPT2) on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset. The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 124M parameters for GPT2). On average, DistilGPT2 is two times faster than GPT2.
-
-On the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test set of 16.3 compared to 21.1 for DistilGPT2 (after fine-tuning on the train set).
-
-We encourage to check [GPT2](https://huggingface.co/gpt2) to know more about usage, limitations and potential biases.
-
 <a href="https://huggingface.co/exbert/?model=distilgpt2">
 	<img width="300px" src="https://cdn-media.huggingface.co/exbert/button.png">
 </a>
@@ -1,50 +1,10 @@
 ---
-language: en
 tags:
 - exbert

 license: apache-2.0
-datasets:
- openwebtext
 ---

-# DistilRoBERTa base model
-
-This model is a distilled version of the [RoBERTa-base model](https://huggingface.co/roberta-base). It follows the same training procedure as [DistilBERT](https://huggingface.co/distilbert-base-uncased).
-The code for the distillation process can be found [here](https://github.com/huggingface/transformers/tree/master/examples/distillation).
-This model is case-sensitive: it makes a difference between english and English.
-
-The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 125M parameters for RoBERTa-base).
-On average DistilRoBERTa is twice as fast as Roberta-base.
-
-We encourage to check [RoBERTa-base model](https://huggingface.co/roberta-base) to know more about usage, limitations and potential biases.
-
-## Training data
-
-DistilRoBERTa was pre-trained on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset (it is ~4 times less training data than the teacher RoBERTa).
-
-## Evaluation results
-
-When fine-tuned on downstream tasks, this model achieves the following results:
-
-Glue test results:
-
-| Task | MNLI | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  |
-|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|
-|      | 84.0 | 89.4 | 90.8 | 92.5  | 59.3 | 88.3  | 86.6 | 67.9 |
-
-### BibTeX entry and citation info
-
-```bibtex
-@article{Sanh2019DistilBERTAD,
-  title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter},
-  author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf},
-  journal={ArXiv},
-  year={2019},
-  volume={abs/1910.01108}
-}
-```
-
 <a href="https://huggingface.co/exbert/?model=distilroberta-base">
 	<img width="300px" src="https://cdn-media.huggingface.co/exbert/button.png">
 </a>
@@ -1,66 +0,0 @@
---
-language: en
-datasets:
- qasc
---
-
-# T5-base fine-tuned on QASC 
-
-[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [QASC](https://allenai.org/data/qasc) for **QA** (via *sentence composition*) downstream task.
-
-## Details of T5
-
-The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract:
-
-Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
-
-![model image](https://i.imgur.com/jVFMMWR.png)
-
-
-## Details of the dataset 📚 
-
-**Question Answering via Sentence Composition** (QASC) is a question-answering dataset with a focus on sentence composition. It consists of 9,980 8-way multiple-choice questions about grade school science (8,134 train, 926 dev, 920 test), and comes with a corpus of 17M sentences.
-
-
-## Model fine-tuning 🏋️‍
-
-The training script is a slightly modified version of [this  awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28). The **context** passed to the *encoder* is the combination of the 2 *facts* (`fact1` and `fact2`). The **question** is just the `formatted_question` field. The **answer** passed to the *decoder* is the`text` right answer instead of the `label` (A, B, C... See `choices` field). More details about the dataset format/fields [here](https://huggingface.co/nlp/viewer/?dataset=qasc)
-
-## Metrics on validation set 📋
-
-| Metric | Score |
-|--------|-------|
-|Accuracy (EM) | **97.73**|
-
-
-## Model in Action 🚀
-
-```python
-from transformers import AutoModelWithLMHead, AutoTokenizer
-
-tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-qasc")
-model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-qasc")
-
-def get_response(question, context, max_length=64):
-  input_text = 'question: %s  context: %s' % (question, context)
-  features = tokenizer([input_text], return_tensors='pt')
-
-  output = model.generate(input_ids=features['input_ids'], 
-               attention_mask=features['attention_mask'],
-               max_length=max_length)
-
-  return tokenizer.decode(output[0])
-  
-fact_1 = 'a watch is used for measuring time'
-fact_2 = 'Times are measured in seconds.'
-context = fact_1 + ' ' + fact_2
-question = 'What can be used to measure seconds? (A) Watch (B) seconds (C) fluid (D) Ruler (E) goggles (F) glasses (G) Drill (H) Scale'
-
-get_response(question, context)
-
-# output: 'Watch'
-```
-
-> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
-
-> Made with <span style="color: #e25555;">&hearts;</span> in Spain
@@ -1,16 +0,0 @@
---
-language: en
-license: apache-2.0
-datasets:
- cnn_dailymail
-tags:
- summarization
---
-
-Bert2Bert Summarization with 🤗EncoderDecoder Framework
-This model is a warm-started *BERT2BERT* model fine-tuned on the *CNN/Dailymail* summarization dataset.
-
-The model achieves a **18.22** ROUGE-2 score on *CNN/Dailymail*'s test dataset.
-
-For more details on how the model was fine-tuned, please refer to 
-[this](https://colab.research.google.com/drive/1Ekd5pUeCX7VOrMx94_czTkwNtLN32Uyu?usp=sharing) notebook.
@@ -1,16 +0,0 @@
---
-language: en
-license: apache-2.0
-datasets:
- xsum
-tags:
- summarization
---
-
-Shared RoBERTa2RoBERTa Summarization with 🤗EncoderDecoder Framework
-This model is a warm-started *RoBERTaShared* model fine-tuned on the *BBC XSum* summarization dataset.
-
-The model achieves a **16.89** ROUGE-2 score on *BBC XSUM*'s test dataset.
-
-For more details on how the model was fine-tuned, please refer to 
-[this](https://colab.research.google.com/drive/1Ekd5pUeCX7VOrMx94_czTkwNtLN32Uyu?usp=sharing) notebook.
@@ -26,7 +26,7 @@ Pull Request so it can be included under the Community notebooks.

 | Notebook     |      Description      |      Author      |      |
 |:----------|:-------------|:-------------|------:|
-| [Train T5 in Tensorflow 2 ](https://github.com/snapthat/TF-T5-text-to-text) | How to train T5 for any task using Tensorflow 2. This notebook demonstrates a Question & Answer task implemented in Tensorflow 2 using SQUAD | [Muhammad Harris](https://github.com/HarrisDePerceptron) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/snapthat/TF-T5-text-to-text/blob/master/snapthatT5/notebooks/TF-T5-Datasets%20Training.ipynb) |
+| [Train T5 in Tensoflow 2 ](https://github.com/snapthat/TF-T5-text-to-text) | How to train T5 for any task using Tensorflow 2. This notebook demonstrates a Question & Answer task implemented in Tensorflow 2 using SQUAD | [Muhammad Harris](https://github.com/HarrisDePerceptron) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/snapthat/TF-T5-text-to-text/blob/master/snapthatT5/notebooks/TF-T5-Datasets%20Training.ipynb) |
 | [Train T5 on TPU](https://github.com/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb)  | How to train T5 on SQUAD with Transformers and Nlp | [Suraj Patil](https://github.com/patil-suraj) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb#scrollTo=QLGiFCDqvuil) |
 | [Fine-tune T5 for Classification and Multiple Choice](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb)  | How to fine-tune T5 for classification and multiple choice tasks using a text-to-text format with PyTorch Lightning |  [Suraj Patil](https://github.com/patil-suraj) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) |
 | [Fine-tune DialoGPT on New Datasets and Languages](https://github.com/ncoop57/i-am-a-nerd/blob/master/_notebooks/2020-05-12-chatbot-part-1.ipynb)  | How to fine-tune the DialoGPT model on a new dataset for open-dialog conversational chatbots |  [Nathan Cooper](https://github.com/ncoop57) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ncoop57/i-am-a-nerd/blob/master/_notebooks/2020-05-12-chatbot-part-1.ipynb) |
@@ -51,5 +51,3 @@ Pull Request so it can be included under the Community notebooks.
 |[Fine-tune Roberta for sentiment analysis](https://github.com/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb) | How to fine-tune an Roberta model for sentiment analysis | [Dhaval Taunk](https://github.com/DhavalTaunk08) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb)|
 |[Evaluating Question Generation Models](https://github.com/flexudy-pipe/qugeev) | How accurate are the answers to questions generated by your seq2seq transformer model? | [Pascal Zoleko](https://github.com/zolekode) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1bpsSqCQU-iw_5nNoRm_crPq6FRuJthq_?usp=sharing)|
 |[Classify text with DistilBERT and Tensorflow](https://github.com/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb) | How to fine-tune DistilBERT for text classification in TensorFlow | [Peter Bayerle](https://github.com/peterbayerle) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb)|
-|[Leverage BERT for Encoder-Decoder Summarization on CNN/Dailymail](https://github.com/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb) | How to warm-start a *EncoderDecoderModel* with a *bert-base-uncased* checkpoint for summarization on CNN/Dailymail | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb)|
-|[Leverage RoBERTa for Encoder-Decoder Summarization on BBC XSum](https://github.com/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb) | How to warm-start a shared *EncoderDecoderModel* with a *roberta-base* checkpoint for summarization on BBC/XSum | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb)|
@@ -105,7 +105,7 @@ extras["sentencepiece"] = ["sentencepiece==0.1.91"]
 extras["retrieval"] = ["faiss-cpu", "datasets"]
 extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil"] + extras["retrieval"]
 # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
-extras["docs"] = ["recommonmark", "sphinx==3.2.1", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
+extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
 extras["quality"] = ["black >= 20.8b1", "isort >= 5.5.4", "flake8 >= 3.8.3"]


@@ -281,9 +281,9 @@ if is_torch_available():
    from .data.data_collator import (
        DataCollator,
        DataCollatorForLanguageModeling,
+        DataCollatorForNextSentencePrediction,
        DataCollatorForPermutationLanguageModeling,
        DataCollatorForSOP,
-        DataCollatorForTokenClassification,
        DataCollatorForWholeWordMask,
        DataCollatorWithPadding,
        default_data_collator,
@@ -299,19 +299,6 @@ if is_torch_available():
        TextDataset,
        TextDatasetForNextSentencePrediction,
    )
-    from .generation_beam_search import BeamScorer, BeamSearchScorer
-    from .generation_logits_process import (
-        LogitsProcessor,
-        LogitsProcessorList,
-        LogitsWarper,
-        MinLengthLogitsProcessor,
-        NoBadWordsLogitsProcessor,
-        NoRepeatNGramLogitsProcessor,
-        RepetitionPenaltyLogitsProcessor,
-        TemperatureLogitsWarper,
-        TopKLogitsWarper,
-        TopPLogitsWarper,
-    )
    from .generation_utils import top_k_top_p_filtering
    from .modeling_albert import (
        ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
@@ -68,7 +68,7 @@ class EncoderDecoderConfig(PretrainedConfig):
        >>> encoder_decoder_config = EncoderDecoderConfig.from_pretrained('my-model')
        >>> model = EncoderDecoderModel.from_pretrained('my-model', config=encoder_decoder_config)
    """
-    model_type = "encoder-decoder"
+    model_type = "encoder_decoder"
    is_composition = True

    def __init__(self, **kwargs):
@@ -198,17 +198,6 @@ class XLNetConfig(PretrainedConfig):
        self.pad_token_id = pad_token_id
        self.eos_token_id = eos_token_id

-        if mem_len is None or mem_len == 0:
-            warnings.warn(
-                "This config doesn't use attention memories, a core feature of XLNet."
-                " Consider setting `mem_len` to a non-zero value, for example "
-                "`xlnet = XLNetLMHeadModel.from_pretrained('xlnet-base-cased'', mem_len=1024)`,"
-                " for accurate training performance as well as an order of magnitude faster inference."
-                " Starting from version 3.5.0, the default parameter will be 1024, following"
-                " the implementation in https://arxiv.org/abs/1906.08237",
-                FutureWarning,
-            )
-
    @property
    def max_position_embeddings(self):
        return -1
@@ -1,5 +1,4 @@
 import random
-import warnings
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union

@@ -115,172 +114,72 @@ class DataCollatorWithPadding:
        return batch


-@dataclass
-class DataCollatorForTokenClassification:
-    """
-    Data collator that will dynamically pad the inputs received, as well as the labels.
-
-    Args:
-        tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
-            The tokenizer used for encoding the data.
-        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
-            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
-            among:
-
-            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
-              sequence if provided).
-            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
-              maximum acceptable input length for the model if that argument is not provided.
-            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
-              different lengths).
-        max_length (:obj:`int`, `optional`):
-            Maximum length of the returned list and optionally padding length (see above).
-        pad_to_multiple_of (:obj:`int`, `optional`):
-            If set will pad the sequence to a multiple of the provided value.
-
-            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
-            7.5 (Volta).
-        label_pad_token_id (:obj:`int`, `optional`, defaults to -100):
-            The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).
-    """
-
-    tokenizer: PreTrainedTokenizerBase
-    padding: Union[bool, str, PaddingStrategy] = True
-    max_length: Optional[int] = None
-    pad_to_multiple_of: Optional[int] = None
-    label_pad_token_id: int = -100
-
-    def __call__(self, features):
-        label_name = "label" if "label" in features[0].keys() else "labels"
-        labels = [feature[label_name] for feature in features] if label_name in features[0].keys() else None
-        batch = self.tokenizer.pad(
-            features,
-            padding=self.padding,
-            max_length=self.max_length,
-            pad_to_multiple_of=self.pad_to_multiple_of,
-            # Conversion to tensors will fail if we have labels as they are not of the same length yet.
-            return_tensors="pt" if labels is None else None,
-        )
-
-        if labels is None:
-            return batch
-
-        sequence_length = torch.tensor(batch["input_ids"]).shape[1]
-        padding_side = self.tokenizer.padding_side
-        if padding_side == "right":
-            batch["labels"] = [label + [self.label_pad_token_id] * (sequence_length - len(label)) for label in labels]
-        else:
-            batch["labels"] = [[self.label_pad_token_id] * (sequence_length - len(label)) + label for label in labels]
-
-        batch = {k: torch.tensor(v, dtype=torch.int64) for k, v in batch.items()}
-        return batch
-
-
-def _collate_batch(examples, tokenizer):
-    """Collate `examples` into a batch, using the information in `tokenizer` for padding if necessary."""
-    # Tensorize if necessary.
-    if isinstance(examples[0], (list, tuple)):
-        examples = [torch.tensor(e, dtype=torch.long) for e in examples]
-
-    # Check if padding is necessary.
-    length_of_first = examples[0].size(0)
-    are_tensors_same_length = all(x.size(0) == length_of_first for x in examples)
-    if are_tensors_same_length:
-        return torch.stack(examples, dim=0)
-
-    # If yes, check if we have a `pad_token`.
-    if tokenizer._pad_token is None:
-        raise ValueError(
-            "You are attempting to pad samples but the tokenizer you are using"
-            f" ({tokenizer.__class__.__name__}) does not have a pad token."
-        )
-
-    # Creating the full tensor and filling it with our data.
-    max_length = max(x.size(0) for x in examples)
-    result = examples[0].new_full([len(examples), max_length], tokenizer.pad_token_id)
-    for i, example in enumerate(examples):
-        if tokenizer.padding_side == "right":
-            result[i, : example.shape[0]] = example
-        else:
-            result[i, -example.shape[0] :] = example
-    return result
-
-
@dataclass
 class DataCollatorForLanguageModeling:
    """
-    Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
-    are not all of the same length.
+    Data collator used for language modeling.

-    Args:
-        tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
-            The tokenizer used for encoding the data.
-        mlm (:obj:`bool`, `optional`, defaults to :obj:`True`):
-            Whether or not to use masked language modeling. If set to :obj:`False`, the labels are the same as the
-            inputs with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for
-            non-masked tokens and the value to predict for the masked token.
-        mlm_probability (:obj:`float`, `optional`, defaults to 0.15):
-            The probability with which to (randomly) mask tokens in the input, when :obj:`mlm` is set to :obj:`True`.
-
-    .. note::
-
-        For best performance, this data collator should be used with a dataset having items that are dictionaries or
-        BatchEncoding, with the :obj:`"special_tokens_mask"` key, as returned by a
-        :class:`~transformers.PreTrainedTokenizer` or a :class:`~transformers.PreTrainedTokenizerFast` with the
-        argument :obj:`return_special_tokens_mask=True`.
+    - collates batches of tensors, honoring their tokenizer's pad_token
+    - preprocesses batches for masked language modeling
    """

    tokenizer: PreTrainedTokenizerBase
    mlm: bool = True
    mlm_probability: float = 0.15

-    def __post_init__(self):
-        if self.mlm and self.tokenizer.mask_token is None:
-            raise ValueError(
-                "This tokenizer does not have a mask token which is necessary for masked language modeling. "
-                "You should pass `mlm=False` to train on causal language modeling instead."
-            )
-
    def __call__(
        self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
    ) -> Dict[str, torch.Tensor]:
-        # Handle dict or lists with proper padding and conversion to tensor.
        if isinstance(examples[0], (dict, BatchEncoding)):
-            batch = self.tokenizer.pad(examples, return_tensors="pt")
-        else:
-            batch = {"input_ids": _collate_batch(examples, self.tokenizer)}
-
-        # If special token mask has been preprocessed, pop it from the dict.
-        special_tokens_mask = batch.pop("special_tokens_mask", None)
+            examples = [e["input_ids"] for e in examples]
+        batch = self._tensorize_batch(examples)
        if self.mlm:
-            batch["input_ids"], batch["labels"] = self.mask_tokens(
-                batch["input_ids"], special_tokens_mask=special_tokens_mask
-            )
+            inputs, labels = self.mask_tokens(batch)
+            return {"input_ids": inputs, "labels": labels}
        else:
-            labels = batch["input_ids"]
+            labels = batch.clone().detach()
            if self.tokenizer.pad_token_id is not None:
                labels[labels == self.tokenizer.pad_token_id] = -100
-            batch["labels"] = labels
-        return batch
+            return {"input_ids": batch, "labels": labels}

-    def mask_tokens(
-        self, inputs: torch.Tensor, special_tokens_mask: Optional[torch.Tensor] = None
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
+    def _tensorize_batch(
+        self, examples: List[Union[List[int], torch.Tensor, Dict[str, torch.Tensor]]]
+    ) -> torch.Tensor:
+        # In order to accept both lists of lists and lists of Tensors
+        if isinstance(examples[0], (list, tuple)):
+            examples = [torch.tensor(e, dtype=torch.long) for e in examples]
+        length_of_first = examples[0].size(0)
+        are_tensors_same_length = all(x.size(0) == length_of_first for x in examples)
+        if are_tensors_same_length:
+            return torch.stack(examples, dim=0)
+        else:
+            if self.tokenizer._pad_token is None:
+                raise ValueError(
+                    "You are attempting to pad samples but the tokenizer you are using"
+                    f" ({self.tokenizer.__class__.__name__}) does not have one."
+                )
+            return pad_sequence(examples, batch_first=True, padding_value=self.tokenizer.pad_token_id)
+
+    def mask_tokens(self, inputs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
        """
        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
        """
-        labels = inputs.clone()
-        # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`)
-        probability_matrix = torch.full(labels.shape, self.mlm_probability)
-        if special_tokens_mask is None:
-            special_tokens_mask = [
-                self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()
-            ]
-            special_tokens_mask = torch.tensor(special_tokens_mask, dtype=torch.bool)
-        else:
-            special_tokens_mask = special_tokens_mask.bool()

-        probability_matrix.masked_fill_(special_tokens_mask, value=0.0)
+        if self.tokenizer.mask_token is None:
+            raise ValueError(
+                "This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the --mlm flag if you want to use this tokenizer."
+            )
+
+        labels = inputs.clone()
+        # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa)
+        probability_matrix = torch.full(labels.shape, self.mlm_probability)
+        special_tokens_mask = [
+            self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()
+        ]
+        probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0)
+        if self.tokenizer._pad_token is not None:
+            padding_mask = labels.eq(self.tokenizer.pad_token_id)
+            probability_matrix.masked_fill_(padding_mask, value=0.0)
        masked_indices = torch.bernoulli(probability_matrix).bool()
        labels[~masked_indices] = -100  # We only compute loss on masked tokens

@@ -425,16 +324,9 @@ class DataCollatorForSOP(DataCollatorForLanguageModeling):
    - preprocesses batches for both masked language modeling and sentence order prediction
    """

-    def __init__(self, *args, **kwargs):
-        warnings.warn(
-            "DataCollatorForSOP is deprecated and will be removed in a future version, you can now use "
-            "DataCollatorForLanguageModeling instead.",
-            FutureWarning,
-        )
-
    def __call__(self, examples: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]:
        input_ids = [example["input_ids"] for example in examples]
-        input_ids = _collate_batch(input_ids, self.tokenizer)
+        input_ids = self._tensorize_batch(input_ids)
        input_ids, labels, attention_mask = self.mask_tokens(input_ids)

        token_type_ids = [example["token_type_ids"] for example in examples]
@@ -629,3 +521,136 @@ class DataCollatorForPermutationLanguageModeling:
            ) & masked_indices[i]

        return inputs.long(), perm_mask, target_mapping, labels.long()
+
+
+@dataclass
+class DataCollatorForNextSentencePrediction:
+    """
+    Data collator used for next sentence prediction. - collates examples which contains pre-generated negative examples
+    - preprocesses batches for masked language modeling
+    """
+
+    tokenizer: PreTrainedTokenizerBase
+    mlm: bool = True
+    block_size: int = 512
+    short_seq_probability: float = 0.1
+    nsp_probability: float = 0.5
+    mlm_probability: float = 0.15
+
+    def __call__(self, examples: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]:
+        """
+        The input should contain negative examples, :class:`~transformers.DataCollatorForNextSentencePrediction` will
+        not generate any negative examples
+
+        Args:
+            examples (:obj:`List[Dict]`): Each dictionary should have the following keys:
+
+                  - ``tokens_a``: A sequence of tokens, which should appear before ``tokens_b`` in the text.
+                  - ``tokens_b``: A sequence of tokens, which should appear after ``tokens_a`` in the text.
+                  - ``is_random_next``: 1 if this pair is generated randomly, else 0.
+        """
+
+        tokens_a = [e["tokens_a"] for e in examples]
+        tokens_b = [e["tokens_b"] for e in examples]
+        nsp_labels = [1 if e["is_random_next"] else 0 for e in examples]
+
+        input_ids = []
+        segment_ids = []
+        attention_masks = []
+
+        assert len(tokens_a) == len(tokens_b)
+        for i in range(len(tokens_a)):
+            input_id, attention_mask, segment_id = self.create_features_from_example(tokens_a[i], tokens_b[i])
+            input_ids.append(input_id)
+            segment_ids.append(segment_id)
+            attention_masks.append(attention_mask)
+        if self.mlm:
+            input_ids, mlm_labels = self.mask_tokens(self._tensorize_batch(input_ids))
+        else:
+            input_ids = self._tensorize_batch(input_ids)
+
+        result = {
+            "input_ids": input_ids,
+            "attention_mask": self._tensorize_batch(attention_masks),
+            "token_type_ids": self._tensorize_batch(segment_ids),
+            "labels": mlm_labels if self.mlm else None,
+            "next_sentence_label": torch.tensor(nsp_labels),
+        }
+        return result
+
+    def _tensorize_batch(self, examples: List[torch.Tensor]) -> torch.Tensor:
+        length_of_first = examples[0].size(0)
+        are_tensors_same_length = all(x.size(0) == length_of_first for x in examples)
+        if are_tensors_same_length:
+            return torch.stack(examples, dim=0)
+        else:
+            if self.tokenizer._pad_token is None:
+                raise ValueError(
+                    "You are attempting to pad samples but the tokenizer you are using"
+                    f" ({self.tokenizer.__class__.__name__}) does not have one."
+                )
+            return pad_sequence(examples, batch_first=True, padding_value=self.tokenizer.pad_token_id)
+
+    def create_features_from_example(self, tokens_a, tokens_b):
+        """Creates examples for a single document."""
+
+        max_num_tokens = self.block_size - self.tokenizer.num_special_tokens_to_add(pair=True)
+
+        tokens_a, tokens_b, _ = self.tokenizer.truncate_sequences(
+            tokens_a,
+            tokens_b,
+            num_tokens_to_remove=len(tokens_a) + len(tokens_b) - max_num_tokens,
+            truncation_strategy="longest_first",
+        )
+
+        input_id = self.tokenizer.build_inputs_with_special_tokens(tokens_a, tokens_b)
+        attention_mask = [1] * len(input_id)
+        segment_id = self.tokenizer.create_token_type_ids_from_sequences(tokens_a, tokens_b)
+        assert len(input_id) <= self.block_size
+
+        # pad
+        while len(input_id) < self.block_size:
+            input_id.append(0)
+            attention_mask.append(0)
+            segment_id.append(0)
+
+        input_id = torch.tensor(input_id)
+        attention_mask = torch.tensor(attention_mask)
+        segment_id = torch.tensor(segment_id)
+
+        return input_id, attention_mask, segment_id
+
+    def mask_tokens(self, inputs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        """
+        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
+        """
+
+        if self.tokenizer.mask_token is None:
+            raise ValueError(
+                "This tokenizer does not have a mask token which is necessary for masked language modeling. Remove the --mlm flag if you want to use this tokenizer."
+            )
+
+        labels = inputs.clone()
+        # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa)
+        probability_matrix = torch.full(labels.shape, self.mlm_probability)
+        special_tokens_mask = [
+            self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()
+        ]
+        probability_matrix.masked_fill_(torch.tensor(special_tokens_mask, dtype=torch.bool), value=0.0)
+        if self.tokenizer._pad_token is not None:
+            padding_mask = labels.eq(self.tokenizer.pad_token_id)
+            probability_matrix.masked_fill_(padding_mask, value=0.0)
+        masked_indices = torch.bernoulli(probability_matrix).bool()
+        labels[~masked_indices] = -100  # We only compute loss on masked tokens
+
+        # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
+        indices_replaced = torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices
+        inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)
+
+        # 10% of the time, we replace masked input tokens with random word
+        indices_random = torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices & ~indices_replaced
+        random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)
+        inputs[indices_random] = random_words[indices_random]
+
+        # The rest of the time (10% of the time) we keep the masked input tokens unchanged
+        return inputs, labels
@@ -3,7 +3,6 @@ import os
 import pickle
 import random
 import time
-import warnings
 from typing import Dict, List, Optional

 import torch
@@ -18,11 +17,6 @@ from ...utils import logging
 logger = logging.get_logger(__name__)


-DEPRECATION_WARNING = (
-    "This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library."
-)
-
-
 class TextDataset(Dataset):
    """
    This will be superseded by a framework-agnostic approach soon.
@@ -36,7 +30,6 @@ class TextDataset(Dataset):
        overwrite_cache=False,
        cache_dir: Optional[str] = None,
    ):
-        warnings.warn(DEPRECATION_WARNING, FutureWarning)
        assert os.path.isfile(file_path), f"Input file path {file_path} not found"

        block_size = block_size - tokenizer.num_special_tokens_to_add(pair=False)
@@ -101,7 +94,6 @@ class LineByLineTextDataset(Dataset):
    """

    def __init__(self, tokenizer: PreTrainedTokenizer, file_path: str, block_size: int):
-        warnings.warn(DEPRECATION_WARNING, FutureWarning)
        assert os.path.isfile(file_path), f"Input file path {file_path} not found"
        # Here, we do not cache the features, operating under the assumption
        # that we will soon use fast multithreaded tokenizers from the
@@ -128,7 +120,6 @@ class LineByLineWithRefDataset(Dataset):
    """

    def __init__(self, tokenizer: PreTrainedTokenizer, file_path: str, block_size: int, ref_path: str):
-        warnings.warn(DEPRECATION_WARNING, FutureWarning)
        assert os.path.isfile(file_path), f"Input file path {file_path} not found"
        assert os.path.isfile(ref_path), f"Ref file path {file_path} not found"
        # Here, we do not cache the features, operating under the assumption
@@ -165,7 +156,6 @@ class LineByLineWithSOPTextDataset(Dataset):
    """

    def __init__(self, tokenizer: PreTrainedTokenizer, file_dir: str, block_size: int):
-        warnings.warn(DEPRECATION_WARNING, FutureWarning)
        assert os.path.isdir(file_dir)
        logger.info(f"Creating features from dataset file folder at {file_dir}")
        self.examples = []
@@ -315,7 +305,6 @@ class TextDatasetForNextSentencePrediction(Dataset):
        short_seq_probability=0.1,
        nsp_probability=0.5,
    ):
-        warnings.warn(DEPRECATION_WARNING, FutureWarning)
        assert os.path.isfile(file_path), f"Input file path {file_path} not found"

        self.block_size = block_size - tokenizer.num_special_tokens_to_add(pair=True)
@@ -460,18 +449,9 @@ class TextDatasetForNextSentencePrediction(Dataset):
                    assert len(tokens_a) >= 1
                    assert len(tokens_b) >= 1

-                    # add special tokens
-                    input_ids = self.tokenizer.build_inputs_with_special_tokens(tokens_a, tokens_b)
-                    # add token type ids, 0 for sentence a, 1 for sentence b
-                    token_type_ids = self.tokenizer.create_token_type_ids_from_sequences(tokens_a, tokens_b)
-
-                    example = {
-                        "input_ids": torch.tensor(input_ids, dtype=torch.long),
-                        "token_type_ids": torch.tensor(token_type_ids, dtype=torch.long),
-                        "next_sentence_label": torch.tensor(1 if is_random_next else 0, dtype=torch.long),
-                    }
-
-                    self.examples.append(example)
+                    self.examples.append(
+                        {"tokens_a": tokens_a, "tokens_b": tokens_b, "is_random_next": is_random_next}
+                    )

                current_chunk = []
                current_length = 0
@@ -1,357 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The HuggingFace Inc. team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from abc import ABC, abstractmethod
-from collections import UserDict
-from typing import Optional, Tuple
-
-import torch
-
-from .file_utils import add_start_docstrings
-
-
-PROCESS_INPUTS_DOCSTRING = r"""
-    Args:
-        input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size * num_beams, sequence_length)`):
-            Indices of input sequence tokens in the vocabulary.
-
-            Indices can be obtained using any class inheriting from :class:`~transformers.PretrainedTokenizer`. See
-            :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for
-            details.
-
-            `What are input IDs? <../glossary.html#input-ids>`__
-        next_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, 2 * num_beams)`):
-            Current scores of the top :obj:`2 * num_beams` non-finished beam hypotheses.
-        next_tokens (:obj:`torch.LongTensor` of shape :obj:`(batch_size, 2 * num_beams)`):
-            :obj:`input_ids` of the tokens corresponding to the top :obj:`2 * num_beams` non-finished beam hypotheses.
-        next_indices (:obj:`torch.LongTensor` of shape :obj:`(batch_size, 2 * num_beams)`):
-            Beam indices indicating to which beam hypothesis the :obj:`next_tokens` correspond.
-        pad_token_id (:obj:`int`, `optional`):
-            The id of the `padding` token.
-        eos_token_id (:obj:`int`, `optional`):
-            The id of the `end-of-sequence` token.
-
-    Return:
-        :obj:`UserDict`: A dictionary composed of the fields as defined above:
-
-            - **next_beam_scores** (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`) -- Updated
-              scores of all non-finished beams.
-            - **next_beam_tokens** (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`) -- Next tokens
-              to be added to the non-finished beam_hypotheses.
-            - **next_beam_indices** (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`) -- Beam indices
-              indicating to which beam the next tokens shall be added.
-
-"""
-
-FINALIZE_INPUTS_DOCSTRING = r"""
-    Args:
-        input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size * num_beams, sequence_length)`):
-            Indices of input sequence tokens in the vocabulary.
-
-            Indices can be obtained using any class inheriting from :class:`~transformers.PretrainedTokenizer`. See
-            :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for
-            details.
-
-            `What are input IDs? <../glossary.html#input-ids>`__
-        final_beam_scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`):
-            The final scores of all non-finished beams.
-        final_beam_tokens (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`):
-            The last tokens to be added to the non-finished beam_hypotheses.
-        final_beam_indices (:obj:`torch.FloatTensor` of shape :obj:`(batch_size * num_beams)`):
-            The beam indices indicating to which beam the :obj:`final_beam_tokens` shall be added.
-        pad_token_id (:obj:`int`, `optional`):
-            The id of the `padding` token.
-        eos_token_id (:obj:`int`, `optional`):
-            The id of the `end-of-sequence` token.
-
-    Return:
-        :obj:`torch.LongTensor` of shape :obj:`(batch_size * num_return_sequences, sequence_length)`: The generated
-        sequences. The second dimension (sequence_length) is either equal to :obj:`max_length` or shorter if all
-        batches finished early due to the :obj:`eos_token_id`.
-
-"""
-
-
-class BeamScorer(ABC):
-    """
-    Abstract base class for all beam scorers that are used for :meth:`~transformers.PretrainedModel.beam_search` and
-    :meth:`~transformers.PretrainedModel.beam_sample`.
-    """
-
-    @abstractmethod
-    @add_start_docstrings(PROCESS_INPUTS_DOCSTRING)
-    def process(
-        self,
-        input_ids: torch.LongTensor,
-        next_scores: torch.FloatTensor,
-        next_tokens: torch.LongTensor,
-        next_indices: torch.LongTensor,
-        **kwargs
-    ) -> Tuple[torch.Tensor]:
-        raise NotImplementedError("This is an abstract method.")
-
-    @abstractmethod
-    @add_start_docstrings(FINALIZE_INPUTS_DOCSTRING)
-    def finalize(
-        self,
-        input_ids: torch.LongTensor,
-        next_scores: torch.FloatTensor,
-        next_tokens: torch.LongTensor,
-        next_indices: torch.LongTensor,
-        **kwargs
-    ) -> torch.LongTensor:
-        raise NotImplementedError("This is an abstract method.")
-
-
-class BeamSearchScorer(BeamScorer):
-    r"""
-    :class:`transformers.BeamScorer` implementing standard beam search decoding.
-
-    Adapted in part from `Facebook's XLM beam search code
-    <https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529>`__.
-
-    Args:
-        batch_size (:obj:`int`):
-            Batch Size of :obj:`input_ids` for which beam search decoding is run in parallel.
-        max_length (:obj:`int`):
-            The maximum length of the sequence to be generated.
-        num_beams (:obj:`int`):
-            Number of beams for beam search.
-        device (:obj:`torch.device`):
-            Defines the device type (*e.g.*, :obj:`"cpu"` or :obj:`"cuda"`) on which this instance of
-            :obj:`BeamSearchScorer` will be allocated.
-        length_penalty (:obj:`float`, `optional`, defaults to 1.0):
-            Exponential penalty to the length. 1.0 means no penalty. Set to values < 1.0 in order to encourage the
-            model to generate shorter sequences, to a value > 1.0 in order to encourage the model to produce longer
-            sequences.
-        do_early_stopping (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether to stop the beam search when at least ``num_beams`` sentences are finished per batch or not.
-        num_beam_hyps_to_keep (:obj:`int`, `optional`, defaults to 1):
-            The number of beam hypotheses that shall be returned upon calling
-            :meth:`~transformer.BeamSearchScorer.finalize`.
-    """
-
-    def __init__(
-        self,
-        batch_size: int,
-        max_length: int,
-        num_beams: int,
-        device: torch.device,
-        length_penalty: Optional[float] = 1.0,
-        do_early_stopping: Optional[bool] = False,
-        num_beam_hyps_to_keep: Optional[int] = 1,
-    ):
-        self.max_length = max_length
-        self.num_beams = num_beams
-        self.device = device
-        self.length_penalty = length_penalty
-        self.do_early_stopping = do_early_stopping
-        self.num_beam_hyps_to_keep = num_beam_hyps_to_keep
-
-        self._is_init = False
-        self._beam_hyps = [
-            BeamHypotheses(
-                num_beams=self.num_beams,
-                max_length=self.max_length,
-                length_penalty=self.length_penalty,
-                early_stopping=self.do_early_stopping,
-            )
-            for _ in range(batch_size)
-        ]
-        self._done = torch.tensor([False for _ in range(batch_size)], dtype=torch.bool, device=self.device)
-
-        if not isinstance(num_beams, int) or num_beams <= 1:
-            raise ValueError(
-                f"`num_beams` has to be an integer strictly greater than 1, but is {num_beams}. For `num_beams` == 1, one should make use of `greedy_search` instead."
-            )
-
-    @property
-    def is_done(self) -> bool:
-        return self._done.all()
-
-    def process(
-        self,
-        input_ids: torch.LongTensor,
-        next_scores: torch.FloatTensor,
-        next_tokens: torch.LongTensor,
-        next_indices: torch.LongTensor,
-        pad_token_id: Optional[int] = None,
-        eos_token_id: Optional[int] = None,
-    ) -> Tuple[torch.Tensor]:
-        cur_len = input_ids.shape[-1]
-        batch_size = len(self._beam_hyps)
-        assert batch_size == (input_ids.shape[0] // self.num_beams)
-
-        device = input_ids.device
-        next_beam_scores = torch.zeros((batch_size, self.num_beams), dtype=next_scores.dtype, device=device)
-        next_beam_tokens = torch.zeros((batch_size, self.num_beams), dtype=next_tokens.dtype, device=device)
-        next_beam_indices = torch.zeros((batch_size, self.num_beams), dtype=next_indices.dtype, device=device)
-
-        for batch_idx, beam_hyp in enumerate(self._beam_hyps):
-            if self._done[batch_idx]:
-                assert (
-                    len(beam_hyp) >= self.num_beams
-                ), "Batch can only be done if at least {} beams have been generated".format(self.num_beams)
-                assert (
-                    eos_token_id is not None and pad_token_id is not None
-                ), "generated beams >= num_beams -> eos_token_id and pad_token have to be defined"
-                # pad the batch
-                next_beam_scores[batch_idx, :] = 0
-                next_beam_tokens[batch_idx, :] = pad_token_id
-                next_beam_indices[batch_idx, :] = 0
-                continue
-
-            # next tokens for this sentence
-            beam_idx = 0
-            for beam_token_rank, (next_token, next_score, next_index) in enumerate(
-                zip(next_tokens[batch_idx], next_scores[batch_idx], next_indices[batch_idx])
-            ):
-                batch_beam_idx = batch_idx * self.num_beams + next_index
-                # add to generated hypotheses if end of sentence
-                if (eos_token_id is not None) and (next_token.item() == eos_token_id):
-                    # if beam_token does not belong to top num_beams tokens, it should not be added
-                    is_beam_token_worse_than_top_num_beams = beam_token_rank >= self.num_beams
-                    if is_beam_token_worse_than_top_num_beams:
-                        continue
-                    beam_hyp.add(
-                        input_ids[batch_beam_idx].clone(),
-                        next_score.item(),
-                    )
-                else:
-                    # add next predicted token since it is not eos_token
-                    next_beam_scores[batch_idx, beam_idx] = next_score
-                    next_beam_tokens[batch_idx, beam_idx] = next_token
-                    next_beam_indices[batch_idx, beam_idx] = batch_beam_idx
-                    beam_idx += 1
-
-                # once the beam for next step is full, don't add more tokens to it.
-                if beam_idx == self.num_beams:
-                    break
-
-            if beam_idx < self.num_beams:
-                raise ValueError(
-                    f"At most {self.num_beams} tokens in {next_tokens[batch_idx]} can be equal to `eos_token_id: {eos_token_id}`. Make sure {next_tokens[batch_idx]} are corrected."
-                )
-
-            # Check if we are done so that we can save a pad step if all(done)
-            self._done[batch_idx] = self._done[batch_idx] or beam_hyp.is_done(
-                next_scores[batch_idx].max().item(), cur_len
-            )
-
-        return UserDict(
-            {
-                "next_beam_scores": next_beam_scores.view(-1),
-                "next_beam_tokens": next_beam_tokens.view(-1),
-                "next_beam_indices": next_beam_indices.view(-1),
-            }
-        )
-
-    def finalize(
-        self,
-        input_ids: torch.LongTensor,
-        final_beam_scores: torch.FloatTensor,
-        final_beam_tokens: torch.LongTensor,
-        final_beam_indices: torch.LongTensor,
-        pad_token_id: Optional[int] = None,
-        eos_token_id: Optional[int] = None,
-    ) -> torch.LongTensor:
-        batch_size = len(self._beam_hyps)
-
-        # finalize all open beam hypotheses and add to generated hypotheses
-        for batch_idx, beam_hyp in enumerate(self._beam_hyps):
-            if self._done[batch_idx]:
-                continue
-
-            # need to add best num_beams hypotheses to generated hyps
-            for beam_id in range(self.num_beams):
-                batch_beam_idx = batch_idx * self.num_beams + beam_id
-                final_score = final_beam_scores[batch_beam_idx].item()
-                final_tokens = input_ids[batch_beam_idx]
-                beam_hyp.add(final_tokens, final_score)
-
-        # select the best hypotheses
-        sent_lengths = input_ids.new(batch_size * self.num_beam_hyps_to_keep)
-        best = []
-
-        # retrieve best hypotheses
-        for i, beam_hyp in enumerate(self._beam_hyps):
-            sorted_hyps = sorted(beam_hyp.beams, key=lambda x: x[0])
-            for j in range(self.num_beam_hyps_to_keep):
-                best_hyp = sorted_hyps.pop()[1]
-                sent_lengths[self.num_beam_hyps_to_keep * i + j] = len(best_hyp)
-                best.append(best_hyp)
-
-        # prepare for adding eos
-        sent_max_len = min(sent_lengths.max().item() + 1, self.max_length)
-        decoded: torch.LongTensor = input_ids.new(batch_size * self.num_beam_hyps_to_keep, sent_max_len)
-        # shorter batches are padded if needed
-        if sent_lengths.min().item() != sent_lengths.max().item():
-            assert pad_token_id is not None, "`pad_token_id` has to be defined"
-            decoded.fill_(pad_token_id)
-
-        # fill with hypotheses and eos_token_id if the latter fits in
-        for i, hypo in enumerate(best):
-            decoded[i, : sent_lengths[i]] = hypo
-            if sent_lengths[i] < self.max_length:
-                decoded[i, sent_lengths[i]] = eos_token_id
-        return decoded
-
-
-class BeamHypotheses:
-    def __init__(self, num_beams: int, max_length: int, length_penalty: float, early_stopping: bool):
-        """
-        Initialize n-best list of hypotheses.
-        """
-        self.max_length = max_length - 1  # ignoring bos_token
-        self.length_penalty = length_penalty
-        self.early_stopping = early_stopping
-        self.num_beams = num_beams
-        self.beams = []
-        self.worst_score = 1e9
-
-    def __len__(self):
-        """
-        Number of hypotheses in the list.
-        """
-        return len(self.beams)
-
-    def add(self, hyp: torch.LongTensor, sum_logprobs: float):
-        """
-        Add a new hypothesis to the list.
-        """
-        score = sum_logprobs / (hyp.shape[-1] ** self.length_penalty)
-        if len(self) < self.num_beams or score > self.worst_score:
-            self.beams.append((score, hyp))
-            if len(self) > self.num_beams:
-                sorted_next_scores = sorted([(s, idx) for idx, (s, _) in enumerate(self.beams)])
-                del self.beams[sorted_next_scores[0][1]]
-                self.worst_score = sorted_next_scores[1][0]
-            else:
-                self.worst_score = min(score, self.worst_score)
-
-    def is_done(self, best_sum_logprobs: float, cur_len: int) -> bool:
-        """
-        If there are enough hypotheses and that none of the hypotheses being generated can become better than the worst
-        one in the heap, then we are done with this sentence.
-        """
-
-        if len(self) < self.num_beams:
-            return False
-        elif self.early_stopping:
-            return True
-        else:
-            cur_score = best_sum_logprobs / cur_len ** self.length_penalty
-            ret = self.worst_score >= cur_score
-            return ret
@@ -1,374 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The HuggingFace Inc. team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from abc import ABC
-from typing import Iterable, List
-
-import numpy as np
-import torch
-from torch.nn import functional as F
-
-from .file_utils import add_start_docstrings
-
-
-LOGITS_PROCESSOR_INPUTS_DOCSTRING = r"""
-    Args:
-        input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`):
-            Indices of input sequence tokens in the vocabulary.
-
-            Indices can be obtained using :class:`~transformers.BertTokenizer`. See
-            :meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for
-            details.
-
-            `What are input IDs? <../glossary.html#input-ids>`__
-        scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.vocab_size)`):
-            Prediction scores of a language modeling head. These can be scores for each vocabulary token before SoftMax
-            or scores for each vocabulary token after SoftMax.
-
-    Return:
-        :obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.vocab_size)`: The processed prediction scores.
-
-"""
-
-
-class LogitsProcessor(ABC):
-    """Abstract base class for all logit processors that can be applied during generation."""
-
-    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
-        """Torch method for processing logits."""
-        raise NotImplementedError(
-            f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."
-        )
-
-
-class LogitsWarper(ABC):
-    """Abstract base class for all logit warpers that can be applied during generation with multinomial sampling."""
-
-    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
-        """Torch method for warping logits."""
-        raise NotImplementedError(
-            f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."
-        )
-
-
-class LogitsProcessorList(list):
-    """
-    This class can be used to create a list of :class:`~transformers.LogitsProcessor` or
-    :class:`~transformers.LogitsWarper` to subsequently process a :obj:`scores` input tensor. This class inherits from
-    list and adds a specific `__call__` method to apply each :class:`~transformers.LogitsProcessor` or
-    :class:`~transformers.LogitsProcessor` to the inputs.
-    """
-
-    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
-        for processor in self:
-            scores = processor(input_ids, scores)
-        return scores
-
-
-class MinLengthLogitsProcessor(LogitsProcessor):
-    r"""
-    :class:`transformers.LogitsProcessor` enforcing a min-length by setting EOS probability to 0.
-
-    Args:
-        min_length (:obj:`int`):
-            The minimum length below which the score of :obj:`eos_token_id` is set to :obj:`-float("Inf")`.
-        eos_token_id (:obj:`int`):
-            The id of the `end-of-sequence` token.
-    """
-
-    def __init__(self, min_length: int, eos_token_id: int):
-        if not isinstance(min_length, int) or min_length < 0:
-            raise ValueError(f"`min_length` has to be a positive integer, but is {min_length}")
-
-        if not isinstance(eos_token_id, int) or eos_token_id < 0:
-            raise ValueError(f"`eos_token_id` has to be a positive integer, but is {eos_token_id}")
-
-        self.min_length = min_length
-        self.eos_token_id = eos_token_id
-
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
-        cur_len = input_ids.shape[-1]
-        if cur_len < self.min_length:
-            scores[:, self.eos_token_id] = -float("inf")
-        return scores
-
-
-class TemperatureLogitsWarper(LogitsWarper):
-    r"""
-    :class:`transformers.LogitsWarper` for temperature (exponential scaling output probability distribution).
-
-    Args:
-        temperature (:obj:`float`):
-            The value used to module the logits distribution.
-    """
-
-    def __init__(self, temperature: float):
-        if not isinstance(temperature, float) or not (temperature > 0):
-            raise ValueError(f"`temperature` has to be a strictly positive float, but is {temperature}")
-
-        self.temperature = temperature
-
-    def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor:
-        scores = scores / self.temperature
-        return scores
-
-
-class RepetitionPenaltyLogitsProcessor(LogitsProcessor):
-    r"""
-    :class:`transformers.LogitsProcessor` enforcing an exponential penalty on repeated sequences.
-
-    Args:
-        repetition_penalty (:obj:`float`):
-            The parameter for repetition penalty. 1.0 means no penalty. See `this paper
-            <https://arxiv.org/pdf/1909.05858.pdf>`__ for more details.
-    """
-
-    def __init__(self, penalty: float):
-        if not isinstance(penalty, float) or not (penalty > 0):
-            raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}")
-
-        self.penalty = penalty
-
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
-        for i in range(scores.shape[0]):
-            for previous_token in set(input_ids[i].tolist()):
-                # if score < 0 then repetition penalty has to be multiplied to reduce the previous token probability
-                if scores[i, previous_token] < 0:
-                    scores[i, previous_token] *= self.penalty
-                else:
-                    scores[i, previous_token] /= self.penalty
-        return scores
-
-
-class TopPLogitsWarper(LogitsWarper):
-    """
-    :class:`transformers.LogitsWarper` that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <=
-    prob_cut_off.
-
-    Args:
-        top_p (:obj:`float`):
-            If set to < 1, only the most probable tokens with probabilities that add up to :obj:`top_p` or higher are
-            kept for generation.
-        filter_value (:obj:`float`, `optional`, defaults to :obj:`-float("Inf")`):
-            All filtered values will be set to this float value.
-        min_tokens_to_keep (:obj:`int`, `optional`, defaults to 1):
-            Minimum number of tokens that cannot be filtered.
-    """
-
-    def __init__(self, top_p: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):
-        if not isinstance(top_p, float) or (top_p < 0 or top_p > 1.0):
-            raise ValueError(f"`top_p` has to be a float > 0 and < 1, but is {top_p}")
-
-        self.top_p = top_p
-        self.filter_value = filter_value
-        self.min_tokens_to_keep = min_tokens_to_keep
-
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
-        sorted_logits, sorted_indices = torch.sort(scores, descending=True)
-        cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
-
-        # Remove tokens with cumulative top_p above the threshold (token with 0 are kept)
-        sorted_indices_to_remove = cumulative_probs > self.top_p
-        if self.min_tokens_to_keep > 1:
-            # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below)
-            sorted_indices_to_remove[..., : self.min_tokens_to_keep - 1] = 0
-        # Shift the indices to the right to keep also the first token above the threshold
-        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
-        sorted_indices_to_remove[..., 0] = 0
-
-        # scatter sorted tensors to original indexing
-        indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
-        scores[indices_to_remove] = self.filter_value
-        return scores
-
-
-class TopKLogitsWarper(LogitsWarper):
-    r"""
-    :class:`transformers.LogitsWarper` that performs top-k, i.e. restricting to the k highest probability elements.
-
-    Args:
-        top_k (:obj:`int`):
-            The number of highest probability vocabulary tokens to keep for top-k-filtering.
-        filter_value (:obj:`float`, `optional`, defaults to :obj:`-float("Inf")`):
-            All filtered values will be set to this float value.
-        min_tokens_to_keep (:obj:`int`, `optional`, defaults to 1):
-            Minimum number of tokens that cannot be filtered.
-    """
-
-    def __init__(self, top_k: int, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):
-        if not isinstance(top_k, int) or top_k <= 0:
-            raise ValueError(f"`top_k` has to be a strictly positive integer, but is {top_k}")
-
-        self.top_k = top_k
-        self.filter_value = filter_value
-        self.min_tokens_to_keep = min_tokens_to_keep
-
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
-        top_k = min(max(self.top_k, self.min_tokens_to_keep), scores.size(-1))  # Safety check
-        # Remove all tokens with a probability less than the last token of the top-k
-        indices_to_remove = scores < torch.topk(scores, top_k)[0][..., -1, None]
-        scores[indices_to_remove] = self.filter_value
-        return scores
-
-
-class NoRepeatNGramLogitsProcessor(LogitsProcessor):
-    r"""
-    :class:`transformers.LogitsProcessor` that enforces no repetition of n-grams. See `Fairseq
-    <https://github.com/pytorch/fairseq/blob/a07cb6f40480928c9e0548b737aadd36ee66ac76/fairseq/sequence_generator.py#L345>`__.
-
-    Args:
-        ngram_size (:obj:`int`):
-            All ngrams of size :obj:`ngram_size` can only occur once.
-    """
-
-    def __init__(self, ngram_size: int):
-        if not isinstance(ngram_size, int) or ngram_size <= 0:
-            raise ValueError(f"`ngram_size` has to be a strictly positive integer, but is {ngram_size}")
-        self.ngram_size = ngram_size
-
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
-        num_batch_hypotheses = scores.shape[0]
-        cur_len = input_ids.shape[-1]
-        banned_batch_tokens = self._calc_banned_ngram_tokens(input_ids, num_batch_hypotheses, cur_len)
-
-        for i, banned_tokens in enumerate(banned_batch_tokens):
-            scores[i, banned_tokens] = -float("inf")
-
-        return scores
-
-    def _calc_banned_ngram_tokens(
-        self, prev_input_ids: torch.Tensor, num_hypos: int, cur_len: int
-    ) -> List[Iterable[int]]:
-        """Copied from fairseq for no_repeat_ngram in beam_search"""
-        if cur_len + 1 < self.ngram_size:
-            # return no banned tokens if we haven't generated no_repeat_ngram_size tokens yet
-            return [[] for _ in range(num_hypos)]
-        generated_ngrams = [{} for _ in range(num_hypos)]
-        for idx in range(num_hypos):
-            gen_tokens = prev_input_ids[idx].tolist()
-            generated_ngram = generated_ngrams[idx]
-            for ngram in zip(*[gen_tokens[i:] for i in range(self.ngram_size)]):
-                prev_ngram_tuple = tuple(ngram[:-1])
-                generated_ngram[prev_ngram_tuple] = generated_ngram.get(prev_ngram_tuple, []) + [ngram[-1]]
-
-        def _get_generated_ngrams(hypo_idx):
-            # Before decoding the next token, prevent decoding of ngrams that have already appeared
-            start_idx = cur_len + 1 - self.ngram_size
-            ngram_idx = tuple(prev_input_ids[hypo_idx, start_idx:cur_len].tolist())
-            return generated_ngrams[hypo_idx].get(ngram_idx, [])
-
-        banned_tokens = [_get_generated_ngrams(hypo_idx) for hypo_idx in range(num_hypos)]
-        return banned_tokens
-
-
-class NoBadWordsLogitsProcessor(LogitsProcessor):
-    """
-    :class:`transformers.LogitsProcessor` that enforces that specified sequences will never be sampled.
-
-    Args:
-        bad_words_ids (:obj:`List[List[int]]`):
-            List of list of token ids that are not allowed to be generated. In order to get the tokens of the words
-            that should not appear in the generated text, use :obj:`tokenizer(bad_word,
-            add_prefix_space=True).input_ids`.
-        eos_token_id (:obj:`int`):
-            The id of the `end-of-sequence` token.
-    """
-
-    def __init__(self, bad_words_ids: Iterable[Iterable[int]], eos_token_id: int):
-
-        if not isinstance(bad_words_ids, List) or len(bad_words_ids) == 0:
-            raise ValueError(f"`bad_words_ids` has to be a non-emtpy list, but is {bad_words_ids}.")
-        if any(not isinstance(bad_word_ids, list) for bad_word_ids in bad_words_ids):
-            raise ValueError(f"`bad_words_ids` has to be a list of lists, but is {bad_words_ids}.")
-        if any(
-            any((not isinstance(token_id, (int, np.integer)) or token_id < 0) for token_id in bad_word_ids)
-            for bad_word_ids in bad_words_ids
-        ):
-            raise ValueError(
-                f"Each list in `bad_words_ids` has to be a list of positive integers, but is {bad_words_ids}."
-            )
-
-        self.bad_words_ids = list(filter(lambda bad_token_seq: bad_token_seq != [eos_token_id], bad_words_ids))
-
-        for banned_token_seq in self.bad_words_ids:
-            assert len(banned_token_seq) > 0, "Banned words token sequences {} cannot have an empty list".format(
-                bad_words_ids
-            )
-
-    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
-        banned_tokens = self._calc_banned_bad_words_ids(input_ids)
-        scores = self._set_scores_to_inf_for_banned_tokens(scores, banned_tokens)
-
-        return scores
-
-    def _tokens_match(self, prev_tokens: torch.LongTensor, tokens: List[int]) -> bool:
-        if len(tokens) == 0:
-            # if bad word tokens is just one token always ban it
-            return True
-        elif len(tokens) > len(prev_tokens):
-            # if bad word tokens are longer then prev input_ids they can't be equal
-            return False
-        elif prev_tokens[-len(tokens) :].tolist() == tokens:
-            # if tokens match
-            return True
-        else:
-            return False
-
-    def _calc_banned_bad_words_ids(self, prev_input_ids: Iterable[int]) -> Iterable[int]:
-        banned_tokens = []
-        for prev_input_ids_slice in prev_input_ids:
-            banned_tokens_slice = []
-            for banned_token_seq in self.bad_words_ids:
-                if self._tokens_match(prev_input_ids_slice, banned_token_seq[:-1]) is False:
-                    # if tokens do not match continue
-                    continue
-
-                banned_tokens_slice.append(banned_token_seq[-1])
-
-            banned_tokens.append(banned_tokens_slice)
-
-        return banned_tokens
-
-    def _set_scores_to_inf_for_banned_tokens(self, scores: torch.Tensor, banned_tokens: List[List[int]]) -> None:
-        """
-        Modifies the scores in place by setting the banned token positions to `-inf`. Banned token is expected to be a
-        list of list of banned tokens to ban in the format [[batch index, vocabulary position],...
-
-        Args:
-            scores: logits distribution of shape (batch size, vocabulary size)
-            banned_tokens: list of list of tokens to ban of length (batch_size)
-        """
-        banned_mask_list = []
-        for idx, batch_banned_tokens in enumerate(banned_tokens):
-            for token in batch_banned_tokens:
-                banned_mask_list.append([idx, token])
-        if not banned_mask_list:
-            return scores
-
-        banned_mask = torch.LongTensor(banned_mask_list)
-        indices = torch.ones(len(banned_mask))
-        # A sparse tensor is generated from a list of coordinates: [[0, 1], [0, 2], [2, 0]]. A conversion to dense tensor generates:
-        # [ 0  1  1 ]
-        # [ 0  0  0 ]
-        # [ 1  0  0 ]
-
-        banned_mask = (
-            torch.sparse.LongTensor(banned_mask.t(), indices, scores.size()).to(scores.device).to_dense().bool()
-        )
-        scores = scores.masked_fill(banned_mask, -float("inf"))
-        return scores
@@ -282,9 +282,7 @@ class TensorBoardCallback(TrainerCallback):
                if hasattr(model, "config") and model.config is not None:
                    model_config_json = model.config.to_json_string()
                    self.tb_writer.add_text("model_config", model_config_json)
-            # Version of TensorBoard coming from tensorboardX does not have this method.
-            if hasattr(self.tb_writer, "add_hparams"):
-                self.tb_writer.add_hparams(args.to_sanitized_dict(), metric_dict={})
+            self.tb_writer.add_hparams(args.to_sanitized_dict(), metric_dict={})

    def on_log(self, args, state, control, logs=None, **kwargs):
        if state.is_world_process_zero:
@@ -494,8 +494,8 @@ AUTO_MODEL_PRETRAINED_DOCSTRING = r"""
                    - The model is a model provided by the library (loaded with the `shortcut name` string of a
                      pretrained model).
                    - The model was saved using :meth:`~transformers.PreTrainedModel.save_pretrained` and is reloaded
-                      by supplying the save directory.
-                    - The model is loaded by supplying a local directory as ``pretrained_model_name_or_path`` and a
+                      by suppling the save directory.
+                    - The model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a
                      configuration JSON file named `config.json` is found in the directory.
            state_dict (`Dict[str, torch.Tensor]`, `optional`):
                A state dictionary to use instead of a state dictionary loaded from saved weights file.
@@ -1084,7 +1084,7 @@ class BartForConditionalGeneration(PretrainedBartModel):
        )

    def prepare_inputs_for_generation(
-        self, decoder_input_ids, past=None, attention_mask=None, use_cache=None, encoder_outputs=None, **kwargs
+        self, decoder_input_ids, past, attention_mask, use_cache, encoder_outputs, **kwargs
    ):
        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
@@ -1328,6 +1328,8 @@ class SinusoidalPositionalEmbedding(nn.Embedding):

    def __init__(self, num_positions, embedding_dim, padding_idx=None):
        super().__init__(num_positions, embedding_dim)
+        if embedding_dim % 2 != 0:
+            raise NotImplementedError(f"odd embedding_dim {embedding_dim} not supported")
        self.weight = self._init_weight(self.weight)

    @staticmethod
@@ -1340,11 +1342,10 @@ class SinusoidalPositionalEmbedding(nn.Embedding):
        position_enc = np.array(
            [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
        )
-        out.requires_grad = False  # set early to avoid an error in pytorch-1.8+
-        sentinel = dim // 2 if dim % 2 == 0 else (dim // 2) + 1
-        out[:, 0:sentinel] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))
-        out[:, sentinel:] = torch.FloatTensor(np.cos(position_enc[:, 1::2]))
+        out[:, 0 : dim // 2] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))  # This line breaks for odd n_pos
+        out[:, dim // 2 :] = torch.FloatTensor(np.cos(position_enc[:, 1::2]))
        out.detach_()
+        out.requires_grad = False
        return out

    @torch.no_grad()
@@ -514,12 +514,12 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
    def get_output_embeddings(self):
        return self.lm_head

-    def prepare_inputs_for_generation(self, input_ids, past=None, use_cache=None, **kwargs):
+    def prepare_inputs_for_generation(self, input_ids, past, **kwargs):
        # only last token for inputs_ids if past is defined in kwargs
        if past:
            input_ids = input_ids[:, -1].unsqueeze(-1)

-        return {"input_ids": input_ids, "past_key_values": past, "use_cache": use_cache}
+        return {"input_ids": input_ids, "past_key_values": past, "use_cache": kwargs["use_cache"]}

    @add_start_docstrings_to_model_forward(CTRL_INPUTS_DOCSTRING)
    @add_code_sample_docstrings(
@@ -431,7 +431,7 @@ class EncoderDecoderModel(PreTrainedModel):
            encoder_attentions=encoder_outputs.attentions,
        )

-    def prepare_inputs_for_generation(self, input_ids, past=None, attention_mask=None, encoder_outputs=None, **kwargs):
+    def prepare_inputs_for_generation(self, input_ids, past, attention_mask, encoder_outputs, **kwargs):
        decoder_inputs = self.decoder.prepare_inputs_for_generation(input_ids)
        decoder_attention_mask = decoder_inputs["attention_mask"] if "attention_mask" in decoder_inputs else None
        input_dict = {
@@ -1107,7 +1107,7 @@ class FSMTForConditionalGeneration(PretrainedFSMTModel):
        )

    def prepare_inputs_for_generation(
-        self, decoder_input_ids, past=None, attention_mask=None, use_cache=None, encoder_outputs=None, **kwargs
+        self, decoder_input_ids, past, attention_mask, use_cache, encoder_outputs, **kwargs
    ):
        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
@@ -1800,7 +1800,7 @@ class ProphetNetForConditionalGeneration(ProphetNetPreTrainedModel):
        return loss

    def prepare_inputs_for_generation(
-        self, decoder_input_ids, past=None, attention_mask=None, use_cache=None, encoder_outputs=None, **kwargs
+        self, decoder_input_ids, past, attention_mask, use_cache, encoder_outputs, **kwargs
    ):
        assert encoder_outputs is not None, "`encoder_outputs` have to be passed for generation."

@@ -22,7 +22,6 @@ import torch
 from .configuration_rag import RagConfig
 from .configuration_utils import PretrainedConfig
 from .file_utils import add_start_docstrings_to_model_forward, replace_return_docstrings
-from .generation_beam_search import BeamSearchScorer
 from .modeling_outputs import ModelOutput
 from .modeling_utils import PreTrainedModel
 from .retrieval_rag import RagRetriever
@@ -826,7 +825,7 @@ class RagSequenceForGeneration(RagPreTrainedModel):
        num_return_sequences=None,  # defaults to 1
        num_beams=None,  # defaults to 1
        n_docs=None,
-        **model_kwargs
+        **kwargs
    ):
        """
        Implements RAG sequence "thorough" decoding. Read the :meth:`~transformers.PreTrainedModel.generate``
@@ -873,6 +872,7 @@ class RagSequenceForGeneration(RagPreTrainedModel):
        )
        num_beams = num_beams if num_beams is not None else self.config.num_beams

+        # TODO(patrick) - clean up generate here
        if self.retriever is not None and context_input_ids is None:
            question_hidden_states = self.question_encoder(input_ids, attention_mask=attention_mask)[0]
            context_input_ids = self.retriever(
@@ -887,9 +887,10 @@ class RagSequenceForGeneration(RagPreTrainedModel):
            context_input_ids = context_input_ids.to(input_ids)

        hypos = []
-        model_kwargs["num_beams"] = num_beams
-        model_kwargs["num_return_sequences"] = num_beams
-        model_kwargs["attention_mask"] = None
+        kwargs["num_beams"] = num_beams
+        kwargs["num_return_sequences"] = num_beams
+        kwargs["attention_mask"] = None
+        kwargs["n_docs"] = n_docs

        for index in range(len(input_ids)):
            # first, generate beams from documents:
@@ -897,7 +898,7 @@ class RagSequenceForGeneration(RagPreTrainedModel):

            output_sequences = self.generator.generate(
                generator_input_ids,
-                **model_kwargs,
+                **kwargs,
            )  # n_docs * n_beam, tgt_len
            if do_deduplication:
                # do_deduplication, max_output_len
@@ -1017,15 +1018,7 @@ class RagTokenForGeneration(RagPreTrainedModel):
        return self.rag.generator.adjust_logits_during_generation(logits, cur_len=cur_len, max_length=max_length)

    def prepare_inputs_for_generation(
-        self,
-        decoder_input_ids,
-        past=None,
-        attention_mask=None,
-        use_cache=None,
-        encoder_outputs=None,
-        doc_scores=None,
-        n_docs=None,
-        **kwargs
+        self, decoder_input_ids, past, attention_mask, use_cache, encoder_outputs, doc_scores, n_docs=None, **kwargs
    ):
        return {
            "input_ids": None,
@@ -1229,12 +1222,11 @@ class RagTokenForGeneration(RagPreTrainedModel):
        eos_token_id=None,
        length_penalty=None,
        no_repeat_ngram_size=None,
-        repetition_penalty=None,
        bad_words_ids=None,
        num_return_sequences=None,
        decoder_start_token_id=None,
        n_docs=None,
-        **model_kwargs
+        **kwargs
    ):
        """
        Implements RAG token decoding.
@@ -1315,15 +1307,22 @@ class RagTokenForGeneration(RagPreTrainedModel):
        """
        # set default parameters
        n_docs = n_docs if n_docs is not None else self.config.n_docs
-        num_beams = num_beams if num_beams is not None else self.config.num_beams
        max_length = max_length if max_length is not None else self.config.max_length
+        min_length = min_length if min_length is not None else self.config.min_length
+        early_stopping = early_stopping if early_stopping is not None else self.config.early_stopping
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        num_beams = num_beams if num_beams is not None else self.config.num_beams
+        bos_token_id = bos_token_id if bos_token_id is not None else self.config.generator.bos_token_id
+        pad_token_id = pad_token_id if pad_token_id is not None else self.config.generator.pad_token_id
+        eos_token_id = eos_token_id if eos_token_id is not None else self.config.generator.eos_token_id
+        length_penalty = length_penalty if length_penalty is not None else self.config.length_penalty
+        no_repeat_ngram_size = (
+            no_repeat_ngram_size if no_repeat_ngram_size is not None else self.config.no_repeat_ngram_size
+        )
+        bad_words_ids = bad_words_ids if bad_words_ids is not None else self.config.bad_words_ids
        num_return_sequences = (
            num_return_sequences if num_return_sequences is not None else self.config.num_return_sequences
        )
-        bos_token_id = bos_token_id if bos_token_id is not None else self.config.generator.bos_token_id
-        eos_token_id = eos_token_id if eos_token_id is not None else self.config.generator.eos_token_id
-        pad_token_id = pad_token_id if pad_token_id is not None else self.config.generator.pad_token_id
-        use_cache = use_cache if use_cache is not None else self.config.use_cache
        decoder_start_token_id = (
            decoder_start_token_id
            if decoder_start_token_id is not None
@@ -1366,7 +1365,7 @@ class RagTokenForGeneration(RagPreTrainedModel):
        encoder = self.rag.generator.get_encoder()
        encoder_outputs = encoder(input_ids=context_input_ids, attention_mask=context_attention_mask, return_dict=True)

-        input_ids = torch.full(
+        decoder_input_ids = torch.full(
            (batch_size * num_beams, 1),
            decoder_start_token_id,
            dtype=torch.long,
@@ -1389,57 +1388,64 @@ class RagTokenForGeneration(RagPreTrainedModel):
        doc_scores = doc_scores.repeat_interleave(num_beams, dim=0)

        # define start_len & additional parameters
-        model_kwargs["doc_scores"] = doc_scores
-        model_kwargs["encoder_outputs"] = encoder_outputs
-        model_kwargs["attention_mask"] = context_attention_mask
-        model_kwargs["n_docs"] = n_docs
+        cur_len = 1
+        vocab_size = self.config.generator.vocab_size
+        kwargs["doc_scores"] = doc_scores
+        kwargs["encoder_outputs"] = encoder_outputs
+        kwargs["n_docs"] = n_docs

-        pre_processor = self._get_logits_processor(
-            repetition_penalty=repetition_penalty,
-            no_repeat_ngram_size=no_repeat_ngram_size,
-            bad_words_ids=bad_words_ids,
-            min_length=min_length,
-            eos_token_id=eos_token_id,
-        )
+        # not needed. TODO(PVP): change after generate refactor
+        do_sample = False
+        temperature = self.config.temperature
+        top_k = self.config.top_k
+        top_p = self.config.top_p
+        repetition_penalty = self.config.repetition_penalty

-        if num_beams == 1:
-            if num_return_sequences > 1:
-                raise ValueError(
-                    f"num_return_sequences has to be 1, but is {num_return_sequences} when doing greedy search."
-                )
-            return self.greedy_search(
-                input_ids,
-                pre_processor=pre_processor,
+        if num_beams > 1:
+            return self._generate_beam_search(
+                decoder_input_ids,
+                cur_len=cur_len,
                max_length=max_length,
+                min_length=min_length,
+                do_sample=do_sample,
+                early_stopping=early_stopping,
+                temperature=temperature,
+                top_k=top_k,
+                top_p=top_p,
+                repetition_penalty=repetition_penalty,
+                no_repeat_ngram_size=no_repeat_ngram_size,
+                bad_words_ids=bad_words_ids,
                pad_token_id=pad_token_id,
                eos_token_id=eos_token_id,
-                **model_kwargs,
-            )
-        elif num_beams > 1:
-            length_penalty = length_penalty if length_penalty is not None else self.config.length_penalty
-            early_stopping = early_stopping if early_stopping is not None else self.config.early_stopping
-            if num_return_sequences > num_beams:
-                raise ValueError("`num_return_sequences` has to be smaller or equal to `num_beams`.")
-            beam_scorer = BeamSearchScorer(
                batch_size=batch_size,
-                max_length=max_length,
-                num_beams=num_beams,
-                device=self.device,
+                num_return_sequences=num_return_sequences,
                length_penalty=length_penalty,
-                do_early_stopping=early_stopping,
-                num_beam_hyps_to_keep=num_return_sequences,
-            )
-            return self.beam_search(
-                input_ids,
-                beam_scorer,
-                pre_processor=pre_processor,
-                max_length=max_length,
-                pad_token_id=pad_token_id,
-                eos_token_id=eos_token_id,
-                **model_kwargs,
+                num_beams=num_beams,
+                vocab_size=vocab_size,
+                attention_mask=context_attention_mask,
+                use_cache=use_cache,
+                model_kwargs=kwargs,
            )
        else:
-            raise ValueError(f"`num_beams` has to be an integer strictly superior to 0 (≥ 1), but is {num_beams}")
+            return self._generate_no_beam_search(
+                decoder_input_ids,
+                cur_len=cur_len,
+                max_length=max_length,
+                min_length=min_length,
+                do_sample=do_sample,
+                temperature=temperature,
+                top_k=top_k,
+                top_p=top_p,
+                repetition_penalty=repetition_penalty,
+                no_repeat_ngram_size=no_repeat_ngram_size,
+                bad_words_ids=bad_words_ids,
+                pad_token_id=pad_token_id,
+                eos_token_id=eos_token_id,
+                batch_size=batch_size,
+                attention_mask=context_attention_mask,
+                use_cache=use_cache,
+                model_kwargs=kwargs,
+            )

    def get_input_embeddings(self):
        return self.rag.generator.get_input_embeddings()
@@ -638,6 +638,7 @@ class LSHSelfAttention(nn.Module, EfficientAttentionMixin):
        rotations_shape = (self.num_attention_heads, vectors.shape[-1], num_hashes, rotation_size // 2)
        # create a random self.attention_head_size x num_hashes x num_buckets/2
        random_rotations = torch.randn(rotations_shape, device=vectors.device, dtype=vectors.dtype)
+
        # Output dim: Batch_Size x Num_Attn_Heads x Num_Hashes x Seq_Len x Num_Buckets/2
        rotated_vectors = torch.einsum("bmtd,mdhr->bmhtr", vectors, random_rotations)

@@ -1470,9 +1471,7 @@ class ReformerLayer(nn.Module):
            # every forward pass we sample a different seed
            # for dropout and save for forward fn in backward pass
            # to have correct dropout
-            if self.training:
-                self._init_attention_seed()
-
+            self._init_attention_seed()
            attn_outputs = self.attention(
                hidden_states=hidden_states,
                head_mask=head_mask,
@@ -1495,8 +1494,7 @@ class ReformerLayer(nn.Module):
            # every forward pass we sample a different seed
            # for dropout and save seed for forward fn in backward
            # to have correct dropout
-            if self.training:
-                self._init_feed_forward_seed()
+            self._init_feed_forward_seed()
            # Y_2 = X_2 + g(Y_1)
            hidden_states = hidden_states + self.feed_forward(attn_output)

@@ -2265,7 +2263,7 @@ class ReformerModelWithLMHead(ReformerPreTrainedModel):
            attentions=reformer_outputs.attentions,
        )

-    def prepare_inputs_for_generation(self, input_ids, past=None, use_cache=None, num_hashes=None, **kwargs):
+    def prepare_inputs_for_generation(self, input_ids, past, **kwargs):
        # only last token for inputs_ids if past is defined in kwargs
        if past is not None:
            input_ids = input_ids[:, -1:]
@@ -2273,10 +2271,12 @@ class ReformerModelWithLMHead(ReformerPreTrainedModel):
        inputs_dict = {
            "input_ids": input_ids,
            "past_buckets_states": past,
-            "use_cache": use_cache,
-            "num_hashes": num_hashes,
+            "use_cache": kwargs["use_cache"],
        }

+        if "num_hashes" in kwargs:
+            inputs_dict["num_hashes"] = kwargs["num_hashes"]
+
        return inputs_dict

    def _reorder_cache(self, past, beam_idx):
@@ -1232,9 +1232,7 @@ class T5ForConditionalGeneration(T5PreTrainedModel):
            encoder_attentions=encoder_outputs.attentions,
        )

-    def prepare_inputs_for_generation(
-        self, input_ids, past=None, attention_mask=None, use_cache=None, encoder_outputs=None, **kwargs
-    ):
+    def prepare_inputs_for_generation(self, input_ids, past, attention_mask, use_cache, encoder_outputs, **kwargs):

        # cut decoder_input_ids if past is used
        if past is not None:
@@ -550,8 +550,8 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin):
                    - The model is a model provided by the library (loaded with the `shortcut name` string of a
                      pretrained model).
                    - The model was saved using :func:`~transformers.TFPreTrainedModel.save_pretrained` and is reloaded
-                      by supplying the save directory.
-                    - The model is loaded by supplying a local directory as ``pretrained_model_name_or_path`` and a
+                      by suppling the save directory.
+                    - The model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a
                      configuration JSON file named `config.json` is found in the directory.
            from_pt: (:obj:`bool`, `optional`, defaults to :obj:`False`):
                Load the model weights from a PyTorch state_dict save file (see docstring of
@@ -33,7 +33,7 @@ from .file_utils import (
    add_start_docstrings_to_model_forward,
 )
 from .modeling_transfo_xl_utilities import ProjectedAdaptiveLogSoftmax
-from .modeling_utils import PreTrainedModel
+from .modeling_utils import PreTrainedModel, ModuleUtilsMixin
 from .utils import logging


@@ -231,7 +231,7 @@ class PositionwiseFF(nn.Module):
        return output


-class RelPartialLearnableMultiHeadAttn(nn.Module):
+class RelPartialLearnableMultiHeadAttn(nn.Module, ModuleUtilsMixin):
    def __init__(
        self,
        n_head,
@@ -330,14 +330,14 @@ class RelPartialLearnableMultiHeadAttn(nn.Module):
        if attn_mask is not None and torch.sum(attn_mask).item():
            attn_mask = attn_mask == 1  # Switch to bool
            if attn_mask.dim() == 2:
-                if next(self.parameters()).dtype == torch.float16:
+                if self.dtype == torch.float16:
                    attn_score = (
                        attn_score.float().masked_fill(attn_mask[None, :, :, None], -65000).type_as(attn_score)
                    )
                else:
                    attn_score = attn_score.float().masked_fill(attn_mask[None, :, :, None], -1e30).type_as(attn_score)
            elif attn_mask.dim() == 3:
-                if next(self.parameters()).dtype == torch.float16:
+                if self.dtype == torch.float16:
                    attn_score = attn_score.float().masked_fill(attn_mask[:, :, :, None], -65000).type_as(attn_score)
                else:
                    attn_score = attn_score.float().masked_fill(attn_mask[:, :, :, None], -1e30).type_as(attn_score)
@@ -401,7 +401,7 @@ class RelPartialLearnableDecoderLayer(nn.Module):
        return outputs


-class AdaptiveEmbedding(nn.Module):
+class AdaptiveEmbedding(nn.Module, ModuleUtilsMixin):
    def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, sample_softmax=False):
        super().__init__()

@@ -435,9 +435,8 @@ class AdaptiveEmbedding(nn.Module):
            if self.d_proj != self.d_embed:
                embed = F.linear(embed, self.emb_projs[0])
        else:
-            param = next(self.parameters())
            inp_flat = inp.view(-1)
-            emb_flat = torch.zeros([inp_flat.size(0), self.d_proj], dtype=param.dtype, device=param.device)
+            emb_flat = torch.zeros([inp_flat.size(0), self.d_proj], dtype=self.dtype, device=self.device)
            for i in range(len(self.cutoffs)):
                l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i + 1]

@@ -806,9 +805,8 @@ class TransfoXLModel(TransfoXLPreTrainedModel):
    def init_mems(self, bsz):
        if self.mem_len > 0:
            mems = []
-            param = next(self.parameters())
            for i in range(self.n_layer):
-                empty = torch.zeros(self.mem_len, bsz, self.config.d_model, dtype=param.dtype, device=param.device)
+                empty = torch.zeros(self.mem_len, bsz, self.config.d_model, dtype=self.dtype, device=self.device)
                mems.append(empty)

            return mems
@@ -885,9 +883,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel):
                head_mask = head_mask.expand(self.n_layer, -1, -1, -1, -1)
            elif head_mask.dim() == 2:
                head_mask = head_mask.unsqueeze(1).unsqueeze(1).unsqueeze(1)
-            head_mask = head_mask.to(
-                dtype=next(self.parameters()).dtype
-            )  # switch to fload if need + fp16 compatibility
+            head_mask = head_mask.to(dtype=self.dtype)  # switch to fload if need + fp16 compatibility
        else:
            head_mask = [None] * self.n_layer

@@ -1091,7 +1087,7 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel):
        else:
            return self.crit.out_layers[-1]

-    def prepare_inputs_for_generation(self, input_ids, past=None, **model_kwargs):
+    def prepare_inputs_for_generation(self, input_ids, past, **model_kwargs):
        inputs = {}

        # if past is defined in model kwargs then use it for faster decoding
@@ -784,8 +784,8 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin):
                    - The model is a model provided by the library (loaded with the `shortcut name` string of a
                      pretrained model).
                    - The model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded
-                      by supplying the save directory.
-                    - The model is loaded by supplying a local directory as ``pretrained_model_name_or_path`` and a
+                      by suppling the save directory.
+                    - The model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a
                      configuration JSON file named `config.json` is found in the directory.
            state_dict (:obj:`Dict[str, torch.Tensor]`, `optional`):
                A state dictionary to use instead of a state dictionary loaded from saved weights file.
@@ -16,7 +16,7 @@
 """
 PyTorch XLNet model.
 """
-
+import warnings

 from dataclasses import dataclass
 from typing import List, Optional, Tuple
@@ -1087,6 +1087,18 @@ class XLNetModel(XLNetPreTrainedModel):
        output_hidden_states=None,
        return_dict=None,
    ):
+
+        if self.config.mem_len is None or self.config.mem_len == 0:
+            warnings.warn(
+                "This XLNet config doesn't use attention memories, a core feature of XLNet."
+                " Consider setting `mem_len` to a non-zero value, for example "
+                "`xlnet = XLNetLMHeadModel.from_pretrained('xlnet-base-cased'', mem_len=1024)`,"
+                " for accurate training performance as well as an order of magnitude faster inference."
+                " Starting from version 3.5.0, the default parameter will be 1024, following"
+                " the implementation in https://arxiv.org/abs/1906.08237",
+                FutureWarning,
+            )
+
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
@@ -1300,7 +1312,7 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
    def get_output_embeddings(self):
        return self.lm_loss

-    def prepare_inputs_for_generation(self, input_ids, past=None, use_cache=None, **kwargs):
+    def prepare_inputs_for_generation(self, input_ids, past, **kwargs):
        # Add dummy token at the end (no attention on this one)

        effective_batch_size = input_ids.shape[0]
@@ -1333,7 +1345,7 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
            "input_ids": input_ids,
            "perm_mask": perm_mask,
            "target_mapping": target_mapping,
-            "use_cache": use_cache,
+            "use_cache": kwargs["use_cache"],
        }

        # if past is defined in model kwargs then use it for faster decoding
@@ -22,7 +22,6 @@ import sys
 import uuid
 import warnings
 from abc import ABC, abstractmethod
-from collections.abc import Iterable
 from contextlib import contextmanager
 from os.path import abspath, exists
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
@@ -837,8 +836,6 @@ class TextGenerationPipeline(Pipeline):
              -- The token ids of the generated text.
        """

-        if isinstance(text_inputs, str):
-            text_inputs = [text_inputs]
        results = []
        for prompt_text in text_inputs:
            # Manage correct placement of the tensors
@@ -1325,29 +1322,6 @@ class FillMaskPipeline(Pipeline):
        return results


-class TokenClassificationArgumentHandler(ArgumentHandler):
-    """
-    Handles arguments for token classification.
-    """
-
-    def __call__(self, *args, **kwargs):
-
-        if args is not None and len(args) > 0:
-            if isinstance(args, str):
-                inputs = [args]
-            else:
-                inputs = args
-            batch_size = len(inputs)
-
-        offset_mapping = kwargs.get("offset_mapping", None)
-        if offset_mapping:
-            if isinstance(offset_mapping, list) and isinstance(offset_mapping[0], tuple):
-                offset_mapping = [offset_mapping]
-            if len(offset_mapping) != batch_size:
-                raise ("offset_mapping should have the same batch size as the input")
-        return inputs, offset_mapping
-
-
@add_end_docstrings(
    PIPELINE_INIT_ARGS,
    r"""
@@ -1385,14 +1359,13 @@ class TokenClassificationPipeline(Pipeline):
        ignore_labels=["O"],
        task: str = "",
        grouped_entities: bool = False,
-        ignore_subwords: bool = True,
    ):
        super().__init__(
            model=model,
            tokenizer=tokenizer,
            modelcard=modelcard,
            framework=framework,
-            args_parser=TokenClassificationArgumentHandler(),
+            args_parser=args_parser,
            device=device,
            binary_output=binary_output,
            task=task,
@@ -1407,7 +1380,6 @@ class TokenClassificationPipeline(Pipeline):
        self._basic_tokenizer = BasicTokenizer(do_lower_case=False)
        self.ignore_labels = ignore_labels
        self.grouped_entities = grouped_entities
-        self.ignore_subwords = ignore_subwords

    def __call__(self, inputs: Union[str, List[str]], **kwargs):
        """
@@ -1428,15 +1400,10 @@ class TokenClassificationPipeline(Pipeline):
            - **index** (:obj:`int`, only present when ``self.grouped_entities=False``) -- The index of the
              corresponding token in the sentence.
        """
-
        if isinstance(inputs, str):
            inputs = [inputs]
-
-        offset_mappings = kwargs.get("offset_mappings")
-
        answers = []
-
-        for i, sentence in enumerate(inputs):
+        for sentence in inputs:

            # Manage correct placement of the tensors
            with self.device_placement():
@@ -1446,18 +1413,7 @@ class TokenClassificationPipeline(Pipeline):
                    return_attention_mask=False,
                    return_tensors=self.framework,
                    truncation=True,
-                    return_special_tokens_mask=True,
-                    return_offsets_mapping=self.tokenizer.is_fast,
                )
-                if self.tokenizer.is_fast:
-                    offset_mapping = tokens["offset_mapping"].cpu().numpy()[0]
-                    del tokens["offset_mapping"]
-                elif offset_mappings:
-                    offset_mapping = offset_mappings[i]
-                else:
-                    raise Exception("To decode [UNK] tokens use a fast tokenizer or provide offset_mapping parameter")
-                special_tokens_mask = tokens["special_tokens_mask"].cpu().numpy()[0]
-                del tokens["special_tokens_mask"]

                # Forward
                if self.framework == "tf":
@@ -1474,35 +1430,24 @@ class TokenClassificationPipeline(Pipeline):

            entities = []
            # Filter to labels not in `self.ignore_labels`
-            # Filter special_tokens
            filtered_labels_idx = [
                (idx, label_idx)
                for idx, label_idx in enumerate(labels_idx)
-                if (self.model.config.id2label[label_idx] not in self.ignore_labels) and not special_tokens_mask[idx]
+                if self.model.config.id2label[label_idx] not in self.ignore_labels
            ]

            for idx, label_idx in filtered_labels_idx:
-                start_ind, end_ind = offset_mapping[idx]
-                word_ref = sentence[start_ind:end_ind]
-                word = self.tokenizer.convert_ids_to_tokens([int(input_ids[idx])])[0]
-                is_subword = len(word_ref) != len(word)
-
-                if int(input_ids[idx]) == self.tokenizer.unk_token_id:
-                    word = word_ref
-                    is_subword = False

                entity = {
-                    "word": word,
+                    "word": self.tokenizer.convert_ids_to_tokens(int(input_ids[idx])),
                    "score": score[idx][label_idx].item(),
                    "entity": self.model.config.id2label[label_idx],
                    "index": idx,
                }

-                if self.grouped_entities and self.ignore_subwords:
-                    entity["is_subword"] = is_subword
-
                entities += [entity]

+            # Append grouped entities
            if self.grouped_entities:
                answers += [self.group_entities(entities)]
            # Append ungrouped entities
@@ -1521,8 +1466,8 @@ class TokenClassificationPipeline(Pipeline):
            entities (:obj:`dict`): The entities predicted by the pipeline.
        """
        # Get the first entity in the entity group
-        entity = entities[0]["entity"].split("-")[-1]
-        scores = np.nanmean([entity["score"] for entity in entities])
+        entity = entities[0]["entity"]
+        scores = np.mean([entity["score"] for entity in entities])
        tokens = [entity["word"] for entity in entities]

        entity_group = {
@@ -1547,9 +1492,7 @@ class TokenClassificationPipeline(Pipeline):
            last_idx = entities[-1]["index"]

        for entity in entities:
-
            is_last_idx = entity["index"] == last_idx
-            is_subword = self.ignore_subwords and entity["is_subword"]
            if not entity_group_disagg:
                entity_group_disagg += [entity]
                if is_last_idx:
@@ -1558,19 +1501,10 @@ class TokenClassificationPipeline(Pipeline):

            # If the current entity is similar and adjacent to the previous entity, append it to the disaggregated entity group
            # The split is meant to account for the "B" and "I" suffixes
-            # Shouldn't merge if both entities are B-type
            if (
-                (
-                    entity["entity"].split("-")[-1] == entity_group_disagg[-1]["entity"].split("-")[-1]
-                    and entity["entity"].split("-")[0] != "B"
-                )
+                entity["entity"].split("-")[-1] == entity_group_disagg[-1]["entity"].split("-")[-1]
                and entity["index"] == entity_group_disagg[-1]["index"] + 1
-            ) or is_subword:
-                # Modify subword type to be previous_type
-                if is_subword:
-                    entity["entity"] = entity_group_disagg[-1]["entity"].split("-")[-1]
-                    entity["score"] = np.nan  # set ignored scores to nan and use np.nanmean
-
+            ):
                entity_group_disagg += [entity]
                # Group the entities at the last entity
                if is_last_idx:
@@ -1598,52 +1532,55 @@ class QuestionAnsweringArgumentHandler(ArgumentHandler):
    command-line supplied arguments.
    """

-    def normalize(self, item):
-        if isinstance(item, SquadExample):
-            return item
-        elif isinstance(item, dict):
-            for k in ["question", "context"]:
-                if k not in item:
-                    raise KeyError("You need to provide a dictionary with keys {question:..., context:...}")
-                elif item[k] is None:
-                    raise ValueError("`{}` cannot be None".format(k))
-                elif isinstance(item[k], str) and len(item[k]) == 0:
-                    raise ValueError("`{}` cannot be empty".format(k))
-
-            return QuestionAnsweringPipeline.create_sample(**item)
-        raise ValueError("{} argument needs to be of type (SquadExample, dict)".format(item))
-
    def __call__(self, *args, **kwargs):
-        # Detect where the actual inputs are
+        # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating
        if args is not None and len(args) > 0:
            if len(args) == 1:
-                inputs = args[0]
-            elif len(args) == 2 and {type(el) for el in args} == {str}:
-                inputs = [{"question": args[0], "context": args[1]}]
+                kwargs["X"] = args[0]
            else:
-                inputs = list(args)
+                kwargs["X"] = list(args)
+
        # Generic compatibility with sklearn and Keras
        # Batched data
-        elif "X" in kwargs:
-            inputs = kwargs["X"]
-        elif "data" in kwargs:
-            inputs = kwargs["data"]
+        if "X" in kwargs or "data" in kwargs:
+            inputs = kwargs["X"] if "X" in kwargs else kwargs["data"]
+
+            if isinstance(inputs, dict):
+                inputs = [inputs]
+            else:
+                # Copy to avoid overriding arguments
+                inputs = [i for i in inputs]
+
+            for i, item in enumerate(inputs):
+                if isinstance(item, dict):
+                    if any(k not in item for k in ["question", "context"]):
+                        raise KeyError("You need to provide a dictionary with keys {question:..., context:...}")
+
+                    inputs[i] = QuestionAnsweringPipeline.create_sample(**item)
+
+                elif not isinstance(item, SquadExample):
+                    raise ValueError(
+                        "{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)".format(
+                            "X" if "X" in kwargs else "data"
+                        )
+                    )
+
+            # Tabular input
        elif "question" in kwargs and "context" in kwargs:
-            inputs = [{"question": kwargs["question"], "context": kwargs["context"]}]
+            if isinstance(kwargs["question"], str):
+                kwargs["question"] = [kwargs["question"]]
+
+            if isinstance(kwargs["context"], str):
+                kwargs["context"] = [kwargs["context"]]
+
+            inputs = [
+                QuestionAnsweringPipeline.create_sample(q, c) for q, c in zip(kwargs["question"], kwargs["context"])
+            ]
        else:
            raise ValueError("Unknown arguments {}".format(kwargs))

-        # Normalize inputs
-        if isinstance(inputs, dict):
+        if not isinstance(inputs, list):
            inputs = [inputs]
-        elif isinstance(inputs, Iterable):
-            # Copy to avoid overriding arguments
-            inputs = [i for i in inputs]
-        else:
-            raise ValueError("Invalid arguments {}".format(inputs))
-
-        for i, item in enumerate(inputs):
-            inputs[i] = self.normalize(item)

        return inputs

@@ -2445,8 +2382,6 @@ class ConversationalPipeline(Pipeline):
            updated generated responses for those containing a new user input.
        """

-        if isinstance(conversations, Conversation):
-            conversations = [conversations]
        # Input validation
        if isinstance(conversations, list):
            for conversation in conversations:
@@ -2463,6 +2398,8 @@ class ConversationalPipeline(Pipeline):
            assert (
                self.tokenizer.pad_token_id is not None or self.tokenizer.eos_token_id is not None
            ), "Please make sure that the tokenizer has a pad_token_id or eos_token_id when using a batch input"
+        elif isinstance(conversations, Conversation):
+            conversations = [conversations]
        else:
            raise ValueError("DialoguePipeline expects a Conversation or list of Conversations as an input")

@@ -2491,31 +2428,18 @@ class ConversationalPipeline(Pipeline):
                **generate_kwargs,
            )

-            if self.model.config.is_encoder_decoder:
-                if self.framework == "pt":
-                    history = torch.cat((inputs["input_ids"], generated_responses[:, 1:]), 1)
-                elif self.framework == "tf":
-                    history = tf.concat([inputs["input_ids"], generated_responses[:, 1:]], 1)
-            else:
-                history = generated_responses
-
-            history = self._clean_padding_history(history)
-            if self.model.config.is_encoder_decoder:
-                start_position = 1
-            else:
-                start_position = input_length
-
+            cleaned_history = self._clean_padding_history(generated_responses)
            output = []
            for conversation_index, conversation in enumerate(conversations):
                conversation.mark_processed()
                conversation.generated_responses.append(
                    self.tokenizer.decode(
-                        generated_responses[conversation_index][start_position:],
+                        cleaned_history[conversation_index][input_length:],
                        skip_special_tokens=True,
                        clean_up_tokenization_spaces=clean_up_tokenization_spaces,
                    )
                )
-                conversation.set_history(history[conversation_index])
+                conversation.set_history(cleaned_history[conversation_index])
                output.append(conversation)
            if len(output) == 1:
                return output[0]
@@ -2549,8 +2473,6 @@ class ConversationalPipeline(Pipeline):
            is_previous_pad = False
            for token in sequence:
                if token == self.tokenizer.pad_token_id:
-                    if self.tokenizer.pad_token_id != self.tokenizer.eos_token_id:
-                        continue
                    if is_previous_pad:
                        continue
                    else:
@@ -88,8 +88,8 @@ def is_pipeline_test(test_case):
    """
    Decorator marking a test as a pipeline test.

-    Pipeline tests are skipped by default and we can run only them by setting RUN_PIPELINE_TESTS environment variable
-    to a truthy value and selecting the is_pipeline_test pytest mark.
+    Pipeline tests are skipped by default and we can run only them by setting RUN_PIPELINE_TEST environment variable to
+    a truthy value and selecting the is_pipeline_test pytest mark.

    """
    if not _run_pipeline_tests:
@@ -695,31 +695,6 @@ def mockenv(**kwargs):
    return unittest.mock.patch.dict(os.environ, kwargs)


-# --- pytest conf functions --- #
-
-# to avoid multiple invocation from tests/conftest.py and examples/conftest.py - make sure it's called only once
-pytest_opt_registered = {}
-
-
-def pytest_addoption_shared(parser):
-    """
-    This function is to be called from `conftest.py` via `pytest_addoption` wrapper that has to be defined there.
-
-    It allows loading both `conftest.py` files at once without causing a failure due to adding the same `pytest`
-    option.
-
-    """
-    option = "--make-reports"
-    if option not in pytest_opt_registered:
-        parser.addoption(
-            option,
-            action="store",
-            default=False,
-            help="generate report files. The value of this option is used as a prefix to report names",
-        )
-        pytest_opt_registered[option] = 1
-
-
 def pytest_terminal_summary_main(tr, id):
    """
    Generate multiple reports at the end of test suite run - each report goes into a dedicated file in the current
@@ -753,7 +728,7 @@ def pytest_terminal_summary_main(tr, id):
    dir = "reports"
    Path(dir).mkdir(parents=True, exist_ok=True)
    report_files = {
-        k: f"{dir}/{id}_{k}.txt"
+        k: f"{dir}/report_{id}_{k}.txt"
        for k in [
            "durations",
            "errors",
@@ -849,7 +824,7 @@ def pytest_terminal_summary_main(tr, id):
    config.option.tbstyle = orig_tbstyle


-# --- distributed testing functions --- #
+# the following code deals with async io between processes

 # adapted from https://stackoverflow.com/a/59041913/9201239
 import asyncio  # noqa
@@ -921,10 +896,8 @@ def execute_subprocess_async(cmd, env=None, stdin=None, timeout=180, quiet=False

    cmd_str = " ".join(cmd)
    if result.returncode > 0:
-        stderr = "\n".join(result.stderr)
        raise RuntimeError(
-            f"'{cmd_str}' failed with returncode {result.returncode}\n\n"
-            f"The combined stderr from workers follows:\n{stderr}"
+            f"'{cmd_str}' failed with returncode {result.returncode} - see the `stderr:` messages from above for details."
        )

    # check that the subprocess actually did run and produced some output, should the test rely on
@@ -113,7 +113,6 @@ else:
    T5Tokenizer = None
    XLMRobertaTokenizer = None
    XLNetTokenizer = None
-    XLMProphetNetTokenizer = None

 if is_tokenizers_available():
    from .tokenization_albert_fast import AlbertTokenizerFast
@@ -160,7 +160,7 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer):
            return self.cache[token]
        token = re.sub("([.,!?()])", r" \1", token)
        token = re.sub("(')", r" \1 ", token)
-        token = re.sub(r"\s{2,}", " ", token)
+        token = re.sub("\s{2,}", " ", token)
        if "\n" in token:
            token = token.replace("\n", " __newln__")

@@ -23,7 +23,7 @@ from typing import List, Optional, Union

 import numpy as np
 import torch
-from packaging import version
+from torch.optim.lr_scheduler import SAVE_STATE_WARNING
 from torch.utils.data.distributed import DistributedSampler
 from torch.utils.data.sampler import RandomSampler, Sampler

@@ -34,11 +34,6 @@ from .utils import logging
 if is_torch_tpu_available():
    import torch_xla.core.xla_model as xm

-if version.parse(torch.__version__) <= version.parse("1.4.1"):
-    SAVE_STATE_WARNING = ""
-else:
-    from torch.optim.lr_scheduler import SAVE_STATE_WARNING
-
 logger = logging.get_logger(__name__)


@@ -48,19 +48,12 @@ class TrainingArguments:
            If :obj:`True`, overwrite the content of the output directory. Use this to continue training if
            :obj:`output_dir` points to a checkpoint directory.
        do_train (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether to run training or not. This argument is not directly used by :class:`~transformers.Trainer`, it's
-            intended to be used by your training/evaluation scripts instead. See the `example scripts
-            <https://github.com/huggingface/transformers/tree/master/examples>`__ for more details.
+            Whether to run training or not.
        do_eval (:obj:`bool`, `optional`):
-            Whether to run evaluation on the dev set or not. Will be set to :obj:`True` if :obj:`evaluation_strategy`
-            is different from :obj:`"no"`. This argument is not directly used by :class:`~transformers.Trainer`, it's
-            intended to be used by your training/evaluation scripts instead. See the `example scripts
-            <https://github.com/huggingface/transformers/tree/master/examples>`__ for more details.
+            Whether to run evaluation on the dev set or not. Will default to :obj:`evaluation_strategy` different from
+            :obj:`"no"`.
        do_predict (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether to run predictions on the test set or not. This argument is not directly used by
-            :class:`~transformers.Trainer`, it's intended to be used by your training/evaluation scripts instead. See
-            the `example scripts <https://github.com/huggingface/transformers/tree/master/examples>`__ for more
-            details.
+            Whether to run predictions on the test set or not.
        evaluation_strategy (:obj:`str` or :class:`~transformers.trainer_utils.EvaluationStrategy`, `optional`, defaults to :obj:`"no"`):
            The evaluation strategy to adopt during training. Possible values are:

@@ -26,6 +26,11 @@ class DataCollatorForLanguageModeling:
        requires_pytorch(self)


+class DataCollatorForNextSentencePrediction:
+    def __init__(self, *args, **kwargs):
+        requires_pytorch(self)
+
+
 class DataCollatorForPermutationLanguageModeling:
    def __init__(self, *args, **kwargs):
        requires_pytorch(self)
@@ -40,15 +45,6 @@ class DataCollatorForSOP:
        requires_pytorch(self)


-class DataCollatorForTokenClassification:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-    @classmethod
-    def from_pretrained(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
 class DataCollatorForWholeWordMask:
    def __init__(self, *args, **kwargs):
        requires_pytorch(self)
@@ -108,66 +104,6 @@ class TextDatasetForNextSentencePrediction:
        requires_pytorch(self)


-class BeamScorer:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class BeamSearchScorer:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class LogitsProcessor:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class LogitsProcessorList:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class LogitsWarper:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class MinLengthLogitsProcessor:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class NoBadWordsLogitsProcessor:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class NoRepeatNGramLogitsProcessor:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class RepetitionPenaltyLogitsProcessor:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class TemperatureLogitsWarper:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class TopKLogitsWarper:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
-class TopPLogitsWarper:
-    def __init__(self, *args, **kwargs):
-        requires_pytorch(self)
-
-
 def top_k_top_p_filtering(*args, **kwargs):
    requires_pytorch(top_k_top_p_filtering)

@@ -205,7 +205,7 @@ def main():
        if data_args.train_file is not None:
            data_files["train"] = data_args.train_file
        if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
+            data_files["validation"] = data_args.train_file
        extension = data_args.train_file.split(".")[-1]
        if extension == "txt":
            extension = "text"
@@ -24,14 +24,17 @@ def pytest_configure(config):


 def pytest_addoption(parser):
-    from transformers.testing_utils import pytest_addoption_shared
-
-    pytest_addoption_shared(parser)
+    parser.addoption(
+        "--make_reports",
+        action="store",
+        default=False,
+        help="generate report files - the value will be used as a `report_`+val+`reportname.txt`",
+    )


 def pytest_terminal_summary(terminalreporter):
    from transformers.testing_utils import pytest_terminal_summary_main

-    make_reports = terminalreporter.config.getoption("--make-reports")
+    make_reports = terminalreporter.config.getoption("--make_reports")
    if make_reports:
        pytest_terminal_summary_main(terminalreporter, id=make_reports)
@@ -1,10 +1,7 @@
-import os
-import shutil
-import tempfile
 import unittest

-from transformers import BertTokenizer, is_torch_available, set_seed
-from transformers.testing_utils import require_torch
+from transformers import AutoTokenizer, is_torch_available
+from transformers.testing_utils import require_torch, slow


 if is_torch_available():
@@ -12,26 +9,25 @@ if is_torch_available():

    from transformers import (
        DataCollatorForLanguageModeling,
+        DataCollatorForNextSentencePrediction,
        DataCollatorForPermutationLanguageModeling,
-        DataCollatorForTokenClassification,
-        DataCollatorWithPadding,
+        DataCollatorForSOP,
+        GlueDataset,
+        GlueDataTrainingArguments,
+        LineByLineTextDataset,
+        LineByLineWithSOPTextDataset,
+        TextDataset,
+        TextDatasetForNextSentencePrediction,
        default_data_collator,
    )


+PATH_SAMPLE_TEXT = "./tests/fixtures/sample_text.txt"
+PATH_SAMPLE_TEXT_DIR = "./tests/fixtures/tests_samples/wiki_text"
+
+
@require_torch
 class DataCollatorIntegrationTest(unittest.TestCase):
-    def setUp(self):
-        self.tmpdirname = tempfile.mkdtemp()
-
-        vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
-        self.vocab_file = os.path.join(self.tmpdirname, "vocab.txt")
-        with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
-            vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdirname)
-
    def test_default_with_dict(self):
        features = [{"label": i, "inputs": [0, 1, 2, 3, 4, 5]} for i in range(8)]
        batch = default_data_collator(features)
@@ -61,17 +57,6 @@ class DataCollatorIntegrationTest(unittest.TestCase):
        self.assertEqual(batch["labels"].dtype, torch.long)
        self.assertEqual(batch["inputs"].shape, torch.Size([8, 10]))

-    def test_default_classification_and_regression(self):
-        data_collator = default_data_collator
-
-        features = [{"input_ids": [0, 1, 2, 3, 4], "label": i} for i in range(4)]
-        batch = data_collator(features)
-        self.assertEqual(batch["labels"].dtype, torch.long)
-
-        features = [{"input_ids": [0, 1, 2, 3, 4], "label": float(i)} for i in range(4)]
-        batch = data_collator(features)
-        self.assertEqual(batch["labels"].dtype, torch.float)
-
    def test_default_with_no_labels(self):
        features = [{"label": None, "inputs": [0, 1, 2, 3, 4, 5]} for i in range(8)]
        batch = default_data_collator(features)
@@ -84,147 +69,128 @@ class DataCollatorIntegrationTest(unittest.TestCase):
        self.assertTrue("labels" not in batch)
        self.assertEqual(batch["inputs"].shape, torch.Size([8, 6]))

-    def test_data_collator_with_padding(self):
-        tokenizer = BertTokenizer(self.vocab_file)
-        features = [{"input_ids": [0, 1, 2]}, {"input_ids": [0, 1, 2, 3, 4, 5]}]
+    @slow
+    def test_default_classification(self):
+        MODEL_ID = "bert-base-cased-finetuned-mrpc"
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+        data_args = GlueDataTrainingArguments(
+            task_name="mrpc", data_dir="./tests/fixtures/tests_samples/MRPC", overwrite_cache=True
+        )
+        dataset = GlueDataset(data_args, tokenizer=tokenizer, mode="dev")
+        data_collator = default_data_collator
+        batch = data_collator(dataset.features)
+        self.assertEqual(batch["labels"].dtype, torch.long)

-        data_collator = DataCollatorWithPadding(tokenizer)
-        batch = data_collator(features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size([2, 6]))
-        self.assertEqual(batch["input_ids"][0].tolist(), [0, 1, 2] + [tokenizer.pad_token_id] * 3)
-
-        data_collator = DataCollatorWithPadding(tokenizer, padding="max_length", max_length=10)
-        batch = data_collator(features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size([2, 10]))
-
-        data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8)
-        batch = data_collator(features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size([2, 8]))
-
-    def test_data_collator_for_token_classification(self):
-        tokenizer = BertTokenizer(self.vocab_file)
-        features = [
-            {"input_ids": [0, 1, 2], "labels": [0, 1, 2]},
-            {"input_ids": [0, 1, 2, 3, 4, 5], "labels": [0, 1, 2, 3, 4, 5]},
-        ]
-
-        data_collator = DataCollatorForTokenClassification(tokenizer)
-        batch = data_collator(features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size([2, 6]))
-        self.assertEqual(batch["input_ids"][0].tolist(), [0, 1, 2] + [tokenizer.pad_token_id] * 3)
-        self.assertEqual(batch["labels"].shape, torch.Size([2, 6]))
-        self.assertEqual(batch["labels"][0].tolist(), [0, 1, 2] + [-100] * 3)
-
-        data_collator = DataCollatorForTokenClassification(tokenizer, padding="max_length", max_length=10)
-        batch = data_collator(features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size([2, 10]))
-        self.assertEqual(batch["labels"].shape, torch.Size([2, 10]))
-
-        data_collator = DataCollatorForTokenClassification(tokenizer, pad_to_multiple_of=8)
-        batch = data_collator(features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size([2, 8]))
-        self.assertEqual(batch["labels"].shape, torch.Size([2, 8]))
-
-        data_collator = DataCollatorForTokenClassification(tokenizer, label_pad_token_id=-1)
-        batch = data_collator(features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size([2, 6]))
-        self.assertEqual(batch["input_ids"][0].tolist(), [0, 1, 2] + [tokenizer.pad_token_id] * 3)
-        self.assertEqual(batch["labels"].shape, torch.Size([2, 6]))
-        self.assertEqual(batch["labels"][0].tolist(), [0, 1, 2] + [-1] * 3)
-
-    def test_data_collator_for_language_modeling(self):
-        tokenizer = BertTokenizer(self.vocab_file)
-        no_pad_features = [{"input_ids": list(range(10))}, {"input_ids": list(range(10))}]
-        pad_features = [{"input_ids": list(range(5))}, {"input_ids": list(range(10))}]
+    @slow
+    def test_default_regression(self):
+        MODEL_ID = "distilroberta-base"
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+        data_args = GlueDataTrainingArguments(
+            task_name="sts-b", data_dir="./tests/fixtures/tests_samples/STS-B", overwrite_cache=True
+        )
+        dataset = GlueDataset(data_args, tokenizer=tokenizer, mode="dev")
+        data_collator = default_data_collator
+        batch = data_collator(dataset.features)
+        self.assertEqual(batch["labels"].dtype, torch.float)

+    @slow
+    def test_lm_tokenizer_without_padding(self):
+        tokenizer = AutoTokenizer.from_pretrained("gpt2")
        data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)
-        batch = data_collator(no_pad_features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 10)))
-        self.assertEqual(batch["labels"].shape, torch.Size((2, 10)))
+        # ^ causal lm

-        batch = data_collator(pad_features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 10)))
-        self.assertEqual(batch["labels"].shape, torch.Size((2, 10)))
-
-        tokenizer._pad_token = None
-        data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)
+        dataset = LineByLineTextDataset(tokenizer, file_path=PATH_SAMPLE_TEXT, block_size=512)
+        examples = [dataset[i] for i in range(len(dataset))]
        with self.assertRaises(ValueError):
-            # Expect error due to padding token missing
-            data_collator(pad_features)
+            # Expect error due to padding token missing on gpt2:
+            data_collator(examples)

-        set_seed(42)  # For reproducibility
-        tokenizer = BertTokenizer(self.vocab_file)
+        dataset = TextDataset(tokenizer, file_path=PATH_SAMPLE_TEXT, block_size=512, overwrite_cache=True)
+        examples = [dataset[i] for i in range(len(dataset))]
+        batch = data_collator(examples)
+        self.assertIsInstance(batch, dict)
+        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 512)))
+        self.assertEqual(batch["labels"].shape, torch.Size((2, 512)))
+
+    @slow
+    def test_lm_tokenizer_with_padding(self):
+        tokenizer = AutoTokenizer.from_pretrained("distilroberta-base")
        data_collator = DataCollatorForLanguageModeling(tokenizer)
-        batch = data_collator(no_pad_features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 10)))
-        self.assertEqual(batch["labels"].shape, torch.Size((2, 10)))
+        # ^ masked lm

-        masked_tokens = batch["input_ids"] == tokenizer.mask_token_id
-        self.assertTrue(torch.any(masked_tokens))
-        self.assertTrue(all(x == -100 for x in batch["labels"][~masked_tokens].tolist()))
+        dataset = LineByLineTextDataset(tokenizer, file_path=PATH_SAMPLE_TEXT, block_size=512)
+        examples = [dataset[i] for i in range(len(dataset))]
+        batch = data_collator(examples)
+        self.assertIsInstance(batch, dict)
+        self.assertEqual(batch["input_ids"].shape, torch.Size((31, 107)))
+        self.assertEqual(batch["labels"].shape, torch.Size((31, 107)))

-        batch = data_collator(pad_features)
-        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 10)))
-        self.assertEqual(batch["labels"].shape, torch.Size((2, 10)))
-
-        masked_tokens = batch["input_ids"] == tokenizer.mask_token_id
-        self.assertTrue(torch.any(masked_tokens))
-        self.assertTrue(all(x == -100 for x in batch["labels"][~masked_tokens].tolist()))
+        dataset = TextDataset(tokenizer, file_path=PATH_SAMPLE_TEXT, block_size=512, overwrite_cache=True)
+        examples = [dataset[i] for i in range(len(dataset))]
+        batch = data_collator(examples)
+        self.assertIsInstance(batch, dict)
+        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 512)))
+        self.assertEqual(batch["labels"].shape, torch.Size((2, 512)))

+    @slow
    def test_plm(self):
-        tokenizer = BertTokenizer(self.vocab_file)
-        no_pad_features = [{"input_ids": list(range(10))}, {"input_ids": list(range(10))}]
-        pad_features = [{"input_ids": list(range(5))}, {"input_ids": list(range(10))}]
-
+        tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")
        data_collator = DataCollatorForPermutationLanguageModeling(tokenizer)
+        # ^ permutation lm

-        batch = data_collator(pad_features)
+        dataset = LineByLineTextDataset(tokenizer, file_path=PATH_SAMPLE_TEXT, block_size=512)
+        examples = [dataset[i] for i in range(len(dataset))]
+        batch = data_collator(examples)
        self.assertIsInstance(batch, dict)
-        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 10)))
-        self.assertEqual(batch["perm_mask"].shape, torch.Size((2, 10, 10)))
-        self.assertEqual(batch["target_mapping"].shape, torch.Size((2, 10, 10)))
-        self.assertEqual(batch["labels"].shape, torch.Size((2, 10)))
+        self.assertEqual(batch["input_ids"].shape, torch.Size((31, 112)))
+        self.assertEqual(batch["perm_mask"].shape, torch.Size((31, 112, 112)))
+        self.assertEqual(batch["target_mapping"].shape, torch.Size((31, 112, 112)))
+        self.assertEqual(batch["labels"].shape, torch.Size((31, 112)))

-        batch = data_collator(no_pad_features)
+        dataset = TextDataset(tokenizer, file_path=PATH_SAMPLE_TEXT, block_size=512, overwrite_cache=True)
+        examples = [dataset[i] for i in range(len(dataset))]
+        batch = data_collator(examples)
        self.assertIsInstance(batch, dict)
-        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 10)))
-        self.assertEqual(batch["perm_mask"].shape, torch.Size((2, 10, 10)))
-        self.assertEqual(batch["target_mapping"].shape, torch.Size((2, 10, 10)))
-        self.assertEqual(batch["labels"].shape, torch.Size((2, 10)))
+        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 512)))
+        self.assertEqual(batch["perm_mask"].shape, torch.Size((2, 512, 512)))
+        self.assertEqual(batch["target_mapping"].shape, torch.Size((2, 512, 512)))
+        self.assertEqual(batch["labels"].shape, torch.Size((2, 512)))

        example = [torch.randint(5, [5])]
        with self.assertRaises(ValueError):
            # Expect error due to odd sequence length
            data_collator(example)

+    @slow
    def test_nsp(self):
-        tokenizer = BertTokenizer(self.vocab_file)
-        features = [
-            {"input_ids": [0, 1, 2, 3, 4], "token_type_ids": [0, 1, 2, 3, 4], "next_sentence_label": i}
-            for i in range(2)
-        ]
-        data_collator = DataCollatorForLanguageModeling(tokenizer)
-        batch = data_collator(features)
+        tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+        data_collator = DataCollatorForNextSentencePrediction(tokenizer)

-        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 5)))
-        self.assertEqual(batch["token_type_ids"].shape, torch.Size((2, 5)))
-        self.assertEqual(batch["labels"].shape, torch.Size((2, 5)))
-        self.assertEqual(batch["next_sentence_label"].shape, torch.Size((2,)))
+        dataset = TextDatasetForNextSentencePrediction(tokenizer, file_path=PATH_SAMPLE_TEXT, block_size=512)
+        examples = [dataset[i] for i in range(len(dataset))]
+        batch = data_collator(examples)
+        self.assertIsInstance(batch, dict)

+        # Since there are randomly generated false samples, the total number of samples is not fixed.
+        total_samples = batch["input_ids"].shape[0]
+        self.assertEqual(batch["input_ids"].shape, torch.Size((total_samples, 512)))
+        self.assertEqual(batch["token_type_ids"].shape, torch.Size((total_samples, 512)))
+        self.assertEqual(batch["labels"].shape, torch.Size((total_samples, 512)))
+        self.assertEqual(batch["next_sentence_label"].shape, torch.Size((total_samples,)))
+
+    @slow
    def test_sop(self):
-        tokenizer = BertTokenizer(self.vocab_file)
-        features = [
-            {
-                "input_ids": torch.tensor([0, 1, 2, 3, 4]),
-                "token_type_ids": torch.tensor([0, 1, 2, 3, 4]),
-                "sentence_order_label": i,
-            }
-            for i in range(2)
-        ]
-        data_collator = DataCollatorForLanguageModeling(tokenizer)
-        batch = data_collator(features)
+        tokenizer = AutoTokenizer.from_pretrained("albert-base-v2")
+        data_collator = DataCollatorForSOP(tokenizer)

-        self.assertEqual(batch["input_ids"].shape, torch.Size((2, 5)))
-        self.assertEqual(batch["token_type_ids"].shape, torch.Size((2, 5)))
-        self.assertEqual(batch["labels"].shape, torch.Size((2, 5)))
-        self.assertEqual(batch["sentence_order_label"].shape, torch.Size((2,)))
+        dataset = LineByLineWithSOPTextDataset(tokenizer, file_dir=PATH_SAMPLE_TEXT_DIR, block_size=512)
+        examples = [dataset[i] for i in range(len(dataset))]
+        batch = data_collator(examples)
+        self.assertIsInstance(batch, dict)
+
+        # Since there are randomly generated false samples, the total number of samples is not fixed.
+        total_samples = batch["input_ids"].shape[0]
+        self.assertEqual(batch["input_ids"].shape, torch.Size((total_samples, 512)))
+        self.assertEqual(batch["token_type_ids"].shape, torch.Size((total_samples, 512)))
+        self.assertEqual(batch["labels"].shape, torch.Size((total_samples, 512)))
+        self.assertEqual(batch["sentence_order_label"].shape, torch.Size((total_samples,)))
@@ -36,8 +36,8 @@ class TestCodeExamples(unittest.TestCase):
        self,
        directory: Path,
        identifier: Union[str, None] = None,
-        ignore_files: Union[List[str], None] = None,
-        n_identifier: Union[str, List[str], None] = None,
+        ignore_files: Union[List[str], None] = [],
+        n_identifier: Union[str, None] = None,
        only_modules: bool = True,
    ):
        """
@@ -45,7 +45,7 @@ class TestCodeExamples(unittest.TestCase):
        the doctests in those files

        Args:
-            directory (:obj:`Path`): Directory containing the files
+            directory (:obj:`str`): Directory containing the files
            identifier (:obj:`str`): Will parse files containing this
            ignore_files (:obj:`List[str]`): List of files to skip
            n_identifier (:obj:`str` or :obj:`List[str]`): Will not parse files containing this/these identifiers.
@@ -63,7 +63,6 @@ class TestCodeExamples(unittest.TestCase):
            else:
                files = [file for file in files if n_identifier not in file]

-        ignore_files = ignore_files or []
        ignore_files.append("__init__.py")
        files = [file for file in files if file not in ignore_files]

@@ -72,8 +71,8 @@ class TestCodeExamples(unittest.TestCase):
            print("Testing", file)

            if only_modules:
-                module_identifier = file.split(".")[0]
                try:
+                    module_identifier = file.split(".")[0]
                    module_identifier = getattr(transformers, module_identifier)
                    suite = doctest.DocTestSuite(module_identifier)
                    result = unittest.TextTestRunner().run(suite)
@@ -85,7 +84,7 @@ class TestCodeExamples(unittest.TestCase):
                self.assertIs(result.failed, 0)

    def test_modeling_examples(self):
-        transformers_directory = Path("src/transformers")
+        transformers_directory = "src/transformers"
        files = "modeling"
        ignore_files = [
            "modeling_ctrl.py",
@@ -1,239 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The HuggingFace Team Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a clone of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import unittest
-
-from transformers import is_torch_available
-from transformers.testing_utils import require_torch, torch_device
-
-from .test_modeling_common import floats_tensor, ids_tensor
-
-
-if is_torch_available():
-    import torch
-
-    from transformers.generation_beam_search import BeamHypotheses, BeamSearchScorer
-
-
-class BeamSearchTester:
-    def __init__(
-        self,
-        parent,
-        batch_size=3,
-        sequence_length=10,
-        vocab_size=99,
-        pad_token_id=0,
-        max_length=20,
-        num_beams=4,
-        length_penalty=2.0,
-        do_early_stopping=True,
-        num_beam_hyps_to_keep=2,
-    ):
-        self.parent = parent
-        self.batch_size = batch_size
-        self.sequence_length = sequence_length
-        self.vocab_size = vocab_size
-        self.pad_token_id = pad_token_id
-        self.max_length = max_length
-        self.num_beams = num_beams
-        self.length_penalty = length_penalty
-        self.do_early_stopping = do_early_stopping
-        self.num_beam_hyps_to_keep = num_beam_hyps_to_keep
-
-        # cannot be randomely generated
-        self.eos_token_id = vocab_size + 1
-
-    def prepare_beam_scorer(self, **kwargs):
-        return BeamSearchScorer(
-            batch_size=kwargs.get("batch_size", self.batch_size),
-            max_length=kwargs.get("max_length", self.max_length),
-            num_beams=kwargs.get("num_beams", self.num_beams),
-            device=torch_device,
-            length_penalty=kwargs.get("length_penalty", self.length_penalty),
-            do_early_stopping=kwargs.get("do_early_stopping", self.do_early_stopping),
-            num_beam_hyps_to_keep=kwargs.get("num_beam_hyps_to_keep", self.num_beam_hyps_to_keep),
-        )
-
-    def prepare_inputs(self):
-        input_ids = ids_tensor((self.batch_size * self.num_beams, self.sequence_length), self.vocab_size)
-        next_tokens = ids_tensor((self.batch_size, 2 * self.num_beams), self.vocab_size).to(torch_device)
-        next_indices = ids_tensor((self.batch_size, 2 * self.num_beams), self.num_beams).to(torch_device)
-        next_scores, _ = (-floats_tensor((self.batch_size, 2 * self.num_beams)).to(torch_device)).sort(descending=True)
-        return (input_ids, next_tokens, next_indices, next_scores)
-
-    def check_beam_hypotheses(self, input_ids, *args):
-        # check that correct number of beam hypotheses is set in beam scorer
-        beam_scorer = self.prepare_beam_scorer(do_early_stopping=True)
-        beam_hyp = beam_scorer._beam_hyps[0]
-
-        self.parent.assertEqual(len(beam_scorer._beam_hyps), self.batch_size)
-
-        # check correct type
-        self.parent.assertTrue(isinstance(beam_hyp, BeamHypotheses))
-
-        # check that num_beams is correctly set
-        self.parent.assertEqual(beam_hyp.num_beams, self.num_beams)
-
-        # check for early stopping deactivated
-        for beam_idx in range(self.num_beams):
-            beam_hyp.add(input_ids[beam_idx], -10.0)
-
-        # if early stopping True -> score does not matter
-        self.parent.assertTrue(beam_hyp.is_done(-10.0, 5))
-
-        # re-init
-        beam_scorer = self.prepare_beam_scorer(do_early_stopping=False)
-        beam_hyp = beam_scorer._beam_hyps[0]
-
-        # add `num_beams + 1` beams to change `worst_score`
-        for beam_idx in range(self.num_beams + 1):
-            beam_hyp.add(input_ids[beam_idx], -10.0 + float(beam_idx))
-
-        # -10.0 is removed => -9.0 is worst score
-        self.parent.assertAlmostEqual(beam_hyp.worst_score, -9.0 / (self.sequence_length ** beam_hyp.length_penalty))
-
-        # -5.0 is better than worst score => should not be finished
-        self.parent.assertFalse(beam_hyp.is_done(-5.0, self.sequence_length))
-
-        # -20.0 is worse than worst score => should be finished
-        self.parent.assertTrue(beam_hyp.is_done(-20.0, self.sequence_length))
-
-    def check_beam_scorer_update(self, input_ids, next_tokens, next_indices, next_scores):
-        # check too many eos tokens
-        beam_scorer = self.prepare_beam_scorer()
-
-        tokens = next_tokens.clone()
-        tokens[0, :] = self.eos_token_id
-
-        with self.parent.assertRaises(ValueError):
-            beam_scorer.process(input_ids, next_scores, tokens, next_indices, eos_token_id=self.eos_token_id)
-
-        # check all batches are done
-        beam_scorer = self.prepare_beam_scorer()
-
-        tokens = next_tokens.clone()
-        tokens[:, : self.num_beams] = self.eos_token_id
-        beam_scorer.process(input_ids, next_scores, tokens, next_indices, eos_token_id=self.eos_token_id)
-        # beam scorer should be done
-        self.parent.assertTrue(beam_scorer.is_done)
-
-        # check
-        beam_scorer = self.prepare_beam_scorer()
-
-        tokens = next_tokens.clone()
-        tokens[:, 1] = self.eos_token_id
-        beam_outputs = beam_scorer.process(
-            input_ids, next_scores, tokens, next_indices, eos_token_id=self.eos_token_id
-        )
-        output_scores = beam_outputs["next_beam_scores"]
-        output_tokens = beam_outputs["next_beam_tokens"]
-        output_indices = beam_outputs["next_beam_indices"]
-
-        def cut_expected_tensor(tensor):
-            return torch.cat([tensor[:, :1], tensor[:, 2 : self.num_beams + 1]], dim=1).flatten()
-
-        # check all outptus
-        # cut out id of eos token and take best `num_beams` outputs
-        expected_output_tokens = cut_expected_tensor(tokens)
-        expected_output_scores = cut_expected_tensor(next_scores)
-
-        # add num_beams * batch_idx
-        expected_output_indices = (
-            cut_expected_tensor(next_indices)
-            + (torch.arange(self.num_beams * self.batch_size, device=torch_device) // self.num_beams) * self.num_beams
-        )
-
-        self.parent.assertListEqual(expected_output_tokens.tolist(), output_tokens.tolist())
-        self.parent.assertListEqual(expected_output_indices.tolist(), output_indices.tolist())
-        self.parent.assertTrue(torch.allclose(expected_output_scores, output_scores, atol=1e-3))
-
-        # make sure ids of eos token are correctly saved in beam_hyps of beam scorer
-        for batch_idx in range(self.batch_size):
-            correct_idx = batch_idx * self.num_beams + next_indices[batch_idx, 1]
-            self.parent.assertListEqual(
-                input_ids[correct_idx].tolist(), beam_scorer._beam_hyps[batch_idx].beams[0][-1].tolist()
-            )
-
-    def check_beam_scores_finalize(self, input_ids, next_tokens, next_indices, next_scores):
-        # max_length should be only one more than current input_ids to check that eos is correctly appended
-        max_length = self.sequence_length + 1
-        beam_scorer = self.prepare_beam_scorer(
-            num_beam_hyps_to_keep=1, max_length=max_length, length_penalty=1.0, do_early_stopping=False
-        )
-
-        # update beams and append to input_ids
-        tokens = next_tokens.clone()
-        # first batch, first output has to finish with eos token id since scores are correctly sorted
-        tokens[0, 0] = self.eos_token_id
-        # make sure corresponding score is as good as possible to surely be picked first
-        next_scores[0, 0] = 0.0
-        beam_outputs = beam_scorer.process(
-            input_ids, next_scores, tokens, next_indices, eos_token_id=self.eos_token_id
-        )
-        output_scores = beam_outputs["next_beam_scores"]
-        output_tokens = beam_outputs["next_beam_tokens"]
-        output_indices = beam_outputs["next_beam_indices"]
-
-        input_ids = torch.cat([input_ids[output_indices, :], output_tokens.unsqueeze(-1)], dim=-1)
-
-        # finalize
-        decoded = beam_scorer.finalize(
-            input_ids,
-            output_scores,
-            output_tokens,
-            output_indices,
-            pad_token_id=self.pad_token_id,
-            eos_token_id=self.eos_token_id,
-        )
-        # since `num_beam_hyps_to_keep` = 1 => only return `batch_size` x `max_length`
-        self.parent.assertListEqual(list(decoded.shape), [self.batch_size, max_length])
-
-        # first batch has to finish with eos_token
-        self.parent.assertEqual(decoded[0, -1].item(), self.eos_token_id)
-
-        # other batches cannot finish with eos token
-        self.parent.assertNotEqual(decoded[1, -1].item(), self.eos_token_id)
-        self.parent.assertNotEqual(decoded[2, -1].item(), self.eos_token_id)
-
-        # now test that if `num_beam_hyps_to_keep` is 3 => all beams are returned
-        beam_scorer.num_beam_hyps_to_keep = self.num_beams
-        decoded = beam_scorer.finalize(
-            input_ids,
-            output_scores,
-            output_tokens,
-            output_indices,
-            pad_token_id=self.pad_token_id,
-            eos_token_id=self.eos_token_id,
-        )
-        self.parent.assertListEqual(list(decoded.shape), [self.num_beams * self.batch_size, max_length])
-
-
-@require_torch
-class BeamSearchTest(unittest.TestCase):
-    def setUp(self):
-        self.beam_search_tester = BeamSearchTester(self)
-
-    def test_beam_hypotheses(self):
-        inputs = self.beam_search_tester.prepare_inputs()
-        self.beam_search_tester.check_beam_hypotheses(*inputs)
-
-    def test_beam_scorer_update(self):
-        inputs = self.beam_search_tester.prepare_inputs()
-        self.beam_search_tester.check_beam_scorer_update(*inputs)
-
-    def test_beam_scorer_finalize(self):
-        inputs = self.beam_search_tester.prepare_inputs()
-        self.beam_search_tester.check_beam_scores_finalize(*inputs)
@@ -1,283 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The HuggingFace Team Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a clone of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import unittest
-
-from transformers import is_torch_available
-from transformers.testing_utils import require_torch, torch_device
-
-from .test_modeling_common import ids_tensor
-
-
-if is_torch_available():
-    import torch
-    import torch.nn.functional as F
-
-    from transformers.generation_logits_process import (
-        LogitsProcessorList,
-        MinLengthLogitsProcessor,
-        NoBadWordsLogitsProcessor,
-        NoRepeatNGramLogitsProcessor,
-        RepetitionPenaltyLogitsProcessor,
-        TemperatureLogitsWarper,
-        TopKLogitsWarper,
-        TopPLogitsWarper,
-    )
-
-
-@require_torch
-class LogitsProcessorTest(unittest.TestCase):
-    def _get_uniform_logits(self, batch_size: int, length: int):
-        scores = torch.ones((batch_size, length), device=torch_device, dtype=torch.float) / length
-        return scores
-
-    def test_min_lenght_dist_processor(self):
-        vocab_size = 20
-        batch_size = 4
-        eos_token_id = 0
-
-        min_dist_processor = MinLengthLogitsProcessor(min_length=10, eos_token_id=eos_token_id)
-
-        # check that min length is applied at length 5
-        input_ids = ids_tensor((batch_size, 5), vocab_size=20)
-        scores = self._get_uniform_logits(batch_size, vocab_size)
-        scores_before_min_length = min_dist_processor(input_ids, scores)
-        self.assertListEqual(scores_before_min_length[:, eos_token_id].tolist(), 4 * [-float("inf")])
-
-        # check that min length is not applied anymore at length 15
-        input_ids = ids_tensor((batch_size, 15), vocab_size=20)
-        scores = self._get_uniform_logits(batch_size, vocab_size)
-        scores_before_min_length = min_dist_processor(input_ids, scores)
-        self.assertFalse(torch.isinf(scores_before_min_length).any())
-
-    def test_temperature_dist_warper(self):
-        input_ids = None
-        length = 20
-
-        scores = self._get_uniform_logits(batch_size=2, length=length)
-
-        # tweak scores to not be uniform anymore
-        scores[1, 5] = (1 / length) + 0.1  # peak, 1st batch
-        scores[1, 10] = (1 / length) - 0.4  # valley, 1st batch
-
-        # compute softmax
-        probs = F.softmax(scores, dim=-1)
-
-        temp_dist_warper_sharper = TemperatureLogitsWarper(temperature=0.5)
-        temp_dist_warper_smoother = TemperatureLogitsWarper(temperature=1.3)
-
-        warped_prob_sharp = F.softmax(temp_dist_warper_sharper(input_ids, scores.clone()), dim=-1)
-        warped_prob_smooth = F.softmax(temp_dist_warper_smoother(input_ids, scores.clone()), dim=-1)
-
-        # uniform distribution stays uniform
-        self.assertTrue(torch.allclose(probs[0, :], warped_prob_sharp[0, :], atol=1e-3))
-        self.assertTrue(torch.allclose(probs[0, :], warped_prob_smooth[0, :], atol=1e-3))
-
-        # sharp peaks get higher, valleys get lower
-        self.assertLess(probs[1, :].max(), warped_prob_sharp[1, :].max())
-        self.assertGreater(probs[1, :].min(), warped_prob_sharp[1, :].min())
-
-        # smooth peaks get lower, valleys get higher
-        self.assertGreater(probs[1, :].max(), warped_prob_smooth[1, :].max())
-        self.assertLess(probs[1, :].min(), warped_prob_smooth[1, :].min())
-
-    def test_repetition_penalty_dist_process(self):
-        input_ids = torch.tensor([[0, 1], [5, 0]], device=torch_device, dtype=torch.long)
-        vocab_size = 10
-
-        scores = self._get_uniform_logits(batch_size=2, length=vocab_size)
-
-        # give values special values
-        scores[0, 0] = -(1 / vocab_size)
-        scores[1, 5] = 4 / vocab_size
-
-        rep_penalty_proc = RepetitionPenaltyLogitsProcessor(penalty=2.0)
-
-        scores = rep_penalty_proc(input_ids, scores.clone())
-
-        # check that values were correctly changed
-        self.assertAlmostEqual(scores[0, 0].item(), -(1 / vocab_size) * 2)
-        self.assertAlmostEqual(scores[0, 1].item(), (1 / vocab_size) / 2)
-
-        self.assertAlmostEqual(scores[1, 0].item(), (1 / vocab_size) / 2)
-        self.assertAlmostEqual(scores[1, 5].item(), (4 / vocab_size) / 2)
-
-    def test_top_k_dist_warper(self):
-        input_ids = None
-        vocab_size = 10
-        batch_size = 2
-
-        # create ramp distribution
-        ramp_logits = (
-            torch.arange(vocab_size, device=torch_device, dtype=torch.float).unsqueeze(0).repeat(batch_size, 1)
-        )
-        ramp_logits[1:, : vocab_size // 2] = ramp_logits[1:, : vocab_size // 2] + vocab_size
-
-        top_k_warp = TopKLogitsWarper(3)
-
-        scores = top_k_warp(input_ids, ramp_logits)
-
-        # check that correct tokens are filtered
-        self.assertListEqual(torch.isinf(scores[0]).tolist(), 7 * [True] + 3 * [False])
-        self.assertListEqual(torch.isinf(scores[1]).tolist(), 2 * [True] + 3 * [False] + 5 * [True])
-
-        # check special cases
-        length = 5
-
-        logits = self._get_uniform_logits(batch_size=batch_size, length=length)
-        top_k_warp_safety_check = TopKLogitsWarper(top_k=1, filter_value=0.0, min_tokens_to_keep=3)
-
-        scores = top_k_warp_safety_check(input_ids, logits)
-        # uniform dist is not changed
-        self.assertListEqual((scores == 0.0).to(torch.long).sum(dim=-1).tolist(), [0, 0])
-
-        ramp_logits = torch.arange(length, device=torch_device, dtype=torch.float).unsqueeze(0).repeat(batch_size, 1)
-        scores = top_k_warp_safety_check(input_ids, ramp_logits)
-
-        # min_tokens overwrites k: 3 tokens are kept => 2 tokens are nullified
-        self.assertListEqual((scores == 0.0).to(torch.long).sum(dim=-1).tolist(), [2, 2])
-
-    def test_top_p_dist_warper(self):
-        input_ids = None
-        vocab_size = 10
-        batch_size = 2
-
-        # create distribution and take log (inverse to Softmax as taken in TopPLogitsWarper)
-        dist = torch.log(
-            torch.tensor([[0.3, 0.1, 0.1, 0.5], [0.15, 0.3, 0.3, 0.25]], device=torch_device, dtype=torch.float)
-        )
-
-        top_p_warp = TopPLogitsWarper(0.7)
-        filtered_dist = torch.exp(top_p_warp(input_ids, dist))
-
-        # dist should be filtered to keep min num values so that sum is >= 0.7
-        # exp (-inf) => 0
-        EXPECTED_FILTERED_DIST = torch.tensor(
-            [[0.3, 0.0, 0.0, 0.5], [0.0, 0.3, 0.3, 0.25]], device=torch_device, dtype=torch.float
-        )
-        self.assertTrue(torch.allclose(filtered_dist, EXPECTED_FILTERED_DIST, atol=1e-3))
-
-        # check edge cases with negative and extreme logits
-        ramp_logits = torch.arange(vocab_size, device=torch_device, dtype=torch.float).unsqueeze(0).repeat(
-            batch_size, 1
-        ) - (vocab_size // 2)
-
-        # make ramp_logits more extreme
-        ramp_logits[1] = ramp_logits[1] * 100.0
-
-        # make sure at least 2 tokens are kept
-        top_p_warp = TopPLogitsWarper(0.9, min_tokens_to_keep=2, filter_value=0.0)
-        filtered_dist = top_p_warp(input_ids, ramp_logits)
-
-        # first batch should keep three tokens, second batch would keep only 1, but due to `min_tokens_to_keep=2` keeps 2.
-        self.assertListEqual((filtered_dist != 0.0).to(torch.long).sum(dim=-1).tolist(), [3, 2])
-
-    def test_no_repeat_ngram_dist_processor(self):
-        vocab_size = 3
-        batch_size = 2
-
-        input_ids = torch.tensor([[1, 1, 2, 1], [0, 1, 0, 1]], device=torch_device, dtype=torch.long)
-        scores = self._get_uniform_logits(batch_size, vocab_size)
-
-        no_repeat_proc_2_gram = NoRepeatNGramLogitsProcessor(2)
-        no_repeat_proc_3_gram = NoRepeatNGramLogitsProcessor(3)
-
-        filtered_scores_2_gram = no_repeat_proc_2_gram(input_ids, scores.clone())
-        filtered_scores_3_gram = no_repeat_proc_3_gram(input_ids, scores.clone())
-
-        # 2-gram would forbid 2nd and 3rd token (1,2) at 1st batch and 1st token (0) at 2nd batch
-        self.assertListEqual(torch.isinf(filtered_scores_2_gram).tolist(), [[False, True, True], [True, False, False]])
-
-        # 3-gram would forbid no token at 1st batch and 1st token (0) at 2nd batch
-        self.assertListEqual(
-            torch.isinf(filtered_scores_3_gram).tolist(), [[False, False, False], [True, False, False]]
-        )
-
-    def test_no_bad_words_dist_processor(self):
-        vocab_size = 5
-        batch_size = 2
-        eos_token_id = 4
-
-        input_ids = torch.tensor([[0, 1, 3, 1], [0, 1, 0, 1]], device=torch_device, dtype=torch.long)
-        bad_word_tokens = [[1], [4], [1, 0], [0, 1, 2], [1, 3, 1, 3]]
-        scores = self._get_uniform_logits(batch_size, vocab_size)
-
-        no_bad_words_dist_proc = NoBadWordsLogitsProcessor(bad_words_ids=bad_word_tokens, eos_token_id=eos_token_id)
-
-        filtered_scores = no_bad_words_dist_proc(input_ids, scores.clone())
-
-        # batch 1: 1st, 2nd, and 4th (0, 1, 3) token are forbidden
-        # batch 2: 1st, 2nd, and 3rd (0, 1, 2) token are forbidden
-        # Note that 5th element cannot be forbidden as it is EOS token
-        self.assertListEqual(
-            torch.isinf(filtered_scores).tolist(), [[True, True, False, True, False], [True, True, True, False, False]]
-        )
-
-        # check edge case
-        no_bad_words_dist_proc = NoBadWordsLogitsProcessor(bad_words_ids=[[4]], eos_token_id=eos_token_id)
-        filtered_scores = no_bad_words_dist_proc(input_ids, scores.clone())
-        self.assertTrue(torch.allclose(scores, filtered_scores, atol=1e-3))
-
-    def test_processor_list(self):
-        batch_size = 4
-        sequence_length = 10
-        vocab_size = 15
-        eos_token_id = 0
-
-        # dummy input_ids and scores
-        input_ids = ids_tensor((batch_size, sequence_length), vocab_size)
-        input_ids_comp = input_ids.clone()
-
-        scores = self._get_uniform_logits(batch_size, vocab_size)
-        scores_comp = scores.clone()
-
-        # instantiate all dist processors
-        min_dist_proc = MinLengthLogitsProcessor(min_length=10, eos_token_id=eos_token_id)
-        temp_dist_warp = TemperatureLogitsWarper(temperature=0.5)
-        rep_penalty_proc = RepetitionPenaltyLogitsProcessor(penalty=2.0)
-        top_k_warp = TopKLogitsWarper(3)
-        top_p_warp = TopPLogitsWarper(0.8)
-        no_repeat_proc = NoRepeatNGramLogitsProcessor(2)
-        no_bad_words_dist_proc = NoBadWordsLogitsProcessor(bad_words_ids=[[1]], eos_token_id=eos_token_id)
-
-        # no processor list
-        scores = min_dist_proc(input_ids, scores)
-        scores = temp_dist_warp(input_ids, scores)
-        scores = rep_penalty_proc(input_ids, scores)
-        scores = top_k_warp(input_ids, scores)
-        scores = top_p_warp(input_ids, scores)
-        scores = no_repeat_proc(input_ids, scores)
-        scores = no_bad_words_dist_proc(input_ids, scores)
-
-        # with processor list
-        processor = LogitsProcessorList(
-            [
-                min_dist_proc,
-                temp_dist_warp,
-                rep_penalty_proc,
-                top_k_warp,
-                top_p_warp,
-                no_repeat_proc,
-                no_bad_words_dist_proc,
-            ]
-        )
-        scores_comp = processor(input_ids, scores_comp)
-
-        # scores should be equal
-        self.assertTrue(torch.allclose(scores, scores_comp, atol=1e-3))
-
-        # input_ids should never be changed
-        self.assertListEqual(input_ids.tolist(), input_ids_comp.tolist())
@@ -1,513 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The HuggingFace Team Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a clone of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import unittest
-
-from transformers import is_torch_available
-from transformers.testing_utils import require_torch, torch_device
-
-
-if is_torch_available():
-    import torch
-
-    from transformers import top_k_top_p_filtering
-    from transformers.generation_beam_search import BeamSearchScorer
-    from transformers.generation_logits_process import (
-        LogitsProcessorList,
-        MinLengthLogitsProcessor,
-        NoBadWordsLogitsProcessor,
-        NoRepeatNGramLogitsProcessor,
-        RepetitionPenaltyLogitsProcessor,
-        TemperatureLogitsWarper,
-        TopKLogitsWarper,
-        TopPLogitsWarper,
-    )
-
-
-class GenerationTesterMixin:
-    model_tester = None
-    all_generative_model_classes = ()
-
-    def _get_input_ids_and_config(self):
-        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-        input_ids = inputs_dict["input_ids"]
-        attention_mask = torch.ones_like(input_ids)
-
-        # cut to half length & take max batch_size 3
-        max_batch_size = 2
-        sequence_length = input_ids.shape[-1] // 2
-        input_ids = input_ids[:max_batch_size, :sequence_length]
-        attention_mask = attention_mask[:max_batch_size, :sequence_length]
-
-        # generate max 3 tokens
-        max_length = input_ids.shape[-1] + 3
-        if config.eos_token_id is not None and config.pad_token_id is None:
-            # hack to allow generate for models such as GPT2 as is done in `generate()`
-            config.pad_token_id = config.eos_token_id
-        return config, input_ids, attention_mask, max_length
-
-    @staticmethod
-    def _get_logits_processor_and_kwargs(input_length, eos_token_id):
-        process_kwargs = {
-            "min_length": input_length + 1,
-            "bad_words_ids": [[1, 0]],
-            "no_repeat_ngram_size": 2,
-            "repetition_penalty": 1.2,
-        }
-        logits_processor = LogitsProcessorList(
-            (
-                [
-                    MinLengthLogitsProcessor(process_kwargs["min_length"], eos_token_id),
-                ]
-                if eos_token_id is not None
-                else []
-            )
-            + [
-                NoBadWordsLogitsProcessor(process_kwargs["bad_words_ids"], eos_token_id),
-                NoRepeatNGramLogitsProcessor(process_kwargs["no_repeat_ngram_size"]),
-                RepetitionPenaltyLogitsProcessor(process_kwargs["repetition_penalty"]),
-            ]
-        )
-        return process_kwargs, logits_processor
-
-    @staticmethod
-    def _get_warper_and_kwargs(num_beams):
-        warp_kwargs = {"top_k": 10, "top_p": 0.7, "temperature": 0.7}
-        logits_warper = LogitsProcessorList(
-            [
-                TopKLogitsWarper(top_k=warp_kwargs["top_k"], min_tokens_to_keep=(2 if num_beams > 1 else 1)),
-                TopPLogitsWarper(top_p=warp_kwargs["top_p"], min_tokens_to_keep=(2 if num_beams > 1 else 1)),
-                TemperatureLogitsWarper(warp_kwargs["temperature"]),
-            ]
-        )
-        return warp_kwargs, logits_warper
-
-    @staticmethod
-    def _get_beam_scorer_and_kwargs(batch_size, max_length, num_return_sequences=1):
-        beam_kwargs = {
-            "early_stopping": False,
-            "length_penalty": 2.0,
-            "num_beams": 2,
-            "num_return_sequences": num_return_sequences,
-        }
-        beam_scorer = BeamSearchScorer(
-            batch_size=batch_size,
-            max_length=max_length,
-            num_beams=beam_kwargs["num_beams"],
-            device=torch_device,
-            length_penalty=beam_kwargs["length_penalty"],
-            do_early_stopping=beam_kwargs["early_stopping"],
-            num_beam_hyps_to_keep=num_return_sequences,
-        )
-        return beam_kwargs, beam_scorer
-
-    @staticmethod
-    def _get_encoder_outputs(model, input_ids, attention_mask, num_interleave=1):
-        encoder = model.get_encoder()
-        encoder_outputs = encoder(input_ids, attention_mask=attention_mask, return_dict=True)
-        encoder_outputs["last_hidden_state"] = encoder_outputs.last_hidden_state.repeat_interleave(
-            num_interleave, dim=0
-        )
-        input_ids = torch.zeros_like(input_ids[:, :1]) + model._get_decoder_start_token_id()
-        attention_mask = None
-        return encoder_outputs, input_ids, attention_mask
-
-    def test_greedy_generate(self):
-        for model_class in self.all_generative_model_classes:
-            config, input_ids, attention_mask, max_length = self._get_input_ids_and_config()
-
-            logits_process_kwargs, logits_processor = self._get_logits_processor_and_kwargs(
-                input_ids.shape[-1], config.eos_token_id
-            )
-
-            model = model_class(config).to(torch_device)
-            model.eval()
-
-            # check `generate()` and `greedy_search()` are equal
-            kwargs = {}
-            if model.config.is_encoder_decoder:
-                max_length = 4
-
-            output_ids_generate = model.generate(
-                input_ids,
-                attention_mask=attention_mask,
-                do_sample=False,
-                num_beams=1,
-                max_length=max_length,
-                **logits_process_kwargs,
-            )
-
-            if model.config.is_encoder_decoder:
-                encoder_outputs, input_ids, attention_mask = self._get_encoder_outputs(
-                    model, input_ids, attention_mask
-                )
-                kwargs["encoder_outputs"] = encoder_outputs
-
-            with torch.no_grad():
-                output_ids_greedy = model.greedy_search(
-                    input_ids,
-                    max_length=max_length,
-                    attention_mask=attention_mask,
-                    logits_processor=logits_processor,
-                    **kwargs,
-                )
-            self.assertListEqual(output_ids_generate.tolist(), output_ids_greedy.tolist())
-
-    def test_sample_generate(self):
-        for model_class in self.all_generative_model_classes:
-            config, input_ids, attention_mask, max_length = self._get_input_ids_and_config()
-            process_kwargs, logits_processor = self._get_logits_processor_and_kwargs(
-                input_ids.shape[-1], config.eos_token_id
-            )
-            logits_warper_kwargs, logits_warper = self._get_warper_and_kwargs(num_beams=1)
-
-            model = model_class(config).to(torch_device)
-            model.eval()
-
-            # check `generate()` and `sample()` are equal
-            if model.config.is_encoder_decoder:
-                max_length = 4
-
-            torch.manual_seed(0)
-            output_ids_generate = model.generate(
-                input_ids,
-                do_sample=True,
-                num_beams=1,
-                max_length=max_length,
-                attention_mask=attention_mask,
-                **logits_warper_kwargs,
-                **process_kwargs,
-            )
-
-            torch.manual_seed(0)
-            kwargs = {}
-            if model.config.is_encoder_decoder:
-                encoder_outputs, input_ids_clone, attention_mask_clone = self._get_encoder_outputs(
-                    model, input_ids, attention_mask
-                )
-                kwargs["encoder_outputs"] = encoder_outputs
-            else:
-                attention_mask_clone = attention_mask
-                input_ids_clone = input_ids
-
-            with torch.no_grad():
-                output_ids_sample = model.sample(
-                    input_ids_clone,
-                    attention_mask=attention_mask_clone,
-                    max_length=max_length,
-                    logits_processor=logits_processor,
-                    logits_warper=logits_warper,
-                    **kwargs,
-                )
-            self.assertListEqual(output_ids_generate.tolist(), output_ids_sample.tolist())
-
-            # check `generate()` and `sample()` yield equal results for `num_return_sequences`
-            num_return_sequences = 3
-            if model.config.is_encoder_decoder:
-                max_length = 4
-
-            torch.manual_seed(0)
-            output_ids_generate = model.generate(
-                input_ids,
-                do_sample=True,
-                num_beams=1,
-                max_length=max_length,
-                num_return_sequences=num_return_sequences,
-                attention_mask=attention_mask,
-                **logits_warper_kwargs,
-                **process_kwargs,
-            )
-
-            torch.manual_seed(0)
-            kwargs = {}
-            if model.config.is_encoder_decoder:
-                encoder_outputs, input_ids_clone, attention_mask_clone = self._get_encoder_outputs(
-                    model, input_ids, attention_mask, num_interleave=num_return_sequences
-                )
-                kwargs["encoder_outputs"] = encoder_outputs
-                input_ids_clone = input_ids_clone.repeat_interleave(num_return_sequences, dim=0)
-            else:
-                attention_mask_clone = attention_mask.repeat_interleave(num_return_sequences, dim=0)
-                input_ids_clone = input_ids.repeat_interleave(num_return_sequences, dim=0)
-
-            with torch.no_grad():
-                output_ids_sample = model.sample(
-                    input_ids_clone,
-                    attention_mask=attention_mask_clone,
-                    max_length=max_length,
-                    logits_processor=logits_processor,
-                    logits_warper=logits_warper,
-                    **kwargs,
-                )
-            self.assertListEqual(output_ids_generate.tolist(), output_ids_sample.tolist())
-
-    def test_beam_search_generate(self):
-        for model_class in self.all_generative_model_classes:
-            config, input_ids, attention_mask, max_length = self._get_input_ids_and_config()
-
-            logits_process_kwargs, logits_processor = self._get_logits_processor_and_kwargs(
-                input_ids.shape[-1], config.eos_token_id
-            )
-
-            model = model_class(config).to(torch_device)
-            model.eval()
-
-            # check `generate()` and `beam_search()` are equal
-            if model.config.is_encoder_decoder:
-                max_length = 4
-            beam_kwargs, beam_scorer = self._get_beam_scorer_and_kwargs(input_ids.shape[0], max_length)
-            output_ids_generate = model.generate(
-                input_ids,
-                attention_mask=attention_mask,
-                do_sample=False,
-                max_length=max_length,
-                **beam_kwargs,
-                **logits_process_kwargs,
-            )
-
-            # beam_search does not automatically interleave `batch_size` dim for `num_beams`
-            kwargs = {}
-            if model.config.is_encoder_decoder:
-                encoder_outputs, input_ids_clone, attention_mask_clone = self._get_encoder_outputs(
-                    model, input_ids, attention_mask, num_interleave=beam_scorer.num_beams
-                )
-                kwargs["encoder_outputs"] = encoder_outputs
-                input_ids_clone = input_ids_clone.repeat_interleave(beam_scorer.num_beams, dim=0)
-            else:
-                attention_mask_clone = attention_mask.repeat_interleave(beam_scorer.num_beams, dim=0)
-                input_ids_clone = input_ids.repeat_interleave(beam_scorer.num_beams, dim=0)
-
-            with torch.no_grad():
-                output_ids_beam_search = model.beam_search(
-                    input_ids_clone,
-                    beam_scorer,
-                    max_length=max_length,
-                    attention_mask=attention_mask_clone,
-                    logits_processor=logits_processor,
-                    **kwargs,
-                )
-            self.assertListEqual(output_ids_generate.tolist(), output_ids_beam_search.tolist())
-
-            # check `generate()` and `beam_search()` are equal for `num_return_sequences`
-            num_return_sequences = 2
-            if model.config.is_encoder_decoder:
-                max_length = 4
-            beam_kwargs, beam_scorer = self._get_beam_scorer_and_kwargs(
-                input_ids.shape[0], max_length, num_return_sequences=num_return_sequences
-            )
-
-            output_ids_generate = model.generate(
-                input_ids,
-                attention_mask=attention_mask,
-                do_sample=False,
-                max_length=max_length,
-                **beam_kwargs,
-                **logits_process_kwargs,
-            )
-            # beam_search does not automatically interleave `batch_size` dim for `num_beams`
-            kwargs = {}
-            if model.config.is_encoder_decoder:
-                encoder_outputs, input_ids_clone, attention_mask_clone = self._get_encoder_outputs(
-                    model, input_ids, attention_mask, num_interleave=beam_scorer.num_beams
-                )
-                kwargs["encoder_outputs"] = encoder_outputs
-                input_ids_clone = input_ids_clone.repeat_interleave(beam_scorer.num_beams, dim=0)
-            else:
-                attention_mask_clone = attention_mask.repeat_interleave(beam_scorer.num_beams, dim=0)
-                input_ids_clone = input_ids.repeat_interleave(beam_scorer.num_beams, dim=0)
-
-            with torch.no_grad():
-                output_ids_beam_search = model.beam_search(
-                    input_ids_clone,
-                    beam_scorer,
-                    max_length=max_length,
-                    attention_mask=attention_mask_clone,
-                    logits_processor=logits_processor,
-                    **kwargs,
-                )
-            self.assertListEqual(output_ids_generate.tolist(), output_ids_beam_search.tolist())
-
-    def test_beam_sample_generate(self):
-        for model_class in self.all_generative_model_classes:
-            config, input_ids, attention_mask, max_length = self._get_input_ids_and_config()
-            logits_warper_kwargs, logits_warper = self._get_warper_and_kwargs(num_beams=1)
-
-            model = model_class(config).to(torch_device)
-            model.eval()
-
-            # check `generate()` and `beam_search()` are equal
-            # change `num_return_sequences = 2` but not for `beam_scorer`
-            num_return_sequences = 2
-            if model.config.is_encoder_decoder:
-                max_length = 4
-            beam_kwargs, beam_scorer = self._get_beam_scorer_and_kwargs(
-                input_ids.shape[0] * num_return_sequences, max_length
-            )
-            beam_kwargs["num_return_sequences"] = num_return_sequences
-            torch.manual_seed(0)
-            output_ids_generate = model.generate(
-                input_ids,
-                attention_mask=attention_mask,
-                do_sample=True,
-                max_length=max_length,
-                **beam_kwargs,
-                **logits_warper_kwargs,
-            )
-            # beam_search does not automatically interleave `batch_size` dim for `num_beams * num_return_sequences`
-            kwargs = {}
-            if model.config.is_encoder_decoder:
-                encoder_outputs, input_ids, attention_mask = self._get_encoder_outputs(
-                    model, input_ids, attention_mask, num_interleave=beam_scorer.num_beams * num_return_sequences
-                )
-                kwargs["encoder_outputs"] = encoder_outputs
-            else:
-                attention_mask = attention_mask.repeat_interleave(beam_scorer.num_beams * num_return_sequences, dim=0)
-
-            torch.manual_seed(0)
-            with torch.no_grad():
-                output_ids_beam_sample = model.beam_sample(
-                    input_ids.repeat_interleave(beam_scorer.num_beams * num_return_sequences, dim=0),
-                    beam_scorer,
-                    max_length=max_length,
-                    attention_mask=attention_mask,
-                    logits_warper=logits_warper,
-                    **kwargs,
-                )
-            self.assertListEqual(output_ids_generate.tolist(), output_ids_beam_sample.tolist())
-
-        def test_generate_without_input_ids(self):
-            config, _, _, max_length = self._get_input_ids_and_config()
-
-            # if no bos token id => cannot generate from None
-            if config.bos_token_id is None:
-                return
-
-            for model_class in self.all_generative_model_classes:
-                model = model_class(config).to(torch_device)
-                model.eval()
-
-                output_ids_generate = model.generate(
-                    do_sample=False,
-                    max_length=max_length,
-                )
-
-                self.assertIsNotNone(output_ids_generate)
-
-
-@require_torch
-class UtilsFunctionsTest(unittest.TestCase):
-
-    # tests whether the top_k_top_p function behaves as expected
-    def test_top_k_top_p_filtering(self):
-        logits = torch.tensor(
-            [
-                [
-                    8.2220991,  # 3rd highest value; idx. 0
-                    -0.5620044,
-                    5.23229752,
-                    4.0386393,
-                    -6.8798378,
-                    -0.54785802,
-                    -3.2012153,
-                    2.92777176,
-                    1.88171953,
-                    7.35341276,
-                    8.43207833,  # 2nd highest value; idx. 10
-                    -9.85711836,
-                    -5.96209236,
-                    -1.13039161,
-                    -7.1115294,
-                    -0.8369633,
-                    -5.3186408,
-                    7.06427407,
-                    0.81369344,
-                    -0.82023817,
-                    -5.9179796,
-                    0.58813443,
-                    -6.99778438,
-                    4.71551189,
-                    -0.18771637,
-                    7.44020759,  # 4th highest value; idx. 25
-                    9.38450987,  # 1st highest value; idx. 26
-                    2.12662941,
-                    -9.32562038,
-                    2.35652522,
-                ],  # cummulative prob of 4 highest values <= 0.6
-                [
-                    0.58425518,
-                    4.53139238,
-                    -5.57510464,
-                    -6.28030699,
-                    -7.19529503,
-                    -4.02122551,
-                    1.39337037,
-                    -6.06707057,
-                    1.59480517,
-                    -9.643119,
-                    0.03907799,
-                    0.67231762,
-                    -8.88206726,
-                    6.27115922,  # 4th highest value; idx. 13
-                    2.28520723,
-                    4.82767506,
-                    4.30421368,
-                    8.8275313,  # 2nd highest value; idx. 17
-                    5.44029958,
-                    -4.4735794,
-                    7.38579536,  # 3rd highest value; idx. 20
-                    -2.91051663,
-                    2.61946077,
-                    -2.5674762,
-                    -9.48959302,
-                    -4.02922645,
-                    -1.35416918,
-                    9.67702323,  # 1st highest value; idx. 27
-                    -5.89478553,
-                    1.85370467,
-                ],  # cummulative prob of 4 highest values <= 0.6
-            ],
-            dtype=torch.float,
-            device=torch_device,
-        )
-
-        non_inf_expected_idx = torch.tensor(
-            [[0, 0], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 20], [1, 27]],
-            dtype=torch.long,
-            device=torch_device,
-        )  # expected non filtered idx as noted above
-
-        non_inf_expected_output = torch.tensor(
-            [
-                8.2221,
-                8.4321,
-                7.4402,
-                9.3845,
-                6.2712,
-                8.8275,
-                7.3858,
-                9.6770,
-            ],  # expected non filtered values as noted above
-            dtype=torch.float,
-            device=torch_device,
-        )
-
-        output = top_k_top_p_filtering(logits, top_k=10, top_p=0.6, min_tokens_to_keep=4)
-        non_inf_output = output[output != -float("inf")].to(device=torch_device)
-        non_inf_idx = (output != -float("inf")).nonzero().to(device=torch_device)
-
-        self.assertTrue(torch.allclose(non_inf_expected_output, non_inf_output, atol=1e-12))
-        self.assertTrue(torch.all(torch.eq(non_inf_expected_idx, non_inf_idx)))
@@ -23,7 +23,6 @@ from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, ids_tensor


@@ -129,7 +128,7 @@ def prepare_bart_inputs_dict(


@require_torch
-class BARTModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class BARTModelTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (
        (BartModel, BartForConditionalGeneration, BartForSequenceClassification, BartForQuestionAnswering)
        if is_torch_available()
@@ -621,8 +620,8 @@ class TestSinusoidalPositionalEmbeddings(unittest.TestCase):
        self.assertListEqual(no_cache[-1].tolist(), yes_cache[0][0].tolist())

    def test_odd_embed_dim(self):
-        # odd embedding_dim is allowed
-        SinusoidalPositionalEmbedding(num_positions=4, embedding_dim=5, padding_idx=0).to(torch_device)
+        with self.assertRaises(NotImplementedError):
+            SinusoidalPositionalEmbedding(num_positions=4, embedding_dim=5, padding_idx=0).to(torch_device)

        # odd num_positions is allowed
        SinusoidalPositionalEmbedding(num_positions=5, embedding_dim=4, padding_idx=0).to(torch_device)
@@ -20,7 +20,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


@@ -358,12 +357,11 @@ class BertModelTester:


@require_torch
-class BertModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class BertModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (
            BertModel,
-            BertLMHeadModel,
            BertForMaskedLM,
            BertForMultipleChoice,
            BertForNextSentencePrediction,
@@ -375,7 +373,6 @@ class BertModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
        if is_torch_available()
        else ()
    )
-    all_generative_model_classes = (BertLMHeadModel,) if is_torch_available() else ()

    def setUp(self):
        self.model_tester = BertModelTester(self)
@@ -20,7 +20,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


@@ -184,10 +183,9 @@ class BertGenerationEncoderTester:


@require_torch
-class BertGenerationEncoderTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class BertGenerationEncoderTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (BertGenerationEncoder, BertGenerationDecoder) if is_torch_available() else ()
-    all_generative_model_classes = (BertGenerationDecoder,) if is_torch_available() else ()

    def setUp(self):
        self.model_tester = BertGenerationEncoderTester(self)
@@ -147,7 +147,6 @@ class Blenderbot3BIntegrationTests(unittest.TestCase):

        src_text = ["Sam"]
        model_inputs = self.tokenizer(src_text, return_tensors="pt").to(torch_device)
-
        generated_utterances = model.generate(**model_inputs, **FASTER_GEN_KWARGS)
        tgt_text = 'Sam is a great name. It means "sun" in Gaelic.'

@@ -157,7 +156,6 @@ class Blenderbot3BIntegrationTests(unittest.TestCase):
        src_text = "Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like i'm going to throw up.\nand why is that?"

        model_inputs = self.tokenizer([src_text], return_tensors="pt").to(torch_device)
-
        generated_ids = model.generate(**model_inputs, **FASTER_GEN_KWARGS)[0]
        reply = self.tokenizer.decode(generated_ids, **TOK_DECODE_KW)

@@ -189,9 +187,6 @@ class Blenderbot90MIntegrationTests(unittest.TestCase):
        ]

        model_inputs = self.tokenizer(src_text, return_tensors="pt").to(torch_device)
-
-        # model does not have "token_type_ids"
-        model_inputs.pop("token_type_ids")
        assert isinstance(self.tokenizer, BlenderbotSmallTokenizer)
        generated_ids = self.model.generate(**model_inputs)[0]
        reply = self.tokenizer.decode(generated_ids, **TOK_DECODE_KW)
@@ -203,11 +198,10 @@ class Blenderbot90MIntegrationTests(unittest.TestCase):

    def test_90_generation_from_short_input(self):
        model_inputs = self.tokenizer(["sam"], return_tensors="pt").to(torch_device)
-
-        # model does not have "token_type_ids"
-        model_inputs.pop("token_type_ids")
        generated_utterances = self.model.generate(**model_inputs)
+        # generated_txt = self.tokenizer.decode(generated_utterances[0])

+        # assert generated_txt == "__start__ have you ever heard of sam harris? he's an american singer, songwriter, and actor. __end__"
        clean_txt = self.tokenizer.decode(generated_utterances[0], **TOK_DECODE_KW)
        assert clean_txt in (
            "have you ever been to a sam club? it's a great club in the south.",
@@ -44,6 +44,7 @@ if is_torch_available():
        BertModel,
        PretrainedConfig,
        PreTrainedModel,
+        top_k_top_p_filtering,
    )


@@ -881,6 +882,126 @@ class ModelTesterMixin:
            with torch.no_grad():
                model(**inputs)[0]

+    def test_lm_head_model_random_no_beam_search_generate(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+        input_ids = inputs_dict["input_ids"] if "input_ids" in inputs_dict else inputs_dict["inputs"]
+
+        # make sure that input_ids is at most of size 15
+        input_ids = input_ids[..., :15]
+
+        # iterate over all generative models
+        for model_class in self.all_generative_model_classes:
+            model = model_class(config).to(torch_device)
+            model.eval()
+
+            if config.bos_token_id is None:
+                # if bos token id is not defined, model needs input_ids
+                with self.assertRaises(AssertionError):
+                    model.generate(do_sample=True, max_length=5)
+                # num_return_sequences = 1
+                self._check_generated_ids(model.generate(input_ids, do_sample=True))
+            else:
+                # num_return_sequences = 1
+                self._check_generated_ids(model.generate(do_sample=True, max_length=5))
+
+            with self.assertRaises(AssertionError):
+                # generating multiple sequences when no beam search generation
+                # is not allowed as it would always generate the same sequences
+                model.generate(input_ids, do_sample=False, num_beams=1, num_return_sequences=2)
+
+            # num_return_sequences > 1, sample
+            self._check_generated_ids(model.generate(input_ids, do_sample=True, num_return_sequences=2))
+
+            # check bad words tokens language generation
+            # create list of 1-seq bad token and list of 2-seq of bad tokens
+            bad_words_ids = [
+                self._generate_random_bad_tokens(1, model.config),
+                self._generate_random_bad_tokens(2, model.config),
+            ]
+            output_tokens = model.generate(
+                input_ids, do_sample=True, bad_words_ids=bad_words_ids, num_return_sequences=2
+            )
+            # only count generated tokens
+            generated_ids = output_tokens[:, input_ids.shape[-1] :]
+            self.assertFalse(self._check_match_tokens(generated_ids.tolist(), bad_words_ids))
+
+    def test_lm_head_model_random_beam_search_generate(self):
+        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
+        input_ids = (inputs_dict["input_ids"] if "input_ids" in inputs_dict else inputs_dict["inputs"]).to(
+            torch_device
+        )
+
+        # make sure that input_ids is at most of size 15
+        input_ids = input_ids[..., :15]
+
+        for model_class in self.all_generative_model_classes:
+            model = model_class(config).to(torch_device)
+            model.eval()
+
+            if config.bos_token_id is None:
+                # if bos token id is not defined mobel needs input_ids, num_return_sequences = 1
+                self._check_generated_ids(model.generate(input_ids, do_sample=True, num_beams=2))
+            else:
+                # num_return_sequences = 1
+                self._check_generated_ids(model.generate(do_sample=True, max_length=5, num_beams=2))
+
+            with self.assertRaises(AssertionError):
+                # generating more sequences than having beams leads is not possible
+                model.generate(input_ids, do_sample=False, num_return_sequences=3, num_beams=2)
+
+            # num_return_sequences > 1, sample
+            self._check_generated_ids(
+                model.generate(
+                    input_ids,
+                    do_sample=True,
+                    num_beams=2,
+                    num_return_sequences=2,
+                )
+            )
+            # num_return_sequences > 1, greedy
+            self._check_generated_ids(model.generate(input_ids, do_sample=False, num_beams=2, num_return_sequences=2))
+
+            # check bad words tokens language generation
+            # create list of 1-seq bad token and list of 2-seq of bad tokens
+            bad_words_ids = [
+                self._generate_random_bad_tokens(1, model.config),
+                self._generate_random_bad_tokens(2, model.config),
+            ]
+            output_tokens = model.generate(
+                input_ids, do_sample=False, bad_words_ids=bad_words_ids, num_beams=2, num_return_sequences=2
+            )
+            # only count generated tokens
+            generated_ids = output_tokens[:, input_ids.shape[-1] :]
+            self.assertFalse(self._check_match_tokens(generated_ids.tolist(), bad_words_ids))
+
+    def _generate_random_bad_tokens(self, num_bad_tokens: int, config) -> List[int]:
+        # special tokens cannot be bad tokens
+        special_tokens = [x for x in [config.bos_token_id, config.eos_token_id, config.pad_token_id] if x is not None]
+        # create random bad tokens that are not special tokens
+        bad_tokens = []
+        while len(bad_tokens) < num_bad_tokens:
+            token = ids_tensor((1, 1), self.model_tester.vocab_size).squeeze(0).cpu().numpy()[0]
+            if token not in special_tokens:
+                bad_tokens.append(token)
+        return bad_tokens
+
+    def _check_generated_ids(self, output_ids):
+        for token_id in output_ids[0].tolist():
+            self.assertGreaterEqual(token_id, 0)
+            self.assertLess(token_id, self.model_tester.vocab_size)
+
+    def _check_match_tokens(self, generated_ids, bad_words_ids):
+        # for all bad word tokens
+        for bad_word_ids in bad_words_ids:
+            # for all slices in batch
+            for generated_ids_slice in generated_ids:
+                # for all word idx
+                for i in range(len(bad_word_ids), len(generated_ids_slice)):
+                    # if tokens match
+                    if generated_ids_slice[i - len(bad_word_ids) : i] == bad_word_ids:
+                        return True
+        return False
+
    @require_torch_multigpu
    def test_multigpu_data_parallel_forward(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
@@ -973,3 +1094,110 @@ class ModelUtilsTest(unittest.TestCase):
            model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True)
            self.assertEqual(model.config.output_hidden_states, True)
            self.assertEqual(model.config, config)
+
+
+@require_torch
+class UtilsFunctionsTest(unittest.TestCase):
+
+    # tests whether the top_k_top_p function behaves as expected
+    def test_top_k_top_p_filtering(self):
+        logits = torch.tensor(
+            [
+                [
+                    8.2220991,  # 3rd highest value; idx. 0
+                    -0.5620044,
+                    5.23229752,
+                    4.0386393,
+                    -6.8798378,
+                    -0.54785802,
+                    -3.2012153,
+                    2.92777176,
+                    1.88171953,
+                    7.35341276,  # 5th highest value; idx. 9
+                    8.43207833,  # 2nd highest value; idx. 10
+                    -9.85711836,
+                    -5.96209236,
+                    -1.13039161,
+                    -7.1115294,
+                    -0.8369633,
+                    -5.3186408,
+                    7.06427407,
+                    0.81369344,
+                    -0.82023817,
+                    -5.9179796,
+                    0.58813443,
+                    -6.99778438,
+                    4.71551189,
+                    -0.18771637,
+                    7.44020759,  # 4th highest value; idx. 25
+                    9.38450987,  # 1st highest value; idx. 26
+                    2.12662941,
+                    -9.32562038,
+                    2.35652522,
+                ],  # cumulative prob of 5 highest values <= 0.6
+                [
+                    0.58425518,
+                    4.53139238,
+                    -5.57510464,
+                    -6.28030699,
+                    -7.19529503,
+                    -4.02122551,
+                    1.39337037,
+                    -6.06707057,
+                    1.59480517,
+                    -9.643119,
+                    0.03907799,
+                    0.67231762,
+                    -8.88206726,
+                    6.27115922,  # 4th highest value; idx. 13
+                    2.28520723,
+                    4.82767506,
+                    4.30421368,
+                    8.8275313,  # 2nd highest value; idx. 17
+                    5.44029958,  # 5th highest value; idx. 18
+                    -4.4735794,
+                    7.38579536,  # 3rd highest value; idx. 20
+                    -2.91051663,
+                    2.61946077,
+                    -2.5674762,
+                    -9.48959302,
+                    -4.02922645,
+                    -1.35416918,
+                    9.67702323,  # 1st highest value; idx. 27
+                    -5.89478553,
+                    1.85370467,
+                ],  # cumulative prob of 5 highest values <= 0.6
+            ],
+            dtype=torch.float,
+            device=torch_device,
+        )
+
+        non_inf_expected_idx = torch.tensor(
+            [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]],
+            dtype=torch.long,
+            device=torch_device,
+        )  # expected non filtered idx as noted above
+
+        non_inf_expected_output = torch.tensor(
+            [
+                8.2221,
+                7.3534,
+                8.4321,
+                7.4402,
+                9.3845,
+                6.2712,
+                8.8275,
+                5.4403,
+                7.3858,
+                9.6770,
+            ],  # expected non filtered values as noted above
+            dtype=torch.float,
+            device=torch_device,
+        )
+
+        output = top_k_top_p_filtering(logits, top_k=10, top_p=0.6, min_tokens_to_keep=4)
+        non_inf_output = output[output != -float("inf")].to(device=torch_device)
+        non_inf_idx = (output != -float("inf")).nonzero().to(device=torch_device)
+
+        self.assertTrue(torch.allclose(non_inf_expected_output, non_inf_output, atol=1e-12))
+        self.assertTrue(torch.all(torch.eq(non_inf_expected_idx, non_inf_idx)))
@@ -19,7 +19,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask


@@ -152,7 +151,7 @@ class CTRLModelTester:


@require_torch
-class CTRLModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class CTRLModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (CTRLModel, CTRLLMHeadModel) if is_torch_available() else ()
    all_generative_model_classes = (CTRLLMHeadModel,) if is_torch_available() else ()
@@ -24,7 +24,6 @@ from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, ids_tensor


@@ -121,7 +120,7 @@ def prepare_fsmt_inputs_dict(


@require_torch
-class FSMTModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class FSMTModelTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (FSMTModel, FSMTForConditionalGeneration) if is_torch_available() else ()
    all_generative_model_classes = (FSMTForConditionalGeneration,) if is_torch_available() else ()
    is_encoder_decoder = True
@@ -20,7 +20,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


@@ -378,7 +377,7 @@ class GPT2ModelTester:


@require_torch
-class GPT2ModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class GPT2ModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel, GPT2ForSequenceClassification)
@@ -511,17 +510,32 @@ class GPT2ModelLanguageGenerationTest(unittest.TestCase):
            self.assertListEqual(output_ids[0].tolist(), expected_output_ids)

    @slow
-    def test_gpt2_sample(self):
-        tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
-        model = GPT2LMHeadModel.from_pretrained("gpt2")
+    def test_lm_generate_distilgpt2(self):
+        model = GPT2LMHeadModel.from_pretrained("distilgpt2")
        model.to(torch_device)
+        input_ids = torch.tensor([[464, 1893]], dtype=torch.long, device=torch_device)  # The president
+        expected_output_ids = [
+            464,
+            1893,
+            286,
+            262,
+            1578,
+            1829,
+            11,
+            290,
+            262,
+            1893,
+            286,
+            262,
+            1578,
+            7526,
+            11,
+            423,
+            587,
+            287,
+            262,
+            2635,
+        ]  # The president of the United States, and the president of the United Kingdom, have been in the White

-        torch.manual_seed(0)
-        input_ids = tokenizer("Today is a nice day and", return_tensors="pt").input_ids.to(torch_device)
-        output_ids = model.generate(input_ids, do_sample=True)
-        output_str = tokenizer.decode(output_ids[0], skip_special_tokens=True)
-
-        EXPECTED_OUTPUT_STR = (
-            "Today is a nice day and if you don't know anything about the state of play during your holiday"
-        )
-        self.assertEqual(output_str, EXPECTED_OUTPUT_STR)
+        output_ids = model.generate(input_ids, do_sample=False)
+        self.assertListEqual(output_ids[0].tolist(), expected_output_ids)
@@ -20,7 +20,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, ids_tensor


@@ -171,7 +170,7 @@ class OpenAIGPTModelTester:


@require_torch
-class OpenAIGPTModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class OpenAIGPTModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
@@ -22,7 +22,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor


@@ -854,7 +853,7 @@ class ProphetNetStandaloneEncoderModelTester:


@require_torch
-class ProphetNetModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class ProphetNetModelTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (ProphetNetModel, ProphetNetForConditionalGeneration) if is_torch_available() else ()
    all_generative_model_classes = (ProphetNetForConditionalGeneration,) if is_torch_available() else ()
    test_pruning = False
@@ -918,7 +917,7 @@ class ProphetNetModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.Test


@require_torch
-class ProphetNetStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class ProphetNetStandaloneDecoderModelTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (ProphetNetDecoder, ProphetNetForCausalLM) if is_torch_available() else ()
    all_generative_model_classes = (ProphetNetForCausalLM,) if is_torch_available() else ()
    test_pruning = False
@@ -26,7 +26,6 @@ from transformers.testing_utils import (
 )

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


@@ -197,14 +196,11 @@ class ReformerModelTester:
        )

    def create_and_check_reformer_model_with_lm_backward(self, config, input_ids, input_mask, choice_labels):
-        if not self.is_training:
-            return
-
        config.is_decoder = False
        config.lsh_num_chunks_after = 1
        model = ReformerForMaskedLM(config=config)
        model.to(torch_device)
-        model.train()
+        model.eval()
        loss = model(input_ids, attention_mask=input_mask, labels=input_ids)["loss"]
        loss.backward()

@@ -573,7 +569,7 @@ class ReformerTesterMixin:


@require_torch
-class ReformerLocalAttnModelTest(ReformerTesterMixin, GenerationTesterMixin, ModelTesterMixin, unittest.TestCase):
+class ReformerLocalAttnModelTest(ReformerTesterMixin, ModelTesterMixin, unittest.TestCase):
    all_model_classes = (
        (ReformerModel, ReformerModelWithLMHead, ReformerForSequenceClassification, ReformerForQuestionAnswering)
        if is_torch_available()
@@ -633,7 +629,7 @@ class ReformerLocalAttnModelTest(ReformerTesterMixin, GenerationTesterMixin, Mod


@require_torch
-class ReformerLSHAttnModelTest(ReformerTesterMixin, ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class ReformerLSHAttnModelTest(ReformerTesterMixin, ModelTesterMixin, unittest.TestCase):
    all_model_classes = (
        (ReformerModel, ReformerModelWithLMHead, ReformerForSequenceClassification, ReformerForQuestionAnswering)
        if is_torch_available()
@@ -20,7 +20,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


@@ -268,7 +267,7 @@ class RobertaModelTester:


@require_torch
-class RobertaModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class RobertaModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (
@@ -283,7 +282,6 @@ class RobertaModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCas
        if is_torch_available()
        else ()
    )
-    all_generative_model_classes = (RobertaForCausalLM,) if is_torch_available() else ()

    def setUp(self):
        self.model_tester = RobertaModelTester(self)
@@ -23,7 +23,6 @@ from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, ids_tensor


@@ -467,7 +466,7 @@ class T5ModelTester:


@require_torch
-class T5ModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class T5ModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (T5Model, T5ForConditionalGeneration) if is_torch_available() else ()
    all_generative_model_classes = (T5ForConditionalGeneration,) if is_torch_available() else ()
@@ -593,7 +592,6 @@ class T5ModelIntegrationTests(unittest.TestCase):
            do_sample=False,
            early_stopping=True,
        )
-
        decoded = tok.batch_decode(hypotheses_batch, skip_special_tokens=True, clean_up_tokenization_spaces=False)
        self.assertListEqual(
            expected_summaries,
@@ -20,7 +20,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, require_torch_multigpu, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, ids_tensor


@@ -157,7 +156,7 @@ class TransfoXLModelTester:


@require_torch
-class TransfoXLModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class TransfoXLModelTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (TransfoXLModel, TransfoXLLMHeadModel) if is_torch_available() else ()
    all_generative_model_classes = (TransfoXLLMHeadModel,) if is_torch_available() else ()
    test_pruning = False
@@ -20,7 +20,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask


@@ -332,7 +331,7 @@ class XLMModelTester:


@require_torch
-class XLMModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class XLMModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (
@@ -21,7 +21,6 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device

 from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
 from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask


@@ -480,7 +479,7 @@ class XLNetModelTester:


@require_torch
-class XLNetModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
+class XLNetModelTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (
        (
            XLNetModel,
@@ -1,6 +1,6 @@
 import unittest

-from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, Conversation, ConversationalPipeline, pipeline
+from transformers import Conversation, pipeline
 from transformers.testing_utils import require_torch, slow, torch_device

 from .test_pipelines_common import MonoInputPipelineCommonMixin
@@ -9,31 +9,26 @@ from .test_pipelines_common import MonoInputPipelineCommonMixin
 DEFAULT_DEVICE_NUM = -1 if torch_device == "cpu" else 0


-class ConversationalPipelineTests(MonoInputPipelineCommonMixin, unittest.TestCase):
+class TextGenerationPipelineTests(MonoInputPipelineCommonMixin, unittest.TestCase):
    pipeline_task = "conversational"
    small_models = []  # Models tested without the @slow decorator
    large_models = ["microsoft/DialoGPT-medium"]  # Models tested with the @slow decorator
+    valid_inputs = [Conversation("Hi there!"), [Conversation("Hi there!"), Conversation("How are you?")]]
    invalid_inputs = ["Hi there!", Conversation()]

    def _test_pipeline(
        self, nlp
-    ):  # override the default test method to check that the output is a `Conversation` object
+    ):  # e overide the default test method to check that the output is a `Conversation` object
        self.assertIsNotNone(nlp)

-        # We need to recreate conversation for successive tests to pass as
-        # Conversation objects get *consumed* by the pipeline
-        conversation = Conversation("Hi there!")
-        mono_result = nlp(conversation)
+        mono_result = nlp(self.valid_inputs[0])
        self.assertIsInstance(mono_result, Conversation)

-        conversations = [Conversation("Hi there!"), Conversation("How are you?")]
-        multi_result = nlp(conversations)
+        multi_result = nlp(self.valid_inputs[1])
        self.assertIsInstance(multi_result, list)
        self.assertIsInstance(multi_result[0], Conversation)
-        # Conversation have been consumed and are not valid anymore
        # Inactive conversations passed to the pipeline raise a ValueError
-        self.assertRaises(ValueError, nlp, conversation)
-        self.assertRaises(ValueError, nlp, conversations)
+        self.assertRaises(ValueError, nlp, self.valid_inputs[1])

        for bad_input in self.invalid_inputs:
            self.assertRaises(Exception, nlp, bad_input)
@@ -96,50 +91,3 @@ class ConversationalPipelineTests(MonoInputPipelineCommonMixin, unittest.TestCas
        self.assertEqual(len(result.generated_responses), 2)
        self.assertEqual(result.past_user_inputs[1], "Is it an action movie?")
        self.assertEqual(result.generated_responses[1], "It's a comedy.")
-
-    @require_torch
-    @slow
-    def test_integration_torch_conversation_encoder_decoder(self):
-        # When
-        tokenizer = AutoTokenizer.from_pretrained("facebook/blenderbot-90M")
-        model = AutoModelForSeq2SeqLM.from_pretrained("facebook/blenderbot-90M")
-        nlp = ConversationalPipeline(model=model, tokenizer=tokenizer, device=DEFAULT_DEVICE_NUM)
-
-        conversation_1 = Conversation("My name is Sarah and I live in London")
-        conversation_2 = Conversation("Going to the movies tonight, What movie would you recommend? ")
-        # Then
-        self.assertEqual(len(conversation_1.past_user_inputs), 0)
-        self.assertEqual(len(conversation_2.past_user_inputs), 0)
-        # When
-        result = nlp([conversation_1, conversation_2], do_sample=False, max_length=1000)
-        # Then
-        self.assertEqual(result, [conversation_1, conversation_2])
-        self.assertEqual(len(result[0].past_user_inputs), 1)
-        self.assertEqual(len(result[1].past_user_inputs), 1)
-        self.assertEqual(len(result[0].generated_responses), 1)
-        self.assertEqual(len(result[1].generated_responses), 1)
-        self.assertEqual(result[0].past_user_inputs[0], "My name is Sarah and I live in London")
-        self.assertEqual(
-            result[0].generated_responses[0],
-            "hi sarah, i live in london as well. do you have any plans for the weekend?",
-        )
-        self.assertEqual(
-            result[1].past_user_inputs[0], "Going to the movies tonight, What movie would you recommend? "
-        )
-        self.assertEqual(
-            result[1].generated_responses[0], "i don't know... i'm not really sure. what movie are you going to see?"
-        )
-        # When
-        conversation_1.add_user_input("Not yet, what about you?")
-        conversation_2.add_user_input("What's your name?")
-        result = nlp([conversation_1, conversation_2], do_sample=False, max_length=1000)
-        # Then
-        self.assertEqual(result, [conversation_1, conversation_2])
-        self.assertEqual(len(result[0].past_user_inputs), 2)
-        self.assertEqual(len(result[1].past_user_inputs), 2)
-        self.assertEqual(len(result[0].generated_responses), 2)
-        self.assertEqual(len(result[1].generated_responses), 2)
-        self.assertEqual(result[0].past_user_inputs[1], "Not yet, what about you?")
-        self.assertEqual(result[0].generated_responses[1], "i don't have any plans yet. i'm not sure what to do yet.")
-        self.assertEqual(result[1].past_user_inputs[1], "What's your name?")
-        self.assertEqual(result[1].generated_responses[1], "i don't have a name, but i'm going to see a horror movie.")
@@ -1,8 +1,8 @@
 import unittest

-from transformers import AutoTokenizer, pipeline
+from transformers import pipeline
 from transformers.pipelines import Pipeline
-from transformers.testing_utils import require_tf, require_torch
+from transformers.testing_utils import require_tf

 from .test_pipelines_common import CustomInputPipelineCommonMixin

@@ -19,54 +19,38 @@ class NerPipelineTests(CustomInputPipelineCommonMixin, unittest.TestCase):

    def _test_pipeline(self, nlp: Pipeline):
        output_keys = {"entity", "word", "score"}
-        if nlp.grouped_entities:
-            output_keys = {"entity_group", "word", "score"}

        ungrouped_ner_inputs = [
            [
-                {"entity": "B-PER", "index": 1, "score": 0.9994944930076599, "is_subword": False, "word": "Cons"},
-                {"entity": "B-PER", "index": 2, "score": 0.8025449514389038, "is_subword": True, "word": "##uelo"},
-                {"entity": "I-PER", "index": 3, "score": 0.9993102550506592, "is_subword": False, "word": "Ara"},
-                {"entity": "I-PER", "index": 4, "score": 0.9993743896484375, "is_subword": True, "word": "##új"},
-                {"entity": "I-PER", "index": 5, "score": 0.9992871880531311, "is_subword": True, "word": "##o"},
-                {"entity": "I-PER", "index": 6, "score": 0.9993029236793518, "is_subword": False, "word": "No"},
-                {"entity": "I-PER", "index": 7, "score": 0.9981776475906372, "is_subword": True, "word": "##guera"},
-                {"entity": "B-PER", "index": 15, "score": 0.9998136162757874, "is_subword": False, "word": "Andrés"},
-                {"entity": "I-PER", "index": 16, "score": 0.999740719795227, "is_subword": False, "word": "Pas"},
-                {"entity": "I-PER", "index": 17, "score": 0.9997414350509644, "is_subword": True, "word": "##tran"},
-                {"entity": "I-PER", "index": 18, "score": 0.9996136426925659, "is_subword": True, "word": "##a"},
-                {"entity": "B-ORG", "index": 28, "score": 0.9989739060401917, "is_subword": False, "word": "Far"},
-                {"entity": "I-ORG", "index": 29, "score": 0.7188422083854675, "is_subword": True, "word": "##c"},
+                {"entity": "B-PER", "index": 1, "score": 0.9994944930076599, "word": "Cons"},
+                {"entity": "B-PER", "index": 2, "score": 0.8025449514389038, "word": "##uelo"},
+                {"entity": "I-PER", "index": 3, "score": 0.9993102550506592, "word": "Ara"},
+                {"entity": "I-PER", "index": 4, "score": 0.9993743896484375, "word": "##új"},
+                {"entity": "I-PER", "index": 5, "score": 0.9992871880531311, "word": "##o"},
+                {"entity": "I-PER", "index": 6, "score": 0.9993029236793518, "word": "No"},
+                {"entity": "I-PER", "index": 7, "score": 0.9981776475906372, "word": "##guera"},
+                {"entity": "B-PER", "index": 15, "score": 0.9998136162757874, "word": "Andrés"},
+                {"entity": "I-PER", "index": 16, "score": 0.999740719795227, "word": "Pas"},
+                {"entity": "I-PER", "index": 17, "score": 0.9997414350509644, "word": "##tran"},
+                {"entity": "I-PER", "index": 18, "score": 0.9996136426925659, "word": "##a"},
+                {"entity": "B-ORG", "index": 28, "score": 0.9989739060401917, "word": "Far"},
+                {"entity": "I-ORG", "index": 29, "score": 0.7188422083854675, "word": "##c"},
            ],
            [
-                {"entity": "I-PER", "index": 1, "score": 0.9968166351318359, "is_subword": False, "word": "En"},
-                {"entity": "I-PER", "index": 2, "score": 0.9957635998725891, "is_subword": True, "word": "##zo"},
-                {"entity": "I-ORG", "index": 7, "score": 0.9986497163772583, "is_subword": False, "word": "UN"},
+                {"entity": "I-PER", "index": 1, "score": 0.9968166351318359, "word": "En"},
+                {"entity": "I-PER", "index": 2, "score": 0.9957635998725891, "word": "##zo"},
+                {"entity": "I-ORG", "index": 7, "score": 0.9986497163772583, "word": "UN"},
            ],
        ]
-
        expected_grouped_ner_results = [
            [
-                {"entity_group": "PER", "score": 0.999369223912557, "word": "Consuelo Araújo Noguera"},
-                {"entity_group": "PER", "score": 0.9997771680355072, "word": "Andrés Pastrana"},
-                {"entity_group": "ORG", "score": 0.9989739060401917, "word": "Farc"},
+                {"entity_group": "B-PER", "score": 0.9710702640669686, "word": "Consuelo Araújo Noguera"},
+                {"entity_group": "B-PER", "score": 0.9997273534536362, "word": "Andrés Pastrana"},
+                {"entity_group": "B-ORG", "score": 0.8589080572128296, "word": "Farc"},
            ],
            [
-                {"entity_group": "PER", "score": 0.9968166351318359, "word": "Enzo"},
-                {"entity_group": "ORG", "score": 0.9986497163772583, "word": "UN"},
-            ],
-        ]
-
-        expected_grouped_ner_results_w_subword = [
-            [
-                {"entity_group": "PER", "score": 0.9994944930076599, "word": "Cons"},
-                {"entity_group": "PER", "score": 0.9663328925768534, "word": "##uelo Araújo Noguera"},
-                {"entity_group": "PER", "score": 0.9997273534536362, "word": "Andrés Pastrana"},
-                {"entity_group": "ORG", "score": 0.8589080572128296, "word": "Farc"},
-            ],
-            [
-                {"entity_group": "PER", "score": 0.9962901175022125, "word": "Enzo"},
-                {"entity_group": "ORG", "score": 0.9986497163772583, "word": "UN"},
+                {"entity_group": "I-PER", "score": 0.9962901175022125, "word": "Enzo"},
+                {"entity_group": "I-ORG", "score": 0.9986497163772583, "word": "UN"},
            ],
        ]

@@ -93,80 +77,12 @@ class NerPipelineTests(CustomInputPipelineCommonMixin, unittest.TestCase):
            for key in output_keys:
                self.assertIn(key, result)

-        if nlp.grouped_entities:
-            if nlp.ignore_subwords:
-                for ungrouped_input, grouped_result in zip(ungrouped_ner_inputs, expected_grouped_ner_results):
-                    self.assertEqual(nlp.group_entities(ungrouped_input), grouped_result)
-            else:
-                for ungrouped_input, grouped_result in zip(
-                    ungrouped_ner_inputs, expected_grouped_ner_results_w_subword
-                ):
-                    self.assertEqual(nlp.group_entities(ungrouped_input), grouped_result)
+        for ungrouped_input, grouped_result in zip(ungrouped_ner_inputs, expected_grouped_ner_results):
+            self.assertEqual(nlp.group_entities(ungrouped_input), grouped_result)

    @require_tf
    def test_tf_only(self):
        model_name = "Narsil/small"  # This model only has a TensorFlow version
        # We test that if we don't specificy framework='tf', it gets detected automatically
-        tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-        nlp = pipeline(task="ner", model=model_name, tokenizer=tokenizer)
+        nlp = pipeline(task="ner", model=model_name, tokenizer=model_name)
        self._test_pipeline(nlp)
-
-    #         offset=tokenizer(VALID_INPUTS[0],return_offsets_mapping=True)['offset_mapping']
-    #         pipeline_running_kwargs = {"offset_mapping"}  # Additional kwargs to run the pipeline with
-
-    @require_tf
-    def test_tf_defaults(self):
-        for model_name in self.small_models:
-            tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-            nlp = pipeline(task="ner", model=model_name, tokenizer=tokenizer, framework="tf")
-        self._test_pipeline(nlp)
-
-    @require_tf
-    def test_tf_small(self):
-        for model_name in self.small_models:
-            print(model_name)
-            tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-            nlp = pipeline(
-                task="ner",
-                model=model_name,
-                tokenizer=tokenizer,
-                framework="tf",
-                grouped_entities=True,
-                ignore_subwords=True,
-            )
-            self._test_pipeline(nlp)
-
-            for model_name in self.small_models:
-                tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-                nlp = pipeline(
-                    task="ner",
-                    model=model_name,
-                    tokenizer=tokenizer,
-                    framework="tf",
-                    grouped_entities=True,
-                    ignore_subwords=False,
-                )
-                self._test_pipeline(nlp)
-
-    @require_torch
-    def test_pt_defaults(self):
-        for model_name in self.small_models:
-            tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-            nlp = pipeline(task="ner", model=model_name, tokenizer=tokenizer)
-            self._test_pipeline(nlp)
-
-    @require_torch
-    def test_torch_small(self):
-        for model_name in self.small_models:
-            tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-            nlp = pipeline(
-                task="ner", model=model_name, tokenizer=tokenizer, grouped_entities=True, ignore_subwords=True
-            )
-            self._test_pipeline(nlp)
-
-        for model_name in self.small_models:
-            tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
-            nlp = pipeline(
-                task="ner", model=model_name, tokenizer=tokenizer, grouped_entities=True, ignore_subwords=False
-            )
-            self._test_pipeline(nlp)
@@ -1,7 +1,6 @@
 import unittest

-from transformers.data.processors.squad import SquadExample
-from transformers.pipelines import Pipeline, QuestionAnsweringArgumentHandler
+from transformers.pipelines import Pipeline

 from .test_pipelines_common import CustomInputPipelineCommonMixin

@@ -44,116 +43,5 @@ class QAPipelineTests(CustomInputPipelineCommonMixin, unittest.TestCase):
            for key in output_keys:
                self.assertIn(key, result)
        for bad_input in invalid_inputs:
-            self.assertRaises(ValueError, nlp, bad_input)
-        self.assertRaises(ValueError, nlp, invalid_inputs)
-
-    def test_argument_handler(self):
-        qa = QuestionAnsweringArgumentHandler()
-
-        Q = "Where was HuggingFace founded ?"
-        C = "HuggingFace was founded in Paris"
-
-        normalized = qa(Q, C)
-        self.assertEqual(type(normalized), list)
-        self.assertEqual(len(normalized), 1)
-        self.assertEqual({type(el) for el in normalized}, {SquadExample})
-
-        normalized = qa(question=Q, context=C)
-        self.assertEqual(type(normalized), list)
-        self.assertEqual(len(normalized), 1)
-        self.assertEqual({type(el) for el in normalized}, {SquadExample})
-
-        normalized = qa(question=Q, context=C)
-        self.assertEqual(type(normalized), list)
-        self.assertEqual(len(normalized), 1)
-        self.assertEqual({type(el) for el in normalized}, {SquadExample})
-
-        normalized = qa({"question": Q, "context": C})
-        self.assertEqual(type(normalized), list)
-        self.assertEqual(len(normalized), 1)
-        self.assertEqual({type(el) for el in normalized}, {SquadExample})
-
-        normalized = qa([{"question": Q, "context": C}])
-        self.assertEqual(type(normalized), list)
-        self.assertEqual(len(normalized), 1)
-        self.assertEqual({type(el) for el in normalized}, {SquadExample})
-
-        normalized = qa([{"question": Q, "context": C}, {"question": Q, "context": C}])
-        self.assertEqual(type(normalized), list)
-        self.assertEqual(len(normalized), 2)
-        self.assertEqual({type(el) for el in normalized}, {SquadExample})
-
-        normalized = qa(X={"question": Q, "context": C})
-        self.assertEqual(type(normalized), list)
-        self.assertEqual(len(normalized), 1)
-        self.assertEqual({type(el) for el in normalized}, {SquadExample})
-
-        normalized = qa(X=[{"question": Q, "context": C}])
-        self.assertEqual(type(normalized), list)
-        self.assertEqual(len(normalized), 1)
-        self.assertEqual({type(el) for el in normalized}, {SquadExample})
-
-        normalized = qa(data={"question": Q, "context": C})
-        self.assertEqual(type(normalized), list)
-        self.assertEqual(len(normalized), 1)
-        self.assertEqual({type(el) for el in normalized}, {SquadExample})
-
-    def test_argument_handler_error_handling(self):
-        qa = QuestionAnsweringArgumentHandler()
-
-        Q = "Where was HuggingFace founded ?"
-        C = "HuggingFace was founded in Paris"
-
-        with self.assertRaises(KeyError):
-            qa({"context": C})
-        with self.assertRaises(KeyError):
-            qa({"question": Q})
-        with self.assertRaises(KeyError):
-            qa([{"context": C}])
-        with self.assertRaises(ValueError):
-            qa(None, C)
-        with self.assertRaises(ValueError):
-            qa("", C)
-        with self.assertRaises(ValueError):
-            qa(Q, None)
-        with self.assertRaises(ValueError):
-            qa(Q, "")
-
-        with self.assertRaises(ValueError):
-            qa(question=None, context=C)
-        with self.assertRaises(ValueError):
-            qa(question="", context=C)
-        with self.assertRaises(ValueError):
-            qa(question=Q, context=None)
-        with self.assertRaises(ValueError):
-            qa(question=Q, context="")
-
-        with self.assertRaises(ValueError):
-            qa({"question": None, "context": C})
-        with self.assertRaises(ValueError):
-            qa({"question": "", "context": C})
-        with self.assertRaises(ValueError):
-            qa({"question": Q, "context": None})
-        with self.assertRaises(ValueError):
-            qa({"question": Q, "context": ""})
-
-        with self.assertRaises(ValueError):
-            qa([{"question": Q, "context": C}, {"question": None, "context": C}])
-        with self.assertRaises(ValueError):
-            qa([{"question": Q, "context": C}, {"question": "", "context": C}])
-
-        with self.assertRaises(ValueError):
-            qa([{"question": Q, "context": C}, {"question": Q, "context": None}])
-        with self.assertRaises(ValueError):
-            qa([{"question": Q, "context": C}, {"question": Q, "context": ""}])
-
-    def test_argument_handler_error_handling_odd(self):
-        qa = QuestionAnsweringArgumentHandler()
-        with self.assertRaises(ValueError):
-            qa(None)
-
-        with self.assertRaises(ValueError):
-            qa(Y=None)
-
-        with self.assertRaises(ValueError):
-            qa(1)
+            self.assertRaises(Exception, nlp, bad_input)
+        self.assertRaises(Exception, nlp, invalid_inputs)
@@ -1,7 +1,5 @@
 import unittest

-from transformers import pipeline
-
 from .test_pipelines_common import MonoInputPipelineCommonMixin


@@ -10,20 +8,3 @@ class TextGenerationPipelineTests(MonoInputPipelineCommonMixin, unittest.TestCas
    pipeline_running_kwargs = {"prefix": "This is "}
    small_models = ["sshleifer/tiny-ctrl"]  # Models tested without the @slow decorator
    large_models = []  # Models tested with the @slow decorator
-
-    def test_simple_generation(self):
-        nlp = pipeline(task="text-generation", model=self.small_models[0])
-        # text-generation is non-deterministic by nature, we can't fully test the output
-
-        outputs = nlp("This is a test")
-
-        self.assertEqual(len(outputs), 1)
-        self.assertEqual(list(outputs[0].keys()), ["generated_text"])
-        self.assertEqual(type(outputs[0]["generated_text"]), str)
-
-        outputs = nlp(["This is a test", "This is a second test"])
-        self.assertEqual(len(outputs[0]), 1)
-        self.assertEqual(list(outputs[0][0].keys()), ["generated_text"])
-        self.assertEqual(type(outputs[0][0]["generated_text"]), str)
-        self.assertEqual(list(outputs[1][0].keys()), ["generated_text"])
-        self.assertEqual(type(outputs[1][0]["generated_text"]), str)
@@ -18,13 +18,13 @@ import os
 import tempfile
 import unittest

+import datasets
 import numpy as np

 from transformers import AutoTokenizer, EvaluationStrategy, PretrainedConfig, TrainingArguments, is_torch_available
 from transformers.file_utils import WEIGHTS_NAME
 from transformers.testing_utils import (
    get_tests_dir,
-    require_datasets,
    require_optuna,
    require_sentencepiece,
    require_tokenizers,
@@ -340,10 +340,7 @@ class TrainerIntegrationTest(unittest.TestCase):
        self.assertTrue(np.array_equal(labels[0], trainer.eval_dataset.ys[0]))
        self.assertTrue(np.array_equal(labels[1], trainer.eval_dataset.ys[1]))

-    @require_datasets
    def test_trainer_with_datasets(self):
-        import datasets
-
        np.random.seed(42)
        x = np.random.normal(size=(64,)).astype(np.float32)
        y = 2.0 * x + 3.0 + np.random.normal(scale=0.1, size=(64,))
@@ -661,17 +658,15 @@ class TrainerHyperParameterIntegrationTest(unittest.TestCase):
        def hp_name(trial):
            return MyTrialShortNamer.shortname(trial.params)

-        with tempfile.TemporaryDirectory() as tmp_dir:
-            trainer = get_regression_trainer(
-                output_dir=tmp_dir,
-                learning_rate=0.1,
-                logging_steps=1,
-                evaluation_strategy=EvaluationStrategy.EPOCH,
-                num_train_epochs=4,
-                disable_tqdm=True,
-                load_best_model_at_end=True,
-                logging_dir="runs",
-                run_name="test",
-                model_init=model_init,
-            )
-            trainer.hyperparameter_search(direction="minimize", hp_space=hp_space, hp_name=hp_name, n_trials=4)
+        trainer = get_regression_trainer(
+            learning_rate=0.1,
+            logging_steps=1,
+            evaluation_strategy=EvaluationStrategy.EPOCH,
+            num_train_epochs=4,
+            disable_tqdm=True,
+            load_best_model_at_end=True,
+            logging_dir="runs",
+            run_name="test",
+            model_init=model_init,
+        )
+        trainer.hyperparameter_search(direction="minimize", hp_space=hp_space, hp_name=hp_name, n_trials=4)
Author	SHA1	Message	Date
TevenLeScao	9909d971c1	style	2020-11-09 13:52:39 +01:00
TevenLeScao	916f302ba1	using Mixin	2020-11-02 16:28:08 +01:00
TevenLeScao	eeb0604261	removed self.parameters() call in TransfoXL	2020-11-02 16:17:18 +01:00
TevenLeScao	7d108a92f2	removing next call to fix StopIteration error	2020-11-02 14:04:28 +01:00