Use the recommended nonzero(..., as_tuple=False) overload.

Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>

.circleci/config.yml

@@ -281,7 +281,7 @@ jobs:
                       - v0.4-build_doc-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[tf,torch,sentencepiece,docs]
+            - run: pip install ."[all, docs]"
             - save_cache:
                   key: v0.4-build_doc-{{ checksum "setup.py" }}
                   paths:
@@ -303,7 +303,7 @@ jobs:
                   keys:
                       - v0.4-deploy_doc-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
-            - run: pip install .[tf,torch,sentencepiece,docs]
+            - run: pip install ."[all,docs]"
             - save_cache:
                   key: v0.4-deploy_doc-{{ checksum "setup.py" }}
                   paths:

.circleci/deploy.sh

@@ -51,4 +51,5 @@ deploy_doc "7fb8bdf" v3.0.2
 deploy_doc "4b3ee9c" v3.1.0
 deploy_doc "3ebb1b3" v3.2.0
 deploy_doc "0613f05" v3.3.1
-deploy_doc "eb0e0ce" # v3.4.0 Latest stable release
+deploy_doc "eb0e0ce" v3.4.0
+deploy_doc "818878d" # v3.5.1 Latest stable release

.github/ISSUE_TEMPLATE/bug-report.md

@@ -30,20 +30,22 @@ assignees: ''
  Trainer: @sgugger
  Speed and Memory Benchmarks: @patrickvonplaten
  Model Cards: @julien-c
- Translation: @sshleifer
- Summarization: @sshleifer
  TextGeneration: @TevenLeScao 
  examples/distillation: @VictorSanh
  nlp datasets: [different repo](https://github.com/huggingface/nlp)
  rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)
- Text Generation: @TevenLeScao
+ Text Generation: @patrickvonplaten @TevenLeScao
- blenderbot: @mariamabarham
+ Blenderbot: @patrickvonplaten
- Bart: @sshleifer
+ Bart: @patrickvonplaten
- Marian: @sshleifer
+ Marian: @patrickvonplaten
+ Pegasus: @patrickvonplaten
+ mBART: @patrickvonplaten
  T5: @patrickvonplaten
  Longformer/Reformer: @patrickvonplaten
- TransfoXL/XLNet: @TevenLeScao 
+ TransfoXL/XLNet: @TevenLeScao
- examples/seq2seq: @sshleifer
+ RAG: @patrickvonplaten, @lhoestq
+ FSMT: @stas00
+ examples/seq2seq: @patil-suraj
  examples/bert-loses-patience: @JetRunner
  tensorflow: @jplu
  examples/token-classification: @stefan-it

.github/ISSUE_TEMPLATE/question-help.md

@@ -1,6 +1,6 @@
 ---
 name: "❓ Questions & Help"
-about: Post your general questions on the Hugging Face forum or Stack Overflow tagged huggingface-transformers
+about: Post your general questions on the Hugging Face forum: https://discuss.huggingface.co/
 title: ''
 labels: ''
 assignees: ''
@@ -10,18 +10,17 @@ assignees: ''
 # ❓ Questions & Help
 
 <!-- The GitHub issue tracker is primarly intended for bugs, feature requests,
-     new models and benchmarks, and migration questions. For all other questions,
+     new models, benchmarks, and migration questions. For all other questions,
      we direct you to the Hugging Face forum: https://discuss.huggingface.co/ .
-     You can also try Stack Overflow (SO) where a whole community of PyTorch and
-     Tensorflow enthusiast can help you out. In this case, make sure to tag your
-     question with the right deep learning framework as well as the
-     huggingface-transformers tag: 
-     https://stackoverflow.com/questions/tagged/huggingface-transformers 
      -->
 
 ## Details
+
 <!-- Description of your issue -->
 
-<!-- You should first ask your question on the forum or SO, and only if
+<!-- You should first ask your question on the forum, and only if
-     you didn't get an answer ask it here on GitHub. -->
+     you didn't get an answer after a few days ask it here on GitHub. -->
-**A link to original question on the forum/Stack Overflow**:
+
+**A link to original question on the forum**:
+
+<!-- Your issue will be closed if you don't fill this part. -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -43,21 +43,20 @@ members/contributors which may be interested in your PR.
  Trainer: @sgugger
  Benchmarks: @patrickvonplaten
  Model Cards: @julien-c
- Translation: @sshleifer
- Summarization: @sshleifer
  examples/distillation: @VictorSanh
  nlp datasets: [different repo](https://github.com/huggingface/nlp)
  rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)
  Text Generation: @patrickvonplaten, @TevenLeScao
- Blenderbot, Bart, Marian, Pegasus: @sshleifer
+ Blenderbot, Bart, Marian, Pegasus: @patrickvonplaten
  T5: @patrickvonplaten
  Rag: @patrickvonplaten, @lhoestq
  EncoderDecoder: @patrickvonplaten
  Longformer, Reformer: @patrickvonplaten
  TransfoXL, XLNet: @TevenLeScao, @patrickvonplaten
- examples/seq2seq: @sshleifer
+ examples/seq2seq: @patil-suraj
  examples/bert-loses-patience: @JetRunner
  tensorflow: @jplu
  examples/token-classification: @stefan-it
  documentation: @sgugger
+ FSTM: @stas00
  -->

.github/workflows/self-push.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - master
+      - model-templates
     paths:
       - "src/**"
       - "tests/**"
       - ".github/**"
+      - "templates/**"
   # pull_request:
   repository_dispatch:
 
@@ -55,6 +57,14 @@ jobs:
           python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
           python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
 
+      - name: Create model files
+        run: |
+          source .env/bin/activate
+          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
+          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
+          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
+          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
+
       - name: Run all non-slow tests on GPU
         env:
           OMP_NUM_THREADS: 1
@@ -116,6 +126,14 @@ jobs:
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
 
+      - name: Create model files
+        run: |
+          source .env/bin/activate
+          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
+          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
+          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
+          transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
+
       - name: Run all non-slow tests on GPU
         env:
           OMP_NUM_THREADS: 1
@@ -135,7 +153,7 @@ jobs:
           name: run_all_tests_tf_gpu_test_reports
           path: reports
 
-  run_tests_torch_multiple_gpu:
+  run_tests_torch_multi_gpu:
     runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
@@ -154,7 +172,7 @@ jobs:
         id: cache
         with:
           path: .env
-          key: v1.1-tests_torch_multiple_gpu-${{ hashFiles('setup.py') }}
+          key: v1.1-tests_torch_multi_gpu-${{ hashFiles('setup.py') }}
 
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         run: |
@@ -181,11 +199,11 @@ jobs:
           OMP_NUM_THREADS: 1
         run: |
           source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_multiple_gpu tests
+          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_torch_multi_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_multiple_gpu_failures_short.txt          
+        run: cat reports/tests_torch_multi_gpu_failures_short.txt          
 
       - name: Test suite reports artifacts
         if: ${{ always() }}
@@ -194,7 +212,7 @@ jobs:
           name: run_all_tests_torch_multi_gpu_test_reports
           path: reports
 
-  run_tests_tf_multiple_gpu:
+  run_tests_tf_multi_gpu:
     runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
@@ -213,7 +231,7 @@ jobs:
         id: cache
         with:
           path: .env
-          key: v1.1-tests_tf_multiple_gpu-${{ hashFiles('setup.py') }}
+          key: v1.1-tests_tf_multi_gpu-${{ hashFiles('setup.py') }}
 
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         run: |
@@ -240,11 +258,11 @@ jobs:
           OMP_NUM_THREADS: 1
         run: |
           source .env/bin/activate
-          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_multiple_gpu tests
+          python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_tf_multi_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_multiple_gpu_failures_short.txt
+        run: cat reports/tests_tf_multi_gpu_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}

.github/workflows/self-scheduled.yml

@@ -1,3 +1,8 @@
+# configuration notes:
+#
+# - `source .env/bin/activate` is currently needed to be run first thing first in each step. Otherwise
+# the step uses the system-wide python interpreter.
+
 name: Self-hosted runner (scheduled)
 
 on:
@@ -182,7 +187,7 @@ jobs:
           name: run_all_tests_tf_gpu_test_reports
           path: reports
           
-  run_all_tests_torch_multiple_gpu:
+  run_all_tests_torch_multi_gpu:
     runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
@@ -227,19 +232,31 @@ jobs:
           python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
           python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
 
-      - name: Run all tests on GPU
+      - name: Run all tests on multi-GPU
         env:
           OMP_NUM_THREADS: 1
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_multiple_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_multi_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_multiple_gpu_failures_short.txt
+        run: cat reports/tests_torch_multi_gpu_failures_short.txt
-        
+
-      - name: Run all pipeline tests on GPU
+      - name: Run examples tests on multi-GPU
+        env:
+          OMP_NUM_THREADS: 1
+          RUN_SLOW: yes
+        run: |
+          source .env/bin/activate
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_multi_gpu examples
+
+      - name: Failure short reports
+        if: ${{ always() }}
+        run: cat reports/examples_torch_multi_gpu_failures_short.txt
+
+      - name: Run all pipeline tests on multi-GPU
         if: ${{ always() }}
         env:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
@@ -248,11 +265,11 @@ jobs:
           RUN_PIPELINE_TESTS: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_multiple_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_torch_pipeline_multi_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_pipeline_multiple_gpu_failures_short.txt
+        run: cat reports/tests_torch_pipeline_multi_gpu_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}
@@ -261,7 +278,7 @@ jobs:
           name: run_all_tests_torch_multi_gpu_test_reports
           path: reports
 
-  run_all_tests_tf_multiple_gpu:
+  run_all_tests_tf_multi_gpu:
     runs-on: [self-hosted, multi-gpu]
     steps:
       - uses: actions/checkout@v2
@@ -306,19 +323,19 @@ jobs:
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
           TF_CPP_MIN_LOG_LEVEL=3 python -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
 
-      - name: Run all tests on GPU
+      - name: Run all tests on multi-GPU
         env:
           OMP_NUM_THREADS: 1
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_multiple_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_tf_multi_gpu tests
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_multiple_gpu_failures_short.txt
+        run: cat reports/tests_tf_multi_gpu_failures_short.txt
 
-      - name: Run all pipeline tests on GPU
+      - name: Run all pipeline tests on multi-GPU
         if: ${{ always() }}
         env:
           TF_FORCE_GPU_ALLOW_GROWTH: "true"
@@ -327,11 +344,11 @@ jobs:
           RUN_PIPELINE_TESTS: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multiple_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multi_gpu tests
           
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_multiple_gpu_pipelines_failures_short.txt
+        run: cat reports/tests_tf_multi_gpu_pipelines_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}

CONTRIBUTING.md

@@ -308,3 +308,12 @@ Check our [documentation writing guide](https://github.com/huggingface/transform
 for more information.
 
 #### This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/master/CONTRIBUTING.md)
+
+
+### Develop on Windows
+
+One way one can run the make command on Window is to pass by MSYS2:
+
+1. [Download MSYS2](https://www.msys2.org/), we assume to have it installed in C:\msys64
+2. Open the command line C:\msys64\msys2.exe (it should be available from the start menu)
+3. Run in the shell: `pacman -Syu` and install make with `pacman -S make`

Makefile

@@ -3,14 +3,9 @@
 
 check_dirs := examples tests src utils
 
-# get modified files since the branch was made
-fork_point_sha := $(shell git merge-base --fork-point master)
-joined_dirs := $(shell echo $(check_dirs) | tr " " "|")
-modified_py_files := $(shell git diff --name-only $(fork_point_sha) | egrep '^($(joined_dirs))' | egrep '\.py$$')
-#$(info modified files are: $(modified_py_files))
-
 modified_only_fixup:
-	@if [ -n "$(modified_py_files)" ]; then \
+	$(eval modified_py_files := $(shell python utils/get_modified_files.py $(check_dirs)))
+	@if test -n "$(modified_py_files)"; then \
 		echo "Checking/fixing $(modified_py_files)"; \
 		black $(modified_py_files); \
 		isort $(modified_py_files); \

docs/source/_static/js/custom.js

@@ -1,10 +1,11 @@
 // These two things need to be updated at each release for the version selector.
 // Last stable version
-const stableVersion = "v3.4.0"
+const stableVersion = "v3.5.0"
 // Dictionary doc folder to label
 const versionMapping = {
     "master": "master",
-    "": "v3.4.0",
+    "": "v3.5.0/v3.5.1",
+    "v3.4.0": "v3.4.0",
     "v3.3.1": "v3.3.0/v3.3.1",
     "v3.2.0": "v3.2.0",
     "v3.1.0": "v3.1.0 (stable)",

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'3.4.0'
+release = u'3.5.0'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/main_classes/output.rst

@@ -65,12 +65,34 @@ BaseModelOutputWithPooling
     :members:
 
 
+BaseModelOutputWithCrossAttentions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.BaseModelOutputWithCrossAttentions
+    :members:
+
+
+BaseModelOutputWithPoolingAndCrossAttentions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.BaseModelOutputWithPoolingAndCrossAttentions
+    :members:
+
+
 BaseModelOutputWithPast
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.modeling_outputs.BaseModelOutputWithPast
     :members:
 
+
+BaseModelOutputWithPastAndCrossAttentions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.BaseModelOutputWithPastAndCrossAttentions
+    :members:
+
+
 Seq2SeqModelOutput
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -85,6 +107,20 @@ CausalLMOutput
     :members:
 
 
+CausalLMOutputWithCrossAttentions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.CausalLMOutputWithCrossAttentions
+    :members:
+
+
+CausalLMOutputWithPastAndCrossAttentions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_outputs.CausalLMOutputWithPastAndCrossAttentions
+    :members:
+
+
 CausalLMOutputWithPast
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/main_classes/trainer.rst

@@ -35,7 +35,7 @@ Here is an example of how to customize :class:`~transformers.Trainer` using a cu
     class MyTrainer(Trainer):
         def compute_loss(self, model, inputs):
             labels = inputs.pop("labels")
-            outputs = models(**inputs)
+            outputs = model(**inputs)
             logits = outputs[0]
             return my_custom_loss(logits, labels)
 

docs/source/model_doc/auto.rst

@@ -81,6 +81,13 @@ AutoModelForMultipleChoice
     :members:
 
 
+AutoModelForNextSentencePrediction
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.AutoModelForNextSentencePrediction
+    :members:
+
+
 AutoModelForTokenClassification
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/model_doc/bart.rst

@@ -3,7 +3,7 @@ BART
 
 **DISCLAIMER:** If you see something strange, file a `Github Issue
 <https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ and assign
-@sshleifer
+@patrickvonplaten
 
 Overview
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -26,6 +26,18 @@ According to the abstract,
 The Authors' code can be found `here <https://github.com/pytorch/fairseq/tree/master/examples/bart>`__.
 
 
+Examples
+_______________________________________________________________________________________________________________________
+
+- Examples and scripts for fine-tuning BART and other models for sequence to sequence tasks can be found in
+  `examples/seq2seq/ <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
+- An example of how to train :class:`~transformers.BartForConditionalGeneration` with a Hugging Face :obj:`datasets`
+  object can be found in this `forum discussion
+  <https://discuss.huggingface.co/t/train-bart-for-conditional-generation-e-g-summarization/1904>`__.
+- `Distilled checkpoints <https://huggingface.co/models?search=distilbart>`__ are described in this `paper
+  <https://arxiv.org/abs/2010.13002>`__.
+
+
 Implementation Notes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -34,14 +46,31 @@ Implementation Notes
 - The forward pass of :class:`~transformers.BartModel` will create decoder inputs (using the helper function
   :func:`transformers.modeling_bart._prepare_bart_decoder_inputs`) if they are not passed. This is different than some
   other modeling APIs.
-- Model predictions are intended to be identical to the original implementation. This only works, however, if the
+- Model predictions are intended to be identical to the original implementation when
-  string you pass to :func:`fairseq.encode` starts with a space.
+  :obj:`force_bos_token_to_be_generated=True`. This only works, however, if the string you pass to
+  :func:`fairseq.encode` starts with a space.
 - :meth:`~transformers.BartForConditionalGeneration.generate` should be used for conditional generation tasks like
   summarization, see the example in that docstrings.
 - Models that load the `facebook/bart-large-cnn` weights will not have a :obj:`mask_token_id`, or be able to perform
   mask-filling tasks.
 - For training/forward passes that don't involve beam search, pass :obj:`use_cache=False`.
 
+Mask Filling
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :obj:`facebook/bart-base` and :obj:`facebook/bart-large` checkpoints can be used to fill multi-token masks.
+
+.. code-block::
+
+    from transformers import BartForConditionalGeneration, BartTokenizer
+    model = BartForConditionalGeneration.from_pretrained("facebook/bart-large", force_bos_token_to_be_generated=True)
+    tok = BartTokenizer.from_pretrained("facebook/bart-large")
+    example_english_phrase = "UN Chief Says There Is No <mask> in Syria"
+    batch = tok(example_english_phrase, return_tensors='pt')
+    generated_ids = model.generate(batch['input_ids'])
+    assert tok.batch_decode(generated_ids, skip_special_tokens=True) == ['UN Chief Says There Is No Plan to Stop Chemical Weapons in Syria']
+
+
 
 BartConfig
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/source/model_doc/bert.rst

@@ -188,3 +188,10 @@ TFBertForQuestionAnswering
 
 .. autoclass:: transformers.TFBertForQuestionAnswering
     :members: call
+
+
+FlaxBertModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaxBertModel
+    :members: __call__

docs/source/model_doc/dpr.rst

@@ -99,3 +99,22 @@ DPRReader
 
 .. autoclass:: transformers.DPRReader
     :members: forward
+
+TFDPRContextEncoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFDPRContextEncoder
+    :members: call
+
+TFDPRQuestionEncoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFDPRQuestionEncoder
+    :members: call
+
+
+TFDPRReader
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFDPRReader
+    :members: call

docs/source/model_doc/longformer.rst

@@ -90,6 +90,32 @@ LongformerTokenizerFast
 .. autoclass:: transformers.LongformerTokenizerFast
     :members: 
 
+Longformer specific outputs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.modeling_longformer.LongformerBaseModelOutput
+    :members: 
+
+.. autoclass:: transformers.modeling_longformer.LongformerBaseModelOutputWithPooling
+    :members: 
+
+.. autoclass:: transformers.modeling_longformer.LongformerMultipleChoiceModelOutput
+    :members: 
+
+.. autoclass:: transformers.modeling_longformer.LongformerQuestionAnsweringModelOutput
+    :members: 
+
+.. autoclass:: transformers.modeling_tf_longformer.TFLongformerBaseModelOutput
+    :members: 
+
+.. autoclass:: transformers.modeling_tf_longformer.TFLongformerBaseModelOutputWithPooling
+    :members: 
+
+.. autoclass:: transformers.modeling_tf_longformer.TFLongformerQuestionAnsweringModelOutput
+    :members: 
+
+LongformerModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 LongformerModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/source/model_doc/marian.rst

@@ -3,9 +3,9 @@ MarianMT
 
 **Bugs:** If you see something strange, file a `Github Issue
 <https://github.com/huggingface/transformers/issues/new?assignees=sshleifer&labels=&template=bug-report.md&title>`__
-and assign @sshleifer.
+and assign @patrickvonplaten.
 
-Translations should be similar, but not identical to, output in the test set linked to in each model card.
+Translations should be similar, but not identical to output in the test set linked to in each model card.
 
 Implementation Notes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -35,24 +35,109 @@ Naming
   <https://developers.google.com/admin-sdk/directory/v1/languages>`__, three digit codes require googling "language
   code {code}".
 - Codes formatted like :obj:`es_AR` are usually :obj:`code_{region}`. That one is Spanish from Argentina.
+- The models were converted in two stages. The first 1000 models use ISO-639-2 codes to identify languages, the second
+  group use a combination of ISO-639-5 codes and ISO-639-2 codes.
 
 
+Examples
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Since Marian models are smaller than many other translation models available in the library, they can be useful for
+  fine-tuning experiments and integration tests.
+- `Fine-tune on TPU
+  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/builtin_trainer/train_distil_marian_enro_tpu.sh>`__
+- `Fine-tune on GPU
+  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/builtin_trainer/train_distil_marian_enro.sh>`__
+- `Fine-tune on GPU with pytorch-lightning
+  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/distil_marian_no_teacher.sh>`__
+
 Multilingual Models
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-All model names use the following format: :obj:`Helsinki-NLP/opus-mt-{src}-{tgt}`:
+- All model names use the following format: :obj:`Helsinki-NLP/opus-mt-{src}-{tgt}`:
+- If a model can output multiple languages, and you should specify a language code by prepending the desired output
+  language to the :obj:`src_text`.
+- You can see a models's supported language codes in its model card, under target constituents, like in `opus-mt-en-roa
+  <https://huggingface.co/Helsinki-NLP/opus-mt-en-roa>`__.
+- Note that if a model is only multilingual on the source side, like :obj:`Helsinki-NLP/opus-mt-roa-en`, no language
+  codes are required.
 
-    - If :obj:`src` is in all caps, the model supports multiple input languages, you can figure out which ones by
+New multi-lingual models from the `Tatoeba-Challenge repo <https://github.com/Helsinki-NLP/Tatoeba-Challenge>`__
-      looking at the model card, or the Group Members `mapping
+require 3 character language codes:
-      <https://gist.github.com/sshleifer/6d20e7761931b08e73c3219027b97b8a>`_ .
-    - If :obj:`tgt` is in all caps, the model can output multiple languages, and you should specify a language code by
-      prepending the desired output language to the :obj:`src_text`.
-    - You can see a tokenizer's supported language codes in ``tokenizer.supported_language_codes``
-
-Example of translating english to many romance languages, using language codes:
 
 .. code-block:: python
 
+    from transformers import MarianMTModel, MarianTokenizer
+    src_text = [
+        '>>fra<< this is a sentence in english that we want to translate to french',
+        '>>por<< This should go to portuguese',
+        '>>esp<< And this to Spanish'
+    ]
+
+    model_name = 'Helsinki-NLP/opus-mt-en-roa'
+    tokenizer = MarianTokenizer.from_pretrained(model_name)
+    print(tokenizer.supported_language_codes)
+    model = MarianMTModel.from_pretrained(model_name)
+    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text))
+    tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
+    # ["c'est une phrase en anglais que nous voulons traduire en français",
+    # 'Isto deve ir para o português.',
+    # 'Y esto al español']
+
+
+
+
+Code to see available pretrained models:
+
+.. code-block:: python
+
+    from transformers.hf_api import HfApi
+    model_list = HfApi().model_list()
+    org = "Helsinki-NLP"
+    model_ids = [x.modelId for x in model_list if x.modelId.startswith(org)]
+    suffix = [x.split('/')[1] for x in model_ids]
+    old_style_multi_models = [f'{org}/{s}' for s in suffix if s != s.lower()]
+
+
+
+Old Style Multi-Lingual Models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These are the old style multi-lingual models ported from the OPUS-MT-Train repo: and the members of each language
+group:
+
+.. code-block:: python
+
+    ['Helsinki-NLP/opus-mt-NORTH_EU-NORTH_EU',
+     'Helsinki-NLP/opus-mt-ROMANCE-en',
+     'Helsinki-NLP/opus-mt-SCANDINAVIA-SCANDINAVIA',
+     'Helsinki-NLP/opus-mt-de-ZH',
+     'Helsinki-NLP/opus-mt-en-CELTIC',
+     'Helsinki-NLP/opus-mt-en-ROMANCE',
+     'Helsinki-NLP/opus-mt-es-NORWAY',
+     'Helsinki-NLP/opus-mt-fi-NORWAY',
+     'Helsinki-NLP/opus-mt-fi-ZH',
+     'Helsinki-NLP/opus-mt-fi_nb_no_nn_ru_sv_en-SAMI',
+     'Helsinki-NLP/opus-mt-sv-NORWAY',
+     'Helsinki-NLP/opus-mt-sv-ZH']
+    GROUP_MEMBERS = {
+     'ZH': ['cmn', 'cn', 'yue', 'ze_zh', 'zh_cn', 'zh_CN', 'zh_HK', 'zh_tw', 'zh_TW', 'zh_yue', 'zhs', 'zht', 'zh'],
+     'ROMANCE': ['fr', 'fr_BE', 'fr_CA', 'fr_FR', 'wa', 'frp', 'oc', 'ca', 'rm', 'lld', 'fur', 'lij', 'lmo', 'es', 'es_AR', 'es_CL', 'es_CO', 'es_CR', 'es_DO', 'es_EC', 'es_ES', 'es_GT', 'es_HN', 'es_MX', 'es_NI', 'es_PA', 'es_PE', 'es_PR', 'es_SV', 'es_UY', 'es_VE', 'pt', 'pt_br', 'pt_BR', 'pt_PT', 'gl', 'lad', 'an', 'mwl', 'it', 'it_IT', 'co', 'nap', 'scn', 'vec', 'sc', 'ro', 'la'],
+     'NORTH_EU': ['de', 'nl', 'fy', 'af', 'da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
+     'SCANDINAVIA': ['da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
+     'SAMI': ['se', 'sma', 'smj', 'smn', 'sms'],
+     'NORWAY': ['nb_NO', 'nb', 'nn_NO', 'nn', 'nog', 'no_nb', 'no'],
+     'CELTIC': ['ga', 'cy', 'br', 'gd', 'kw', 'gv']
+    }
+
+
+
+
+Example of translating english to many romance languages, using old-style 2 character language codes
+
+
+.. code-block::python
+
     from transformers import MarianMTModel, MarianTokenizer
     src_text = [
         '>>fr<< this is a sentence in english that we want to translate to french',
@@ -63,52 +148,12 @@ Example of translating english to many romance languages, using language codes:
     model_name = 'Helsinki-NLP/opus-mt-en-ROMANCE'
     tokenizer = MarianTokenizer.from_pretrained(model_name)
     print(tokenizer.supported_language_codes)
+
     model = MarianMTModel.from_pretrained(model_name)
     translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text))
     tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
-    # ["c'est une phrase en anglais que nous voulons traduire en français",
+    # ["c'est une phrase en anglais que nous voulons traduire en français", 'Isto deve ir para o português.',  'Y esto al español']
-    # 'Isto deve ir para o português.',
-    # 'Y esto al español']
 
-Sometimes, models were trained on collections of languages that do not resolve to a group. In this case, _ is used as a
-separator for src or tgt, as in :obj:`Helsinki-NLP/opus-mt-en_el_es_fi-en_el_es_fi`. These still require language
-codes.
-
-There are many supported regional language codes, like :obj:`>>es_ES<<` (Spain) and :obj:`>>es_AR<<` (Argentina), that
-do not seem to change translations. I have not found these to provide different results than just using :obj:`>>es<<`.
-
-For example:
-
-    - `Helsinki-NLP/opus-mt-NORTH_EU-NORTH_EU`: translates from all NORTH_EU languages (see `mapping
-      <https://gist.github.com/sshleifer/6d20e7761931b08e73c3219027b97b8a>`_) to all NORTH_EU languages. Use a special
-      language code like :obj:`>>de<<` to specify output language.
-    - `Helsinki-NLP/opus-mt-ROMANCE-en`: translates from many romance languages to english, no codes needed since there
-      is only one target language.
-
-
-
-.. code-block:: python
-
-    GROUP_MEMBERS = {
-     'ZH': ['cmn', 'cn', 'yue', 'ze_zh', 'zh_cn', 'zh_CN', 'zh_HK', 'zh_tw', 'zh_TW', 'zh_yue', 'zhs', 'zht', 'zh'],
-     'ROMANCE': ['fr', 'fr_BE', 'fr_CA', 'fr_FR', 'wa', 'frp', 'oc', 'ca', 'rm', 'lld', 'fur', 'lij', 'lmo', 'es', 'es_AR', 'es_CL', 'es_CO', 'es_CR', 'es_DO', 'es_EC', 'es_ES', 'es_GT', 'es_HN', 'es_MX', 'es_NI', 'es_PA', 'es_PE', 'es_PR', 'es_SV', 'es_UY', 'es_VE', 'pt', 'pt_br', 'pt_BR', 'pt_PT', 'gl', 'lad', 'an', 'mwl', 'it', 'it_IT', 'co', 'nap', 'scn', 'vec', 'sc', 'ro', 'la'],
-     'NORTH_EU': ['de', 'nl', 'fy', 'af', 'da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
-     'SCANDINAVIA': ['da', 'fo', 'is', 'no', 'nb', 'nn', 'sv'],
-     'SAMI': ['se', 'sma', 'smj', 'smn', 'sms'],
-     'NORWAY': ['nb_NO', 'nb', 'nn_NO', 'nn', 'nog', 'no_nb', 'no'],
-     'CELTIC': ['ga', 'cy', 'br', 'gd', 'kw', 'gv']
-    }
-
-Code to see available pretrained models:
-
-.. code-block:: python
-
-    from transformers.hf_api import HfApi
-    model_list = HfApi().model_list()
-    org = "Helsinki-NLP"
-    model_ids = [x.modelId for x in model_list if x.modelId.startswith(org)]
-    suffix = [x.split('/')[1] for x in model_ids]
-    multi_models = [f'{org}/{s}' for s in suffix if s != s.lower()]
 
 
 MarianConfig

docs/source/model_doc/mbart.rst

@@ -3,7 +3,7 @@ MBart
 
 **DISCLAIMER:** If you see something strange, file a `Github Issue
 <https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ and assign
-@sshleifer
+@patrickvonplaten
 
 Overview
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -19,6 +19,13 @@ on the encoder, decoder, or reconstructing parts of the text.
 
 The Authors' code can be found `here <https://github.com/pytorch/fairseq/tree/master/examples/mbart>`__
 
+Examples
+_______________________________________________________________________________________________________________________
+
+- Examples and scripts for fine-tuning mBART and other models for sequence to sequence tasks can be found in
+  `examples/seq2seq/ <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
+- Given the large embeddings table, mBART consumes a large amount of GPU RAM, especially for fine-tuning.
+  :class:`MarianMTModel` is usually a better choice for bilingual machine translation.
 
 Training
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -38,11 +45,7 @@ the sequences for sequence-to-sequence fine-tuning.
     example_english_phrase = "UN Chief Says There Is No Military Solution in Syria"
     expected_translation_romanian = "Şeful ONU declară că nu există o soluţie militară în Siria"
     batch = tokenizer.prepare_seq2seq_batch(example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian)
-    input_ids = batch["input_ids"]
+    model(input_ids=batch['input_ids'], labels=batch['labels']) # forward pass
-    target_ids = batch["decoder_input_ids"]
-    decoder_input_ids = target_ids[:, :-1].contiguous()
-    labels = target_ids[:, 1:].clone()
-    model(input_ids=input_ids, decoder_input_ids=decoder_input_ids, labels=labels) #forward
 
 - Generation
 

docs/source/model_doc/pegasus.rst

@@ -3,7 +3,7 @@ Pegasus
 
 **DISCLAIMER:** If you see something strange, file a `Github Issue
 <https://github.com/huggingface/transformers/issues/new?assignees=sshleifer&labels=&template=bug-report.md&title>`__
-and assign @sshleifer.
+and assign @patrickvonplaten.
 
 
 Overview
@@ -31,10 +31,19 @@ All the `checkpoints <https://huggingface.co/models?search=pegasus>`__ are fine-
 - Each checkpoint is 2.2 GB on disk and 568M parameters.
 - FP16 is not supported (help/ideas on this appreciated!).
 - Summarizing xsum in fp32 takes about 400ms/sample, with default parameters on a v100 GPU.
-- For XSUM, The paper reports rouge1,rouge2, rougeL of paper: 47.21/24.56/39.25. As of Aug 9, this port scores
+- Full replication results and correctly pre-processed data can be found in this `Issue
-  46.91/24.34/39.1.
+  <https://github.com/huggingface/transformers/issues/6844#issue-689259666>`__.
+- `Distilled checkpoints <https://huggingface.co/models?search=distill-pegasus>`__ are described in this `paper
+  <https://arxiv.org/abs/2010.13002>`__.
 
-The gap is likely because of different alpha/length_penalty implementations in beam search.
+Examples
+_______________________________________________________________________________________________________________________
+
+- `Script <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/finetune_pegasus_xsum.sh>`__ to
+  fine-tune pegasus on the XSUM dataset. Data download instructions at `examples/seq2seq/
+  <https://github.com/huggingface/transformers/blob/master/examples/seq2seq/README.md>`__.
+- FP16 is not supported (help/ideas on this appreciated!).
+- The adafactor optimizer is recommended for pegasus fine-tuning.
 
 
 Implementation Notes
@@ -45,7 +54,7 @@ Implementation Notes
 - Some key configuration differences:
 
     - static, sinusoidal position embeddings
-    - no :obj:`layernorm_embedding` (:obj`PegasusConfig.normalize_embedding=False`)
+    - no :obj:`layernorm_embedding` (:obj:`PegasusConfig.normalize_embedding=False`)
     - the model starts generating with pad_token_id (which has 0 token_embedding) as the prefix.
     - more beams are used (:obj:`num_beams=8`)
 - All pretrained pegasus checkpoints are the same besides three attributes: :obj:`tokenizer.model_max_length` (maximum

docs/source/model_doc/roberta.rst

@@ -146,3 +146,10 @@ TFRobertaForQuestionAnswering
 
 .. autoclass:: transformers.TFRobertaForQuestionAnswering
     :members: call
+
+
+FlaxRobertaModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.FlaxRobertaModel
+    :members: __call__

docs/source/model_sharing.rst

@@ -18,39 +18,76 @@ done something similar on your task, either using the model directly in your own
 :class:`~.transformers.Trainer`/:class:`~.transformers.TFTrainer` class. Let's see how you can share the result on the
 `model hub <https://huggingface.co/models>`__.
 
+Model versioning
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Since version v3.5.0, the model hub has built-in model versioning based on git and git-lfs. It is based on the paradigm
+that one model *is* one repo.
+
+This allows:
+
+- built-in versioning
+- access control
+- scalability
+
+This is built around *revisions*, which is a way to pin a specific version of a model, using a commit hash, tag or
+branch.
+
+For instance:
+
+.. code-block::
+
+    >>> model = AutoModel.from_pretrained(
+    >>>   "julien-c/EsperBERTo-small",
+    >>>   revision="v2.0.1" # tag name, or branch name, or commit hash
+    >>> )
+
 Basic steps
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. 
+In order to upload a model, you'll need to first create a git repo. This repo will live on the model hub, allowing
-    When #5258 is merged, we can remove the need to create the directory.
+users to clone it and you (and your organization members) to push to it.
 
-First, pick a directory with the name you want your model to have on the model hub (its full name will then be
+You can create a model repo directly from the website, `here <https://huggingface.co/new>`.
-`username/awesome-name-you-picked` or `organization/awesome-name-you-picked`) and create it with either
 
-.. code-block::
+Alternatively, you can use the ``transformers-cli``. The next steps describe that process:
 
-    mkdir path/to/awesome-name-you-picked
+Go to a terminal and run the following command. It should be in the virtual environment where you installed 🤗
+Transformers, since that command :obj:`transformers-cli` comes from the library.
 
-or in python
+.. code-block:: bash
 
-.. code-block::
+    transformers-cli login
 
-    import os
-    os.makedirs("path/to/awesome-name-you-picked")
 
-then you can save your model and tokenizer with:
+Once you are logged in with your model hub credentials, you can start building your repositories. To create a repo:
 
-.. code-block::
+.. code-block:: bash
 
-    model.save_pretrained("path/to/awesome-name-you-picked")
+    transformers-cli repo create your-model-name
-    tokenizer.save_pretrained("path/to/awesome-name-you-picked")
 
-Or, if you're using the Trainer API
+This creates a repo on the model hub, which can be cloned.
 
-.. code-block::
+.. code-block:: bash
+
+    git clone https://huggingface.co/username/your-model-name
+
+    # Make sure you have git-lfs installed
+    # (https://git-lfs.github.com/)
+    git lfs install
+
+When you have your local clone of your repo and lfs installed, you can then add/remove from that clone as you would
+with any other git repo.
+
+.. code-block:: bash
+
+    # Commit as usual
+    cd your-model-name
+    echo "hello" >> README.md
+    git add . && git commit -m "Update from $USER"
+
+We are intentionally not wrapping git too much, so as to stay intuitive and easy-to-use.
 
-    trainer.save_model("path/to/awesome-name-you-picked")
-    tokenizer.save_pretrained("path/to/awesome-name-you-picked")
 
 Make your model work on all frameworks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -71,13 +108,13 @@ or removing TF. For instance, if you trained a :class:`~transformers.DistilBertF
 
 .. code-block::
 
-    from transformers import TFDistilBertForSequenceClassification
+    >>> from transformers import TFDistilBertForSequenceClassification
 
 and if you trained a :class:`~transformers.TFDistilBertForSequenceClassification`, try to type
 
 .. code-block::
 
-    from transformers import DistilBertForSequenceClassification
+    >>> from transformers import DistilBertForSequenceClassification
 
 This will give back an error if your model does not exist in the other framework (something that should be pretty rare
 since we're aiming for full parity between the two frameworks). In this case, skip this and go to the next step.
@@ -87,20 +124,20 @@ model class:
 
 .. code-block::
 
-    tf_model = TFDistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_pt=True)
+    >>> tf_model = TFDistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_pt=True)
-    tf_model.save_pretrained("path/to/awesome-name-you-picked")
+    >>> tf_model.save_pretrained("path/to/awesome-name-you-picked")
 
 and if you trained your model in TensorFlow and have to create a PyTorch version, adapt the following code to your
 model class:
 
 .. code-block::
 
-    pt_model = DistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_tf=True)
+    >>> pt_model = DistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_tf=True)
-    pt_model.save_pretrained("path/to/awesome-name-you-picked")
+    >>> pt_model.save_pretrained("path/to/awesome-name-you-picked")
 
 That's all there is to it!
 
-Check the directory before uploading
+Check the directory before pushing to the model hub.
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Make sure there are no garbage files in the directory you'll upload. It should only have:
@@ -116,62 +153,47 @@ Make sure there are no garbage files in the directory you'll upload. It should o
 
 Other files can safely be deleted.
 
-Upload your model with the CLI
+
+Uploading your files
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Now go in a terminal and run the following command. It should be in the virtual environment where you installed 🤗
+Once the repo is cloned, you can add the model, configuration and tokenizer files. For instance, saving the model and
-Transformers, since that command :obj:`transformers-cli` comes from the library.
+tokenizer files:
 
 .. code-block::
 
-    transformers-cli login
+    >>> model.save_pretrained("path/to/repo/clone/your-model-name")
+    >>> tokenizer.save_pretrained("path/to/repo/clone/your-model-name")
 
-Then log in using the same credentials as on huggingface.co. To upload your model, just type
+Or, if you're using the Trainer API
 
 .. code-block::
 
-    transformers-cli upload path/to/awesome-name-you-picked/
+    >>> trainer.save_model("path/to/awesome-name-you-picked")
+    >>> tokenizer.save_pretrained("path/to/repo/clone/your-model-name")
 
-This will upload the folder containing the weights, tokenizer and configuration we prepared in the previous section.
+You can then add these files to the staging environment and verify that they have been correctly staged with the ``git
+status`` command:
 
-By default you will be prompted to confirm that you want these files to be uploaded. If you are uploading multiple
+.. code-block:: bash
-models and need to script that process, you can add `-y` to bypass the prompt. For example:
 
-.. code-block::
+    git add --all
+    git status
 
-    transformers-cli upload -y path/to/awesome-name-you-picked/
+Finally, the files should be comitted:
 
+.. code-block:: bash
 
-If you want to upload a single file (a new version of your model, or the other framework checkpoint you want to add),
+    git commit -m "First version of the your-model-name model and tokenizer."
-just type:
 
-.. code-block::
+And pushed to the remote:
 
-    transformers-cli upload path/to/awesome-name-you-picked/that-file 
+.. code-block:: bash
 
-or
+    git push
 
-.. code-block::
+This will upload the folder containing the weights, tokenizer and configuration we have just prepared.
 
-   transformers-cli upload path/to/awesome-name-you-picked/that-file --filename awesome-name-you-picked/new_name
-
-if you want to change its filename.
-
-This uploads the model to your personal account. If you want your model to be namespaced by your organization name
-rather than your username, add the following flag to any command:
-
-.. code-block::
-
-    --organization organization_name
-
-so for instance:
-
-.. code-block::
-
-    transformers-cli upload path/to/awesome-name-you-picked/ --organization organization_name
-
-Your model will then be accessible through its identifier, which is, as we saw above,
-`username/awesome-name-you-picked` or `organization/awesome-name-you-picked`.
 
 Add a model card
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -189,7 +211,7 @@ don't forget to link to its model card so that people can fully trace how your m
 If you have never made a pull request to the 🤗 Transformers repo, look at the :doc:`contributing guide <contributing>`
 to see the steps to follow.
 
-.. Note::
+.. note::
 
     You can also send your model card in the folder you uploaded with the CLI by placing it in a `README.md` file
     inside `path/to/awesome-name-you-picked/`.
@@ -203,20 +225,59 @@ Anyone can load it from code:
 
 .. code-block::
 
-    tokenizer = AutoTokenizer.from_pretrained("namespace/awesome-name-you-picked")
+    >>> tokenizer = AutoTokenizer.from_pretrained("namespace/awesome-name-you-picked")
-    model = AutoModel.from_pretrained("namespace/awesome-name-you-picked")
+    >>> model = AutoModel.from_pretrained("namespace/awesome-name-you-picked")
 
-Additional commands
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-You can list all the files you uploaded on the hub like this:
+You may specify a revision by using the ``revision`` flag in the ``from_pretrained`` method:
 
 .. code-block::
 
-    transformers-cli s3 ls
+    >>> tokenizer = AutoTokenizer.from_pretrained(
+    >>>   "julien-c/EsperBERTo-small",
+    >>>   revision="v2.0.1" # tag name, or branch name, or commit hash
+    >>> )
 
-You can also delete unneeded files with
+Workflow in a Colab notebook
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. code-block::
+If you're in a Colab notebook (or similar) with no direct access to a terminal, here is the workflow you can use to
+upload your model. You can execute each one of them in a cell by adding a ! at the beginning.
 
-    transformers-cli s3 rm awesome-name-you-picked/filename
+First you need to install `git-lfs` in the environment used by the notebook:
+
+.. code-block:: bash
+
+    sudo apt-get install git-lfs
+
+Then you can use the :obj:`transformers-cli` to create your new repo:
+
+
+.. code-block:: bash
+
+    transformers-cli login
+    transformers-cli repo create your-model-name
+
+Once it's created, you can clone it and configure it (replace username by your username on huggingface.co):
+
+.. code-block:: bash
+
+    git clone https://username:password@huggingface.co/username/your-model-name
+    # Alternatively if you have a token,
+    # you can use it instead of your password
+    git clone https://username:token@huggingface.co/username/your-model-name
+
+    cd your-model-name
+    git lfs install
+    git config --global user.email "email@example.com"
+    # Tip: using the same email than for your huggingface.co account will link your commits to your profile
+    git config --global user.name "Your name"
+
+Once you've saved your model inside, and your clone is setup with the right remote URL, you can add it and push it with
+usual git commands.
+
+.. code-block:: bash
+
+    git add .
+    git commit -m "Initial commit"
+    git push

docs/source/task_summary.rst

@@ -513,7 +513,7 @@ Here, the model generates a random text with a total maximal length of *50* toke
 concerned, I will"*. The default arguments of ``PreTrainedModel.generate()`` can be directly overridden in the
 pipeline, as is shown above for the argument ``max_length``.
 
-Here is an example of text generation using ``XLNet`` and its tokenzier.
+Here is an example of text generation using ``XLNet`` and its tokenizer.
 
 .. code-block::
 
@@ -834,7 +834,7 @@ Here is an example of doing translation using a model and a tokenizer. The proce
 
 1. Instantiate a tokenizer and a model from the checkpoint name. Summarization is usually done using an encoder-decoder
    model, such as ``Bart`` or ``T5``.
-2. Define the article that should be summarizaed.
+2. Define the article that should be summarized.
 3. Add the T5 specific prefix "translate English to German: "
 4. Use the ``PreTrainedModel.generate()`` method to perform the translation.
 

docs/source/testing.rst

@@ -405,16 +405,32 @@ decorators are used to set the requirements of tests CPU/GPU/TPU-wise:
 
 * ``require_torch`` - this test will run only under torch
 * ``require_torch_gpu`` - as ``require_torch`` plus requires at least 1 GPU
-* ``require_torch_multigpu`` - as ``require_torch`` plus requires at least 2 GPUs
+* ``require_torch_multi_gpu`` - as ``require_torch`` plus requires at least 2 GPUs
-* ``require_torch_non_multigpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
+* ``require_torch_non_multi_gpu`` - as ``require_torch`` plus requires 0 or 1 GPUs
 * ``require_torch_tpu`` - as ``require_torch`` plus requires at least 1 TPU
 
+Let's depict the GPU requirements in the following table:
+
+
++----------+----------------------------------+
+| n gpus   |  decorator                       |
++==========+==================================+
+| ``>= 0`` | ``@require_torch``               |
++----------+----------------------------------+
+| ``>= 1`` | ``@require_torch_gpu``           |
++----------+----------------------------------+
+| ``>= 2`` | ``@require_torch_multi_gpu``     |
++----------+----------------------------------+
+| ``< 2``  | ``@require_torch_non_multi_gpu`` |
++----------+----------------------------------+
+
+
 For example, here is a test that must be run only when there are 2 or more GPUs available and pytorch is installed:
 
 .. code-block:: python
 
-    @require_torch_multigpu
+    @require_torch_multi_gpu
-    def test_example_with_multigpu():
+    def test_example_with_multi_gpu():
 
 If a test requires ``tensorflow`` use the ``require_tf`` decorator. For example:
 
@@ -438,7 +454,7 @@ last for them to work correctly. Here is an example of the correct usage:
 .. code-block:: python
 
     @parameterized.expand(...)
-    @require_torch_multigpu
+    @require_torch_multi_gpu
     def test_integration_foo():
 
 This order problem doesn't exist with ``@pytest.mark.parametrize``, you can put it first or last and it will still
@@ -450,7 +466,8 @@ Inside tests:
 
 .. code-block:: bash
 
-   torch.cuda.device_count()
+   from transformers.testing_utils import get_gpu_count
+   n_gpu = get_gpu_count() # works with torch and tf
 
 
 
@@ -699,11 +716,11 @@ Temporary files and directories
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Using unique temporary files and directories are essential for parallel test running, so that the tests won't overwrite
-each other's data. Also we want to get the temp files and directories removed at the end of each test that created
+each other's data. Also we want to get the temporary files and directories removed at the end of each test that created
 them. Therefore, using packages like ``tempfile``, which address these needs is essential.
 
-However, when debugging tests, you need to be able to see what goes into the temp file or directory and you want to
+However, when debugging tests, you need to be able to see what goes into the temporary file or directory and you want
-know it's exact path and not having it randomized on every test re-run.
+to know it's exact path and not having it randomized on every test re-run.
 
 A helper class :obj:`transformers.test_utils.TestCasePlus` is best used for such purposes. It's a sub-class of
 :obj:`unittest.TestCase`, so we can easily inherit from it in the test modules.
@@ -719,32 +736,33 @@ Here is an example of its usage:
 
 This code creates a unique temporary directory, and sets :obj:`tmp_dir` to its location.
 
-In this and all the following scenarios the temporary directory will be auto-removed at the end of test, unless
+* Create a unique temporary dir:
-``after=False`` is passed to the helper function.
-
-* Create a temporary directory of my choice and delete it at the end - useful for debugging when you want to monitor a
-  specific directory:
 
 .. code-block:: python
 
     def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test")
+        tmp_dir = self.get_auto_remove_tmp_dir()
 
-* Create a temporary directory of my choice and do not delete it at the end---useful for when you want to look at the
+``tmp_dir`` will contain the path to the created temporary dir. It will be automatically removed at the end of the
-  temp results:
+test.
+
+* Create a temporary dir of my choice, ensure it's empty before the test starts and don't empty it after the test.
 
 .. code-block:: python
 
     def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", after=False)
+        tmp_dir = self.get_auto_remove_tmp_dir("./xxx")
 
-* Create a temporary directory of my choice and ensure to delete it right away---useful for when you disabled deletion
+This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn't
-  in the previous test run and want to make sure the that temporary directory is empty before the new test is run:
+leave any data in there.
 
-.. code-block:: python
+* You can override the default behavior by directly overriding the ``before`` and ``after`` args, leading to one of the
+  following behaviors:
 
-   def test_whatever(self):
+    - ``before=True``: the temporary dir will always be cleared at the beginning of the test.
-        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", before=True)
+    - ``before=False``: if the temporary dir already existed, any existing files will remain there.
+    - ``after=True``: the temporary dir will always be deleted at the end of the test.
+    - ``after=False``: the temporary dir will always be left intact at the end of the test.
 
 .. note::
    In order to run the equivalent of ``rm -r`` safely, only subdirs of the project repository checkout are allowed if
@@ -798,7 +816,7 @@ or the ``xfail`` way:
     @pytest.mark.xfail
     def test_feature_x():
 
-Here is how to skip a test based on some internal check inside the test:
+- Here is how to skip a test based on some internal check inside the test:
 
 .. code-block:: python
 
@@ -821,7 +839,7 @@ or the ``xfail`` way:
     def test_feature_x():
         pytest.xfail("expected to fail until bug XYZ is fixed")
 
-Here is how to skip all tests in a module if some import is missing:
+- Here is how to skip all tests in a module if some import is missing:
 
 .. code-block:: python
 

examples/README.md

@@ -37,7 +37,7 @@ git checkout tags/v3.4.0
 |---|---|:---:|:---:|:---:|:---:|
 | [**`language-modeling`**](https://github.com/huggingface/transformers/tree/master/examples/language-modeling)       | Raw text        | ✅ | -  | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb)
 | [**`text-classification`**](https://github.com/huggingface/transformers/tree/master/examples/text-classification)   | GLUE, XNLI      | ✅ | ✅ | ✅ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://github.com/huggingface/notebooks/blob/master/examples/text_classification.ipynb)
-| [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER       | ✅ | ✅ | - | -
+| [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER       | ✅ | ✅ | ✅ | -
 | [**`multiple-choice`**](https://github.com/huggingface/transformers/tree/master/examples/multiple-choice)           | SWAG, RACE, ARC | ✅ | ✅ | - | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ViktorAlm/notebooks/blob/master/MPC_GPU_Demo_for_TF_and_PT.ipynb)
 | [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering)     | SQuAD           | ✅ | ✅ | - | -
 | [**`text-generation`**](https://github.com/huggingface/transformers/tree/master/examples/text-generation)           | -               | n/a | n/a | - | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)
@@ -62,13 +62,14 @@ When using PyTorch, we support TPUs thanks to `pytorch/xla`. For more context an
 very detailed [pytorch/xla README](https://github.com/pytorch/xla/blob/master/README.md).
 
 In this repo, we provide a very simple launcher script named [xla_spawn.py](https://github.com/huggingface/transformers/tree/master/examples/xla_spawn.py) that lets you run our example scripts on multiple TPU cores without any boilerplate.
-Just pass a `--num_cores` flag to this script, then your regular training script with its arguments (this is similar to the `torch.distributed.launch` helper for torch.distributed).
+Just pass a `--num_cores` flag to this script, then your regular training script with its arguments (this is similar to the `torch.distributed.launch` helper for torch.distributed). 
+Note that this approach does not work for examples that use `pytorch-lightning`.
 
 For example for `run_glue`:
 
 ```bash
 python examples/xla_spawn.py --num_cores 8 \
-	examples/text-classification/run_glue.py
+	examples/text-classification/run_glue.py \
 	--model_name_or_path bert-base-cased \
 	--task_name mnli \
 	--data_dir ./data/glue_data/MNLI \

examples/adversarial/run_hans.py

@@ -23,6 +23,7 @@ from typing import Dict, List, Optional
 import numpy as np
 import torch
 
+import transformers
 from transformers import (
     AutoConfig,
     AutoModelForSequenceClassification,
@@ -33,6 +34,7 @@ from transformers import (
     default_data_collator,
     set_seed,
 )
+from transformers.trainer_utils import is_main_process
 from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels
 
 
@@ -124,6 +126,11 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(training_args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/bert-loses-patience/run_glue_with_pabee.py

@@ -29,6 +29,7 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
+import transformers
 from pabee.modeling_pabee_albert import AlbertForSequenceClassificationWithPabee
 from pabee.modeling_pabee_bert import BertForSequenceClassificationWithPabee
 from transformers import (
@@ -44,6 +45,7 @@ from transformers import glue_compute_metrics as compute_metrics
 from transformers import glue_convert_examples_to_features as convert_examples_to_features
 from transformers import glue_output_modes as output_modes
 from transformers import glue_processors as processors
+from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -630,7 +632,11 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     # Set seed
     set_seed(args)
 

examples/bert-loses-patience/test_run_glue_with_pabee.py

@@ -4,7 +4,7 @@ import sys
 from unittest.mock import patch
 
 import run_glue_with_pabee
-from transformers.testing_utils import TestCasePlus
+from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -20,6 +20,7 @@ def get_setup_file():
 
 
 class PabeeTests(TestCasePlus):
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)

examples/bertology/run_bertology.py

@@ -30,6 +30,7 @@ from torch.utils.data import DataLoader, SequentialSampler, Subset
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm
 
+import transformers
 from transformers import (
     AutoConfig,
     AutoModelForSequenceClassification,
@@ -41,6 +42,7 @@ from transformers import (
     glue_processors,
     set_seed,
 )
+from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -368,6 +370,11 @@ def main():
     # Setup logging
     logging.basicConfig(level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
     logger.info("device: {} n_gpu: {}, distributed: {}".format(args.device, args.n_gpu, bool(args.local_rank != -1)))
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
 
     # Set seeds
     set_seed(args.seed)

examples/contrib/legacy/run_language_modeling.py

@@ -29,6 +29,7 @@ from typing import Optional
 
 from torch.utils.data import ConcatDataset
 
+import transformers
 from transformers import (
     CONFIG_MAPPING,
     MODEL_WITH_LM_HEAD_MAPPING,
@@ -47,6 +48,7 @@ from transformers import (
     TrainingArguments,
     set_seed,
 )
+from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -219,6 +221,11 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(training_args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/contrib/mm-imdb/run_mmimdb.py

@@ -31,6 +31,7 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
+import transformers
 from transformers import (
     WEIGHTS_NAME,
     AdamW,
@@ -41,6 +42,7 @@ from transformers import (
     MMBTForClassification,
     get_linear_schedule_with_warmup,
 )
+from transformers.trainer_utils import is_main_process
 from utils_mmimdb import ImageEncoder, JsonlDataset, collate_fn, get_image_transforms, get_mmimdb_labels
 
 
@@ -476,7 +478,11 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     # Set seed
     set_seed(args)
 

examples/contrib/run_swag.py

@@ -31,8 +31,10 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
+import transformers
 from transformers import WEIGHTS_NAME, AdamW, AutoConfig, AutoTokenizer, get_linear_schedule_with_warmup
 from transformers.modeling_auto import AutoModelForMultipleChoice
+from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -620,6 +622,11 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
 
     # Set seed
     set_seed(args)

examples/deebert/run_glue_deebert.py

@@ -13,6 +13,7 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
+import transformers
 from src.modeling_highway_bert import DeeBertForSequenceClassification
 from src.modeling_highway_roberta import DeeRobertaForSequenceClassification
 from transformers import (
@@ -28,6 +29,7 @@ from transformers import glue_compute_metrics as compute_metrics
 from transformers import glue_convert_examples_to_features as convert_examples_to_features
 from transformers import glue_output_modes as output_modes
 from transformers import glue_processors as processors
+from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -580,7 +582,11 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     # Set seed
     set_seed(args)
 

examples/deebert/test_glue_deebert.py

@@ -5,7 +5,7 @@ import unittest
 from unittest.mock import patch
 
 import run_glue_deebert
-from transformers.testing_utils import slow
+from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -26,6 +26,7 @@ class DeeBertTests(unittest.TestCase):
         logger.addHandler(stream_handler)
 
     @slow
+    @require_torch_non_multi_gpu_but_fix_me
     def test_glue_deebert_train(self):
 
         train_args = """

examples/distillation/README.md

@@ -12,7 +12,7 @@ This folder contains the original code used to train Distil* as well as examples
 
 **October 3, 2019 - Update** We release our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108) explaining our approach on **DistilBERT**. It includes updated results and further experiments. We applied the same method to GPT2 and release the weights of **DistilGPT2**. DistilGPT2 is two times faster and 33% smaller than GPT2. **The paper supersedes our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. Please use the paper as a reference when comparing/reporting results on DistilBERT.**
 
-**September 19, 2019 - Update:** We fixed bugs in the code and released an upadted version of the weights trained with a modification of the distillation loss. DistilBERT now reaches 99% of `BERT-base`'s performance on GLUE, and 86.9 F1 score on SQuAD v1.1 dev set (compared to 88.5 for `BERT-base`). We will publish a formal write-up of our approach in the near future!
+**September 19, 2019 - Update:** We fixed bugs in the code and released an updated version of the weights trained with a modification of the distillation loss. DistilBERT now reaches 99% of `BERT-base`'s performance on GLUE, and 86.9 F1 score on SQuAD v1.1 dev set (compared to 88.5 for `BERT-base`). We will publish a formal write-up of our approach in the near future!
 
 
 ## What is Distil*

examples/distillation/run_squad_w_distillation.py

@@ -30,6 +30,7 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
+import transformers
 from transformers import (
     WEIGHTS_NAME,
     AdamW,
@@ -57,6 +58,7 @@ from transformers.data.metrics.squad_metrics import (
     squad_evaluate,
 )
 from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor
+from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -745,7 +747,11 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     # Set seed
     set_seed(args)
 

examples/language-modeling/README.md

@@ -10,7 +10,7 @@ These scripts leverage the 🤗 Datasets library and the Trainer API. You can ea
 need extra processing on your datasets.
 
 **Note:** The old script `run_language_modeling.py` is still available
-[here](https://github.com/huggingface/transformers/blob/master/examples/contrib/legacy/language-modeling/run_language_modeling.py).
+[here](https://github.com/huggingface/transformers/blob/master/examples/contrib/legacy/run_language_modeling.py).
 
 The following examples, will run on a datasets hosted on our [hub](https://huggingface.co/datasets) or with your own
 text files for training and validation. We give examples of both below.
@@ -68,13 +68,13 @@ python run_mlm.py \
 To run on your own training and validation files, use the following command:
 
 ```bash
-python run_clm.py \
+python run_mlm.py \
     --model_name_or_path roberta-base \
     --train_file path_to_train_file \
     --validation_file path_to_validation_file \
     --do_train \
     --do_eval \
-    --output_dir /tmp/test-clm
+    --output_dir /tmp/test-mlm
 ```
 
 If your dataset is organized with one sample per line, you can use the `--line_by_line` flag (otherwise the script

examples/language-modeling/run_clm.py

@@ -168,6 +168,8 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -190,7 +192,7 @@ def main():
         if data_args.train_file is not None:
             data_files["train"] = data_args.train_file
         if data_args.validation_file is not None:
-            data_files["validation"] = data_args.train_file
+            data_files["validation"] = data_args.validation_file
         extension = data_args.train_file.split(".")[-1]
         if extension == "txt":
             extension = "text"
@@ -254,7 +256,7 @@ def main():
         tokenize_function,
         batched=True,
         num_proc=data_args.preprocessing_num_workers,
-        remove_columns=[text_column_name],
+        remove_columns=column_names,
         load_from_cache_file=not data_args.overwrite_cache,
     )
 

examples/language-modeling/run_mlm.py

@@ -179,6 +179,8 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -201,7 +203,7 @@ def main():
         if data_args.train_file is not None:
             data_files["train"] = data_args.train_file
         if data_args.validation_file is not None:
-            data_files["validation"] = data_args.train_file
+            data_files["validation"] = data_args.validation_file
         extension = data_args.train_file.split(".")[-1]
         if extension == "txt":
             extension = "text"
@@ -264,7 +266,15 @@ def main():
         def tokenize_function(examples):
             # Remove empty lines
             examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
-            return tokenizer(examples["text"], padding=padding, truncation=True, max_length=data_args.max_seq_length)
+            return tokenizer(
+                examples["text"],
+                padding=padding,
+                truncation=True,
+                max_length=data_args.max_seq_length,
+                # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it
+                # receives the `special_tokens_mask`.
+                return_special_tokens_mask=True,
+            )
 
         tokenized_datasets = datasets.map(
             tokenize_function,
@@ -275,14 +285,16 @@ def main():
         )
     else:
         # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts.
+        # We use `return_special_tokens_mask=True` because DataCollatorForLanguageModeling (see below) is more
+        # efficient when it receives the `special_tokens_mask`.
         def tokenize_function(examples):
-            return tokenizer(examples[text_column_name])
+            return tokenizer(examples[text_column_name], return_special_tokens_mask=True)
 
         tokenized_datasets = datasets.map(
             tokenize_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
+            remove_columns=column_names,
             load_from_cache_file=not data_args.overwrite_cache,
         )
 

examples/language-modeling/run_mlm_wwm.py

@@ -186,6 +186,8 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -204,7 +206,7 @@ def main():
     if data_args.train_file is not None:
         data_files["train"] = data_args.train_file
     if data_args.validation_file is not None:
-        data_files["validation"] = data_args.train_file
+        data_files["validation"] = data_args.validation_file
     extension = data_args.train_file.split(".")[-1]
     if extension == "txt":
         extension = "text"

examples/language-modeling/run_plm.py

@@ -176,6 +176,8 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed before initializing model.
@@ -198,7 +200,7 @@ def main():
         if data_args.train_file is not None:
             data_files["train"] = data_args.train_file
         if data_args.validation_file is not None:
-            data_files["validation"] = data_args.train_file
+            data_files["validation"] = data_args.validation_file
         extension = data_args.train_file.split(".")[-1]
         if extension == "txt":
             extension = "text"
@@ -279,7 +281,7 @@ def main():
             tokenize_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
-            remove_columns=[text_column_name],
+            remove_columns=column_names,
             load_from_cache_file=not data_args.overwrite_cache,
         )
 

examples/lightning_base.py

@@ -4,6 +4,7 @@ import os
 from pathlib import Path
 from typing import Any, Dict
 
+import packaging
 import pytorch_lightning as pl
 from pytorch_lightning.utilities import rank_zero_info
 
@@ -33,15 +34,17 @@ from transformers.optimization import (
 
 logger = logging.getLogger(__name__)
 
-try:
-    pkg = "pytorch_lightning"
-    min_ver = "1.0.4"
-    pkg_resources.require(f"{pkg}>={min_ver}")
-except pkg_resources.VersionConflict:
-    logger.warning(
-        f"{pkg}>={min_ver} is required for a normal functioning of this module, but found {pkg}=={pkg_resources.get_distribution(pkg).version}. Try pip install -r examples/requirements.txt"
-    )
 
+def require_min_ver(pkg, min_ver):
+    got_ver = pkg_resources.get_distribution(pkg).version
+    if packaging.version.parse(got_ver) < packaging.version.parse(min_ver):
+        logger.warning(
+            f"{pkg}>={min_ver} is required for a normal functioning of this module, but found {pkg}=={got_ver}. "
+            "Try: pip install -r examples/requirements.txt"
+        )
+
+
+require_min_ver("pytorch_lightning", "1.0.4")
 
 MODEL_MODES = {
     "base": AutoModel,

examples/lxmert/modeling_frcnn.py

@@ -1801,7 +1801,7 @@ class GeneralizedRCNN(nn.Module):
                 f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when "
                 f"initializing {model.__class__.__name__}: {unexpected_keys}\n"
                 f"- This IS expected if you are initializing {model.__class__.__name__} from the checkpoint of a model trained on another task "
-                f"or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n"
+                f"or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n"
                 f"- This IS NOT expected if you are initializing {model.__class__.__name__} from the checkpoint of a model that you expect "
                 f"to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model)."
             )

examples/multiple-choice/run_multiple_choice.py

@@ -23,6 +23,7 @@ from typing import Dict, Optional
 
 import numpy as np
 
+import transformers
 from transformers import (
     AutoConfig,
     AutoModelForMultipleChoice,
@@ -33,6 +34,7 @@ from transformers import (
     TrainingArguments,
     set_seed,
 )
+from transformers.trainer_utils import is_main_process
 from utils_multiple_choice import MultipleChoiceDataset, Split, processors
 
 
@@ -115,6 +117,11 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(training_args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/multiple-choice/run_tf_multiple_choice.py

@@ -33,9 +33,15 @@ from transformers import (
     TFTrainingArguments,
     set_seed,
 )
+from transformers.utils import logging as hf_logging
 from utils_multiple_choice import Split, TFMultipleChoiceDataset, processors
 
 
+hf_logging.set_verbosity_info()
+hf_logging.enable_default_handler()
+hf_logging.enable_explicit_format()
+
+
 logger = logging.getLogger(__name__)
 
 

examples/question-answering/run_squad.py

@@ -29,6 +29,7 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
+import transformers
 from transformers import (
     MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     WEIGHTS_NAME,
@@ -45,6 +46,7 @@ from transformers.data.metrics.squad_metrics import (
     squad_evaluate,
 )
 from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor
+from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -712,7 +714,11 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     # Set seed
     set_seed(args)
 
@@ -730,6 +736,7 @@ def main():
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,
         cache_dir=args.cache_dir if args.cache_dir else None,
+        use_fast=False,  # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
     )
     model = AutoModelForQuestionAnswering.from_pretrained(
         args.model_name_or_path,
@@ -778,7 +785,10 @@ def main():
 
         # Load a trained model and vocabulary that you have fine-tuned
         model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)  # , force_download=True)
-        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+
+        # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
+        # So we use use_fast=False here for now until Fast-tokenizer-compatible-examples are out
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case, use_fast=False)
         model.to(args.device)
 
     # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory

examples/question-answering/run_squad_trainer.py

@@ -22,9 +22,11 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional
 
+import transformers
 from transformers import AutoConfig, AutoModelForQuestionAnswering, AutoTokenizer, HfArgumentParser, SquadDataset
 from transformers import SquadDataTrainingArguments as DataTrainingArguments
 from transformers import Trainer, TrainingArguments
+from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -91,6 +93,11 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(training_args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Prepare Question-Answering task
@@ -107,6 +114,7 @@ def main():
     tokenizer = AutoTokenizer.from_pretrained(
         model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
         cache_dir=model_args.cache_dir,
+        use_fast=False,  # SquadDataset is not compatible with Fast tokenizers which have a smarter overflow handeling
     )
     model = AutoModelForQuestionAnswering.from_pretrained(
         model_args.model_name_or_path,

examples/question-answering/run_tf_squad.py

@@ -33,6 +33,12 @@ from transformers import (
     squad_convert_examples_to_features,
 )
 from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor
+from transformers.utils import logging as hf_logging
+
+
+hf_logging.set_verbosity_info()
+hf_logging.enable_default_handler()
+hf_logging.enable_explicit_format()
 
 
 logger = logging.getLogger(__name__)

examples/rag/test_distributed_retriever.py

@@ -16,6 +16,7 @@ from transformers.configuration_dpr import DPRConfig
 from transformers.configuration_rag import RagConfig
 from transformers.file_utils import is_datasets_available, is_faiss_available, is_psutil_available, is_torch_available
 from transformers.retrieval_rag import CustomHFIndex
+from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me
 from transformers.tokenization_bart import BartTokenizer
 from transformers.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
 from transformers.tokenization_dpr import DPRQuestionEncoderTokenizer
@@ -178,6 +179,7 @@ class RagRetrieverTest(TestCase):
             retriever.init_retrieval(port)
         return retriever
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_pytorch_distributed_retriever_retrieve(self):
         n_docs = 1
         retriever = self.get_dummy_pytorch_distributed_retriever(init_retrieval=True)
@@ -193,6 +195,7 @@ class RagRetrieverTest(TestCase):
         self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
         self.assertListEqual(doc_ids.tolist(), [[1], [0]])
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_custom_hf_index_retriever_retrieve(self):
         n_docs = 1
         retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=False)
@@ -208,6 +211,7 @@ class RagRetrieverTest(TestCase):
         self.assertEqual(doc_dicts[1]["id"][0], "0")  # max inner product is reached with first doc
         self.assertListEqual(doc_ids.tolist(), [[1], [0]])
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_custom_pytorch_distributed_retriever_retrieve_from_disk(self):
         n_docs = 1
         retriever = self.get_dummy_custom_hf_index_retriever(init_retrieval=True, from_disk=True)

examples/requirements.txt

@@ -18,3 +18,4 @@ fire
 pytest
 conllu
 sentencepiece != 0.1.92
+protobuf

examples/seq2seq/README.md

@@ -1,7 +1,7 @@
 ## Sequence to Sequence Training and Evaluation
 
 This directory contains examples for finetuning and evaluating transformers on summarization and translation tasks.
-Please tag @sshleifer with any issues/unexpected behaviors, or send a PR!
+Please tag @patil-suraj with any issues/unexpected behaviors, or send a PR!
 For deprecated `bertabs` instructions, see [`bertabs/README.md`](bertabs/README.md).
 
 ### Supported Architectures
@@ -380,7 +380,7 @@ cp xsum/test* all_pl
 then use `all_pl` as DATA in the command above.
 
 #### Direct Knowledge Distillation (KD)
-+ In this method, we use try to enforce that the student and teacher produce similar encoder_outputs, logits, and hidden_states using `BartSummarizationDistiller`.
++ In this method, we use try to enforce that the student and teacher produce similar encoder_outputs, logits, and hidden_states using `SummarizationDistiller`.
 + This method was used for `sshleifer/distilbart-xsum-12-6`, `6-6`, and `9-6` checkpoints were produced.
 + You must use [`distillation.py`](./distillation.py). Note that this command initializes the student for you.
 

examples/seq2seq/bertabs/configuration_bertabs.py

@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 BERTABS_FINETUNED_CONFIG_MAP = {
-    "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization/config.json",
+    "bertabs-finetuned-cnndm": "https://huggingface.co/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization/resolve/main/config.json",
 }
 
 

examples/seq2seq/distillation.py

@@ -25,8 +25,8 @@ sys.path.insert(2, str(Path(__file__).resolve().parents[1]))
 from lightning_base import generic_train  # noqa
 
 
-class BartSummarizationDistiller(SummarizationModule):
+class SummarizationDistiller(SummarizationModule):
-    """Supports Bart, Pegasus and other models that inherit from Bart."""
+    """Supports T5, Bart, Pegasus and other models that inherit from Bart."""
 
     loss_names = ["loss", "ce_loss", "mlm_loss", "hid_loss_enc", "hid_loss_dec"]
 
@@ -40,25 +40,38 @@ class BartSummarizationDistiller(SummarizationModule):
         hparams.model_name_or_path = str(save_dir)  # Tell lightning we are training the student
         teacher = AutoModelForSeq2SeqLM.from_pretrained(hparams.teacher).eval()
         use_task_specific_params(teacher, hparams.task)  # We copy good generation parameters to student by default
-        student, e_layer_ids, d_layer_ids = create_student_by_copying_alternating_layers(
+        if hparams.student is not None:
-            teacher, e=hparams.student_encoder_layers, d=hparams.student_decoder_layers, save_path=save_dir
+            student = AutoModelForSeq2SeqLM.from_pretrained(hparams.student)
-        )
+            use_task_specific_params(student, hparams.task)
+            e_layer_ids, d_layer_ids = None, None
+        else:
+            student, e_layer_ids, d_layer_ids = create_student_by_copying_alternating_layers(
+                teacher, e=hparams.student_encoder_layers, d=hparams.student_decoder_layers, save_path=save_dir
+            )
+
         if hparams.length_penalty != -1:
             student.config.length_penalty = hparams.length_penalty
+        hparams.tokenizer_name = hparams.teacher  # Use teacher's tokenizer
         super().__init__(hparams, model=student, config=student.config)
-        model_type = student.config.model_type
+        assert (
-        self.e_layer_ids, self.d_layer_ids = e_layer_ids, d_layer_ids  # type: List[int], List[int]
+            student.config.model_type == teacher.config.model_type
+        ), f"teacher, student model types should be the same, got {student.config.model_type} != {teacher.config.model_type}"
 
-        if model_type == "t5":
+        if student.config.model_type == "t5":
+            student_encoder_layers = len(student.get_encoder().block)
+            student_decoder_layers = len(student.get_decoder().block)
             teacher_encoder_layers = len(teacher.get_encoder().block)
             teacher_decoder_layers = len(teacher.get_decoder().block)
         else:
+            student_encoder_layers = student.config.encoder_layers
+            student_decoder_layers = student.config.decoder_layers
             teacher_encoder_layers = teacher.config.encoder_layers
             teacher_decoder_layers = teacher.config.decoder_layers
 
-        self.different_encoder = hparams.student_encoder_layers != teacher_encoder_layers
+        self.different_base_models = not (hparams.student is None or hparams.teacher == hparams.student)
-        self.different_decoder = hparams.student_decoder_layers != teacher_decoder_layers
+        self.do_calc_hidden_loss = (not self.different_base_models) and hparams.alpha_hid > 0
-
+        self.different_encoder = self.different_base_models or (student_encoder_layers != teacher_encoder_layers)
+        # self.different_encoder determines whether we need to run the teacher encoder
         self.teacher = teacher
         freeze_params(self.teacher)
 
@@ -67,13 +80,28 @@ class BartSummarizationDistiller(SummarizationModule):
                 del self.teacher.model.encoder
             except AttributeError:  # T5
                 del self.teacher.encoder
-        # Intermediate supervision: Decide which layers to supervise
+
-        if hparams.supervise_forward:
+        if e_layer_ids is None:
-            self.e_matches = get_layers_to_supervise(n_student=len(self.e_layer_ids), n_teacher=teacher_encoder_layers)
+            e_layer_ids = list(range(student_encoder_layers))
-            self.d_matches = get_layers_to_supervise(n_student=len(self.d_layer_ids), n_teacher=teacher_decoder_layers)
+        if d_layer_ids is None:
-        else:  # student layer should emulate hidden states of the teacher layer it was copied from
+            d_layer_ids = list(range(student_decoder_layers))
-            self.e_matches = self.e_layer_ids
+
-            self.d_matches = self.d_layer_ids
+        self.e_layer_ids, self.d_layer_ids = e_layer_ids, d_layer_ids  # type: List[int], List[int]
+
+        if self.do_calc_hidden_loss:  # Intermediate supervision: Decide which layers to supervise
+            if hparams.supervise_forward:
+                self.e_matches = get_layers_to_supervise(
+                    n_student=len(self.e_layer_ids), n_teacher=teacher_encoder_layers
+                )
+                self.d_matches = get_layers_to_supervise(
+                    n_student=len(self.d_layer_ids), n_teacher=teacher_decoder_layers
+                )
+            else:  # student layer should emulate hidden states of the teacher layer it was copied from
+                self.e_matches = self.e_layer_ids
+                self.d_matches = self.d_layer_ids
+        else:
+            self.e_matches = None
+            self.d_matches = None
 
         self.ce_loss_fct = nn.KLDivLoss(reduction="batchmean")
         self.temperature = 2.0
@@ -83,22 +111,8 @@ class BartSummarizationDistiller(SummarizationModule):
         gc.collect()
         torch.cuda.empty_cache()
 
-    def calc_mse_loss(self, teacher_outputs: torch.Tensor, student_outputs: torch.Tensor, mask) -> torch.FloatTensor:
-        """Supervise MSE(teacher.encoder_outputs, student.encoder_outputs)."""
-        # raise NotImplementedError()
-        if mask is not None:
-            # mask has False at padding_idx
-            sel_mask = mask[:, :, None].expand_as(student_outputs).bool()
-            s_logits_slct = torch.masked_select(student_outputs, sel_mask)
-            t_logits_slct = torch.masked_select(teacher_outputs, sel_mask)
-        else:
-            t_logits_slct = teacher_outputs
-            s_logits_slct = student_outputs
-        return F.mse_loss(s_logits_slct, t_logits_slct)
-
     def calc_ce_loss(self, mask, s_logits, t_logits):
         """Copy pasted from distillbert (transformers/examples/distillation/)"""
-
         # mask has False at padding_idx
         sel_mask = mask[:, :, None].expand_as(s_logits)
         vocab_size = s_logits.size(-1)
@@ -122,8 +136,8 @@ class BartSummarizationDistiller(SummarizationModule):
         add_distill_args(parser)
         return parser
 
-    def _step(self, batch):
+    def _step(self, batch: dict) -> tuple:
-        # assert is_frozen(self.teacher) copied_decoder_layers
+        """Compute the loss for a batch"""
         pad_token_id = self.tokenizer.pad_token_id
         input_ids, src_mask, labels = batch["input_ids"], batch["attention_mask"], batch["labels"]
         if isinstance(self.model, T5ForConditionalGeneration):
@@ -132,14 +146,16 @@ class BartSummarizationDistiller(SummarizationModule):
             decoder_input_ids = shift_tokens_right(labels, pad_token_id)
 
         # noinspection PyCallingNonCallable
-        lm_logits, dec_hidden, enc_outputs, enc_hidden_state = self(
+        student_outputs = self(
             input_ids,
             attention_mask=src_mask,
             decoder_input_ids=decoder_input_ids,
-            output_hidden_states=True,
+            output_hidden_states=self.do_calc_hidden_loss,
             output_attentions=False,
             use_cache=False,
+            return_dict=True,
         )
+        lm_logits = student_outputs.logits
 
         # Same cross entropy vs. label smoothing logic as finetune.py
         assert lm_logits.shape[-1] == self.model.config.vocab_size
@@ -148,7 +164,7 @@ class BartSummarizationDistiller(SummarizationModule):
             loss_fct = torch.nn.CrossEntropyLoss(ignore_index=pad_token_id)
             student_lm_loss = loss_fct(lm_logits.view(-1, lm_logits.shape[-1]), labels.view(-1))
         else:
-            lprobs = torch.nn.functional.log_softmax(lm_logits, dim=-1)
+            lprobs = F.log_softmax(lm_logits, dim=-1)
             student_lm_loss, _ = label_smoothed_nll_loss(
                 lprobs, labels, self.hparams.label_smoothing, ignore_index=pad_token_id
             )
@@ -156,37 +172,44 @@ class BartSummarizationDistiller(SummarizationModule):
         def zero_tensor():
             return torch.tensor(0.0).type_as(student_lm_loss)
 
+        teacher_enc_outputs = student_outputs.encoder_last_hidden_state  # use this unless self.different_base_models
         hid_loss_enc, hid_loss_dec = zero_tensor(), zero_tensor()
         if self.different_encoder:  # compute encoder hidden state loss
-            with torch.no_grad():
+            all_teacher_encoder_outputs = self.teacher.get_encoder()(
-                teacher_enc_hid = self.teacher.get_encoder()(
-                    input_ids, attention_mask=src_mask, output_hidden_states=True, return_dict=True
-                ).hidden_states
-
-            hid_loss_enc = self.calc_hidden_loss(
-                src_mask,
-                enc_hidden_state,
-                teacher_enc_hid,
-                self.e_matches,
-                normalize_hidden=self.hparams.normalize_hidden,
-            )
-
-        with torch.no_grad():
-            outputs = self.teacher(
                 input_ids,
                 attention_mask=src_mask,
-                encoder_outputs=(enc_outputs,),
+                output_hidden_states=self.do_calc_hidden_loss,
-                decoder_input_ids=decoder_input_ids,
-                lm_labels=labels,
-                output_hidden_states=True,
                 return_dict=True,
             )
-            tlogits, tdec_hidden = outputs.logits, outputs.decoder_hidden_states
+            if self.different_base_models:
+                teacher_enc_outputs = all_teacher_encoder_outputs.last_hidden_state
+            elif self.do_calc_hidden_loss:
+                hid_loss_enc = self.calc_hidden_loss(
+                    src_mask,
+                    student_outputs.encoder_hidden_states,
+                    all_teacher_encoder_outputs.hidden_states,
+                    self.e_matches,
+                    normalize_hidden=self.hparams.normalize_hidden,
+                )
+
+        teacher_outputs = self.teacher(
+            input_ids,
+            attention_mask=src_mask,
+            encoder_outputs=(teacher_enc_outputs,),
+            decoder_input_ids=decoder_input_ids,
+            output_hidden_states=self.do_calc_hidden_loss,
+            use_cache=False,  # since we are not passing labels, never let this default to True
+            return_dict=True,
+        )
         dec_mask = decoder_input_ids.ne(pad_token_id)
-        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, tlogits)
+        loss_ce = self.calc_ce_loss(dec_mask, lm_logits, teacher_outputs.logits)
-        if self.alpha_hid > 0:  # Intermediate supervision of decoder hidden states
+        if self.do_calc_hidden_loss:  # Intermediate supervision of decoder hidden states
             hid_loss_dec = self.calc_hidden_loss(
-                dec_mask, dec_hidden, tdec_hidden, self.d_matches, normalize_hidden=self.hparams.normalize_hidden
+                dec_mask,
+                student_outputs.decoder_hidden_states,
+                teacher_outputs.decoder_hidden_states,
+                self.d_matches,
+                normalize_hidden=self.hparams.normalize_hidden,
             )
 
         blended_loss = (
@@ -206,6 +229,7 @@ class BartSummarizationDistiller(SummarizationModule):
         valid_count = mask.sum() * hidden_states[0].size(-1)
         student_states = torch.stack([hidden_states[i] for i in range(len(matches))])
         teacher_states = torch.stack([hidden_states_T[j] for j in matches])
+        assert student_states.shape == teacher_states.shape, f"{student_states.shape} != {teacher_states.shape}"
         if normalize_hidden:
             student_states = F.layer_norm(student_states, student_states.shape[1:])
             teacher_states = F.layer_norm(teacher_states, teacher_states.shape[1:])
@@ -215,10 +239,16 @@ class BartSummarizationDistiller(SummarizationModule):
 
 
 def add_distill_args(parser):
+    # NOTE: if --student argument was specified and the teacher and student base models
+    # are different, the models still have to have the same tokenizer, specified by
+    # --tokenizer_name. So, for example, you can distill from t5_large to t5_small but not
+    # from bart to t5. This s because if the tokenizers are different, the output space
+    # for the two models is also different and their logits are not comparable.
     parser.add_argument("--teacher", type=str)
     parser.add_argument("--alpha_ce", default=0.8, type=float)
     parser.add_argument("--alpha_mlm", default=0.2, type=float)
     parser.add_argument("--alpha_hid", default=0.0, type=float, required=False)
+    parser.add_argument("--student", type=str, required=False)
     parser.add_argument("--student_decoder_layers", default=12, type=int, required=False)
     parser.add_argument("--student_encoder_layers", default=12, type=int, required=False)
     parser.add_argument("--no_teacher", action="store_true", default=False)
@@ -227,8 +257,8 @@ def add_distill_args(parser):
     parser.add_argument("--normalize_hidden", action="store_true", default=False)
 
 
-class BartTranslationDistiller(BartSummarizationDistiller):
+class TranslationDistiller(SummarizationDistiller):
-    """Supports Mbart, Marian, other models that inherit from Bart."""
+    """Supports T5, mBART, Marian, other models that inherit from Bart."""
 
     mode = "translation"
     metric_names = ["bleu"]
@@ -257,7 +287,7 @@ def create_module(args):
     if args.no_teacher:
         module_cls = TranslationModule if "translation" in args.task else SummarizationModule
     else:  # DISTILL WITH TEACHER
-        module_cls = BartTranslationDistiller if "translation" in args.task else BartSummarizationDistiller
+        module_cls = TranslationDistiller if "translation" in args.task else SummarizationDistiller
     args.setup_cls: str = module_cls.__name__
     print(f"using module {args.setup_cls}")
     model = module_cls(args)
@@ -275,7 +305,7 @@ def distill_main(args):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser = pl.Trainer.add_argparse_args(parser)
-    parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+    parser = SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
     args = parser.parse_args()
 
     distill_main(args)

examples/seq2seq/finetune_trainer.py

@@ -4,10 +4,11 @@ import sys
 from dataclasses import dataclass, field
 from typing import Optional
 
+import transformers
 from seq2seq_trainer import Seq2SeqTrainer
 from seq2seq_training_args import Seq2SeqTrainingArguments
 from transformers import AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer, HfArgumentParser, MBartTokenizer, set_seed
-from transformers.trainer_utils import EvaluationStrategy
+from transformers.trainer_utils import EvaluationStrategy, is_main_process
 from utils import (
     Seq2SeqDataCollator,
     Seq2SeqDataset,
@@ -131,6 +132,11 @@ def main():
         bool(training_args.local_rank != -1),
         training_args.fp16,
     )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(training_args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
     # Set seed

examples/seq2seq/run_distiller.sh

@@ -1,10 +0,0 @@
-# Add parent directory to python path to access lightning_base.py
-export PYTHONPATH="../":"${PYTHONPATH}"
-
-python distillation.py \
---learning_rate=3e-4 \
---do_train \
---do_predict \
---fp16 \
---val_check_interval 0.1 \
-"$@"

examples/seq2seq/test_bash_script.py

@@ -3,93 +3,110 @@
 import argparse
 import os
 import sys
-from pathlib import Path
 from unittest.mock import patch
 
-import pytest
 import pytorch_lightning as pl
 import timeout_decorator
 import torch
 
-from distillation import BartSummarizationDistiller, distill_main
+from distillation import SummarizationDistiller, distill_main
 from finetune import SummarizationModule, main
-from test_seq2seq_examples import CUDA_AVAILABLE, MBART_TINY
+from transformers import MarianMTModel
-from transformers import BartForConditionalGeneration, MarianMTModel
+from transformers.file_utils import cached_path
-from transformers.testing_utils import TestCasePlus, slow
+from transformers.testing_utils import TestCasePlus, require_torch_gpu, require_torch_non_multi_gpu_but_fix_me, slow
 from utils import load_json
 
 
-MODEL_NAME = MBART_TINY
+MARIAN_MODEL = "sshleifer/mar_enro_6_3_student"
-MARIAN_MODEL = "sshleifer/student_marian_en_ro_6_1"
 
 
-class TestAll(TestCasePlus):
+class TestMbartCc25Enro(TestCasePlus):
+    def setUp(self):
+        super().setUp()
+
+        data_cached = cached_path(
+            "https://cdn-datasets.huggingface.co/translation/wmt_en_ro-tr40k-va0.5k-te0.5k.tar.gz",
+            extract_compressed_file=True,
+        )
+        self.data_dir = f"{data_cached}/wmt_en_ro-tr40k-va0.5k-te0.5k"
+
     @slow
-    @pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
+    @require_torch_gpu
+    @require_torch_non_multi_gpu_but_fix_me
     def test_model_download(self):
         """This warms up the cache so that we can time the next test without including download time, which varies between machines."""
-        BartForConditionalGeneration.from_pretrained(MODEL_NAME)
         MarianMTModel.from_pretrained(MARIAN_MODEL)
 
-    @timeout_decorator.timeout(120)
+    # @timeout_decorator.timeout(1200)
     @slow
-    @pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
+    @require_torch_gpu
+    @require_torch_non_multi_gpu_but_fix_me
     def test_train_mbart_cc25_enro_script(self):
-        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
         env_vars_to_replace = {
-            "--fp16_opt_level=O1": "",
+            "$MAX_LEN": 64,
-            "$MAX_LEN": 128,
+            "$BS": 64,
-            "$BS": 4,
             "$GAS": 1,
-            "$ENRO_DIR": data_dir,
+            "$ENRO_DIR": self.data_dir,
-            "facebook/mbart-large-cc25": MODEL_NAME,
+            "facebook/mbart-large-cc25": MARIAN_MODEL,
-            # Download is 120MB in previous test.
+            # "val_check_interval=0.25": "val_check_interval=1.0",
-            "val_check_interval=0.25": "val_check_interval=1.0",
+            "--learning_rate=3e-5": "--learning_rate 3e-4",
+            "--num_train_epochs 6": "--num_train_epochs 1",
         }
 
         # Clean up bash script
-        bash_script = Path("examples/seq2seq/train_mbart_cc25_enro.sh").open().read().split("finetune.py")[1].strip()
+        bash_script = (self.test_file_dir / "train_mbart_cc25_enro.sh").open().read().split("finetune.py")[1].strip()
         bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
         for k, v in env_vars_to_replace.items():
             bash_script = bash_script.replace(k, str(v))
         output_dir = self.get_auto_remove_tmp_dir()
 
-        bash_script = bash_script.replace("--fp16 ", "")
+        # bash_script = bash_script.replace("--fp16 ", "")
-        testargs = (
+        args = f"""
-            ["finetune.py"]
+            --output_dir {output_dir}
-            + bash_script.split()
+            --tokenizer_name Helsinki-NLP/opus-mt-en-ro
-            + [
+            --sortish_sampler
-                f"--output_dir={output_dir}",
+            --do_predict
-                "--gpus=1",
+            --gpus 1
-                "--learning_rate=3e-1",
+            --freeze_encoder
-                "--warmup_steps=0",
+            --n_train 40000
-                "--val_check_interval=1.0",
+            --n_val 500
-                "--tokenizer_name=facebook/mbart-large-en-ro",
+            --n_test 500
-            ]
+            --fp16_opt_level O1
-        )
+            --num_sanity_val_steps 0
+            --eval_beams 2
+        """.split()
+        # XXX: args.gpus > 1 : handle multi_gpu in the future
+
+        testargs = ["finetune.py"] + bash_script.split() + args
         with patch.object(sys, "argv", testargs):
             parser = argparse.ArgumentParser()
             parser = pl.Trainer.add_argparse_args(parser)
             parser = SummarizationModule.add_model_specific_args(parser, os.getcwd())
             args = parser.parse_args()
-            args.do_predict = False
-            # assert args.gpus == gpus THIS BREAKS for multigpu
             model = main(args)
 
         # Check metrics
         metrics = load_json(model.metrics_save_path)
         first_step_stats = metrics["val"][0]
         last_step_stats = metrics["val"][-1]
-        assert (
+        self.assertEqual(len(metrics["val"]), (args.max_epochs / args.val_check_interval))
-            len(metrics["val"]) == (args.max_epochs / args.val_check_interval) + 1
-        )  # +1 accounts for val_sanity_check
-
-        assert last_step_stats["val_avg_gen_time"] >= 0.01
-
-        assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
-        assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
         assert isinstance(last_step_stats[f"val_avg_{model.val_metric}"], float)
 
+        self.assertGreater(last_step_stats["val_avg_gen_time"], 0.01)
+        # model hanging on generate. Maybe bad config was saved. (XXX: old comment/assert?)
+        self.assertLessEqual(last_step_stats["val_avg_gen_time"], 1.0)
+
+        # test learning requirements:
+
+        # 1. BLEU improves over the course of training by more than 2 pts
+        self.assertGreater(last_step_stats["val_avg_bleu"] - first_step_stats["val_avg_bleu"], 2)
+
+        # 2. BLEU finishes above 17
+        self.assertGreater(last_step_stats["val_avg_bleu"], 17)
+
+        # 3. test BLEU and val BLEU within ~1.1 pt.
+        self.assertLess(abs(metrics["val"][-1]["val_avg_bleu"] - metrics["test"][-1]["test_avg_bleu"]), 1.1)
+
         # check lightning ckpt can be loaded and has a reasonable statedict
         contents = os.listdir(output_dir)
         ckpt_path = [x for x in contents if x.endswith(".ckpt")][0]
@@ -107,11 +124,14 @@ class TestAll(TestCasePlus):
             # assert len(metrics["val"]) ==  desired_n_evals
             assert len(metrics["test"]) == 1
 
+
+class TestDistilMarianNoTeacher(TestCasePlus):
     @timeout_decorator.timeout(600)
     @slow
-    @pytest.mark.skipif(not CUDA_AVAILABLE, reason="too slow to run on CPU")
+    @require_torch_gpu
+    @require_torch_non_multi_gpu_but_fix_me
     def test_opus_mt_distill_script(self):
-        data_dir = "examples/seq2seq/test_data/wmt_en_ro"
+        data_dir = f"{self.test_file_dir_str}/test_data/wmt_en_ro"
         env_vars_to_replace = {
             "--fp16_opt_level=O1": "",
             "$MAX_LEN": 128,
@@ -124,7 +144,7 @@ class TestAll(TestCasePlus):
 
         # Clean up bash script
         bash_script = (
-            Path("examples/seq2seq/distil_marian_no_teacher.sh").open().read().split("distillation.py")[1].strip()
+            (self.test_file_dir / "distil_marian_no_teacher.sh").open().read().split("distillation.py")[1].strip()
         )
         bash_script = bash_script.replace("\\\n", "").strip().replace('"$@"', "")
         bash_script = bash_script.replace("--fp16 ", " ")
@@ -150,9 +170,9 @@ class TestAll(TestCasePlus):
         with patch.object(sys, "argv", testargs):
             parser = argparse.ArgumentParser()
             parser = pl.Trainer.add_argparse_args(parser)
-            parser = BartSummarizationDistiller.add_model_specific_args(parser, os.getcwd())
+            parser = SummarizationDistiller.add_model_specific_args(parser, os.getcwd())
             args = parser.parse_args()
-            # assert args.gpus == gpus THIS BREAKS for multigpu
+            # assert args.gpus == gpus THIS BREAKS for multi_gpu
 
             model = distill_main(args)
 

examples/seq2seq/test_datasets.py

@@ -11,7 +11,7 @@ from save_len_file import save_len_file
 from test_seq2seq_examples import ARTICLES, BART_TINY, MARIAN_TINY, MBART_TINY, SUMMARIES, T5_TINY, make_test_data_dir
 from transformers import AutoTokenizer
 from transformers.modeling_bart import shift_tokens_right
-from transformers.testing_utils import TestCasePlus, slow
+from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me, slow
 from utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeq2SeqDataset, Seq2SeqDataset
 
 
@@ -30,6 +30,7 @@ class TestAll(TestCasePlus):
         ],
     )
     @slow
+    @require_torch_non_multi_gpu_but_fix_me
     def test_seq2seq_dataset_truncation(self, tok_name):
         tokenizer = AutoTokenizer.from_pretrained(tok_name)
         tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
@@ -69,6 +70,7 @@ class TestAll(TestCasePlus):
             break  # No need to test every batch
 
     @parameterized.expand([BART_TINY, BERT_BASE_CASED])
+    @require_torch_non_multi_gpu_but_fix_me
     def test_legacy_dataset_truncation(self, tok):
         tokenizer = AutoTokenizer.from_pretrained(tok)
         tmp_dir = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir())
@@ -93,6 +95,7 @@ class TestAll(TestCasePlus):
             assert max_len_target > trunc_target  # Truncated
             break  # No need to test every batch
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_pack_dataset(self):
         tokenizer = AutoTokenizer.from_pretrained("facebook/mbart-large-cc25")
 
@@ -111,6 +114,7 @@ class TestAll(TestCasePlus):
         assert orig_paths == new_paths
 
     @pytest.mark.skipif(not FAIRSEQ_AVAILABLE, reason="This test requires fairseq")
+    @require_torch_non_multi_gpu_but_fix_me
     def test_dynamic_batch_size(self):
         if not FAIRSEQ_AVAILABLE:
             return
@@ -135,6 +139,7 @@ class TestAll(TestCasePlus):
         if failures:
             raise AssertionError(f"too many tokens in {len(failures)} batches")
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_sortish_sampler_reduces_padding(self):
         ds, _, tokenizer = self._get_dataset(max_len=512)
         bs = 2
@@ -174,6 +179,7 @@ class TestAll(TestCasePlus):
         )
         return ds, max_tokens, tokenizer
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_distributed_sortish_sampler_splits_indices_between_procs(self):
         ds, max_tokens, tokenizer = self._get_dataset()
         ids1 = set(DistributedSortishSampler(ds, 256, num_replicas=2, rank=0, add_extra_examples=False))
@@ -189,8 +195,9 @@ class TestAll(TestCasePlus):
             PEGASUS_XSUM,
         ],
     )
+    @require_torch_non_multi_gpu_but_fix_me
     def test_dataset_kwargs(self, tok_name):
-        tokenizer = AutoTokenizer.from_pretrained(tok_name)
+        tokenizer = AutoTokenizer.from_pretrained(tok_name, use_fast=False)
         if tok_name == MBART_TINY:
             train_dataset = Seq2SeqDataset(
                 tokenizer,

examples/seq2seq/test_finetune_trainer.py

@@ -2,9 +2,9 @@ import os
 import sys
 from unittest.mock import patch
 
-from transformers import BertTokenizer, EncoderDecoderModel, is_torch_available
+from transformers import BertTokenizer, EncoderDecoderModel
 from transformers.file_utils import is_datasets_available
-from transformers.testing_utils import TestCasePlus, execute_subprocess_async, slow
+from transformers.testing_utils import TestCasePlus, execute_subprocess_async, get_gpu_count, slow
 from transformers.trainer_callback import TrainerState
 from transformers.trainer_utils import set_seed
 
@@ -13,9 +13,6 @@ from .seq2seq_trainer import Seq2SeqTrainer
 from .test_seq2seq_examples import MBART_TINY
 
 
-if is_torch_available():
-    import torch
-
 set_seed(42)
 MARIAN_MODEL = "sshleifer/student_marian_en_ro_6_1"
 
@@ -180,7 +177,7 @@ class TestFinetuneTrainer(TestCasePlus):
             --num_train_epochs {str(num_train_epochs)}
             --per_device_train_batch_size 4
             --per_device_eval_batch_size 4
-            --learning_rate 3e-4
+            --learning_rate 3e-3
             --warmup_steps 8
             --evaluate_during_training
             --predict_with_generate
@@ -196,7 +193,7 @@ class TestFinetuneTrainer(TestCasePlus):
         """.split()
         # --eval_beams  2
 
-        n_gpu = torch.cuda.device_count()
+        n_gpu = get_gpu_count()
         if n_gpu > 1:
             distributed_args = f"""
                 -m torch.distributed.launch

examples/seq2seq/test_fsmt_bleu_score.py

@@ -19,7 +19,13 @@ import unittest
 
 from parameterized import parameterized
 from transformers import FSMTForConditionalGeneration, FSMTTokenizer
-from transformers.testing_utils import get_tests_dir, require_torch, slow, torch_device
+from transformers.testing_utils import (
+    get_tests_dir,
+    require_torch,
+    require_torch_non_multi_gpu_but_fix_me,
+    slow,
+    torch_device,
+)
 from utils import calculate_bleu
 
 
@@ -48,6 +54,7 @@ class ModelEvalTester(unittest.TestCase):
         ]
     )
     @slow
+    @require_torch_non_multi_gpu_but_fix_me
     def test_bleu_scores(self, pair, min_bleu_score):
         # note: this test is not testing the best performance since it only evals a small batch
         # but it should be enough to detect a regression in the output quality

examples/seq2seq/test_make_student.py

@@ -4,7 +4,7 @@ import unittest
 from make_student import create_student_by_copying_alternating_layers
 from transformers import AutoConfig
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_torch
+from transformers.testing_utils import require_torch, require_torch_non_multi_gpu_but_fix_me
 
 
 TINY_BART = "sshleifer/bart-tiny-random"
@@ -17,23 +17,28 @@ class MakeStudentTester(unittest.TestCase):
     def teacher_config(self):
         return AutoConfig.from_pretrained(TINY_BART)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_valid_t5(self):
         student, *_ = create_student_by_copying_alternating_layers(TINY_T5, tempfile.mkdtemp(), e=1, d=1)
         self.assertEqual(student.config.num_hidden_layers, 1)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_asymmetric_t5(self):
         student, *_ = create_student_by_copying_alternating_layers(TINY_T5, tempfile.mkdtemp(), e=1, d=None)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_same_decoder_small_encoder(self):
         student, *_ = create_student_by_copying_alternating_layers(TINY_BART, tempfile.mkdtemp(), e=1, d=None)
         self.assertEqual(student.config.encoder_layers, 1)
         self.assertEqual(student.config.decoder_layers, self.teacher_config.encoder_layers)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_small_enc_small_dec(self):
         student, *_ = create_student_by_copying_alternating_layers(TINY_BART, tempfile.mkdtemp(), e=1, d=1)
         self.assertEqual(student.config.encoder_layers, 1)
         self.assertEqual(student.config.decoder_layers, 1)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_raises_assert(self):
         with self.assertRaises(AssertionError):
             create_student_by_copying_alternating_layers(TINY_BART, tempfile.mkdtemp(), e=None, d=None)

examples/seq2seq/test_seq2seq_examples.py

@@ -19,7 +19,14 @@ from run_eval import generate_summaries_or_translations, run_generate
 from run_eval_search import run_search
 from transformers import AutoConfig, AutoModelForSeq2SeqLM
 from transformers.hf_api import HfApi
-from transformers.testing_utils import CaptureStderr, CaptureStdout, TestCasePlus, require_torch_gpu, slow
+from transformers.testing_utils import (
+    CaptureStderr,
+    CaptureStdout,
+    TestCasePlus,
+    require_torch_gpu,
+    require_torch_non_multi_gpu_but_fix_me,
+    slow,
+)
 from utils import ROUGE_KEYS, label_smoothed_nll_loss, lmap, load_json
 
 
@@ -89,6 +96,7 @@ CHEAP_ARGS = {
     "freeze_encoder": False,
     "auto_scale_batch_size": False,
     "overwrite_output_dir": False,
+    "student": None,
 }
 
 
@@ -100,6 +108,7 @@ def _dump_articles(path: Path, articles: list):
 ARTICLES = [" Sam ate lunch today.", "Sams lunch ingredients."]
 SUMMARIES = ["A very interesting story about what I ate for lunch.", "Avocado, celery, turkey, coffee"]
 T5_TINY = "patrickvonplaten/t5-tiny-random"
+T5_TINIER = "sshleifer/t5-tinier-random"
 BART_TINY = "sshleifer/bart-tiny-random"
 MBART_TINY = "sshleifer/tiny-mbart"
 MARIAN_TINY = "sshleifer/tiny-marian-en-de"
@@ -126,6 +135,7 @@ class TestSummarizationDistiller(TestCasePlus):
 
     @slow
     @require_torch_gpu
+    @require_torch_non_multi_gpu_but_fix_me
     def test_hub_configs(self):
         """I put require_torch_gpu cause I only want this to run with self-scheduled."""
 
@@ -143,10 +153,12 @@ class TestSummarizationDistiller(TestCasePlus):
                 failures.append(m)
         assert not failures, f"The following models could not be loaded through AutoConfig: {failures}"
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_distill_no_teacher(self):
         updates = dict(student_encoder_layers=2, student_decoder_layers=1, no_teacher=True)
         self._test_distiller_cli(updates)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_distill_checkpointing_with_teacher(self):
         updates = dict(
             student_encoder_layers=2,
@@ -171,6 +183,7 @@ class TestSummarizationDistiller(TestCasePlus):
         convert_pl_to_hf(ckpts[0], transformer_ckpts[0].parent, out_path_new)
         assert os.path.exists(os.path.join(out_path_new, "pytorch_model.bin"))
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_loss_fn(self):
         model = AutoModelForSeq2SeqLM.from_pretrained(BART_TINY, return_dict=True)
         input_ids, mask = model.dummy_inputs["input_ids"], model.dummy_inputs["attention_mask"]
@@ -191,6 +204,7 @@ class TestSummarizationDistiller(TestCasePlus):
             # TODO: understand why this breaks
             self.assertEqual(nll_loss, model_computed_loss)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_distill_mbart(self):
         updates = dict(
             student_encoder_layers=2,
@@ -215,6 +229,7 @@ class TestSummarizationDistiller(TestCasePlus):
         assert len(all_files) > 2
         self.assertEqual(len(transformer_ckpts), 2)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_distill_t5(self):
         updates = dict(
             student_encoder_layers=1,
@@ -226,6 +241,16 @@ class TestSummarizationDistiller(TestCasePlus):
         )
         self._test_distiller_cli(updates)
 
+    @require_torch_non_multi_gpu_but_fix_me
+    def test_distill_different_base_models(self):
+        updates = dict(
+            teacher=T5_TINY,
+            student=T5_TINIER,
+            model_name_or_path=T5_TINIER,
+            tokenizer_name=T5_TINIER,
+        )
+        self._test_distiller_cli(updates)
+
     def _test_distiller_cli(self, updates, check_contents=True):
         default_updates = dict(
             label_smoothing=0.0,
@@ -296,18 +321,21 @@ class TestTheRest(TestCasePlus):
 
     # test one model to quickly (no-@slow) catch simple problems and do an
     # extensive testing of functionality with multiple models as @slow separately
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_eval(self):
         self.run_eval_tester(T5_TINY)
 
     # any extra models should go into the list here - can be slow
     @parameterized.expand([BART_TINY, MBART_TINY])
     @slow
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_eval_slow(self, model):
         self.run_eval_tester(model)
 
     # testing with 2 models to validate: 1. translation (t5) 2. summarization (mbart)
     @parameterized.expand([T5_TINY, MBART_TINY])
     @slow
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_eval_search(self, model):
         input_file_name = Path(self.get_auto_remove_tmp_dir()) / "utest_input.source"
         output_file_name = input_file_name.parent / "utest_output.txt"
@@ -358,6 +386,7 @@ class TestTheRest(TestCasePlus):
     @parameterized.expand(
         [T5_TINY, BART_TINY, MBART_TINY, MARIAN_TINY, FSMT_TINY],
     )
+    @require_torch_non_multi_gpu_but_fix_me
     def test_finetune(self, model):
         args_d: dict = CHEAP_ARGS.copy()
         task = "translation" if model in [MBART_TINY, MARIAN_TINY, FSMT_TINY] else "summarization"
@@ -409,6 +438,7 @@ class TestTheRest(TestCasePlus):
         assert isinstance(example_batch, dict)
         assert len(example_batch) >= 4
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_finetune_extra_model_args(self):
         args_d: dict = CHEAP_ARGS.copy()
 
@@ -459,6 +489,7 @@ class TestTheRest(TestCasePlus):
             model = main(args)
         assert str(excinfo.value) == f"model config doesn't have a `{unsupported_param}` attribute"
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_finetune_lr_schedulers(self):
         args_d: dict = CHEAP_ARGS.copy()
 

examples/seq2seq/test_seq2seq_examples_multi_gpu.py

@@ -3,7 +3,14 @@
 import os
 import sys
 
-from transformers.testing_utils import TestCasePlus, execute_subprocess_async, require_torch_multigpu
+from transformers.testing_utils import (
+    TestCasePlus,
+    execute_subprocess_async,
+    get_gpu_count,
+    require_torch_gpu,
+    require_torch_multi_gpu,
+    slow,
+)
 
 from .test_seq2seq_examples import CHEAP_ARGS, make_test_data_dir
 from .utils import load_json
@@ -14,8 +21,8 @@ class TestSummarizationDistillerMultiGPU(TestCasePlus):
     def setUpClass(cls):
         return cls
 
-    @require_torch_multigpu
+    @require_torch_multi_gpu
-    def test_multigpu(self):
+    def test_multi_gpu(self):
 
         updates = dict(
             no_teacher=True,
@@ -80,3 +87,30 @@ class TestSummarizationDistillerMultiGPU(TestCasePlus):
         self.assertEqual(len(metrics["test"]), 1)
         desired_n_evals = int(args_d["max_epochs"] * (1 / args_d["val_check_interval"]) / 2 + 1)
         self.assertEqual(len(metrics["val"]), desired_n_evals)
+
+    @slow
+    @require_torch_gpu
+    def test_distributed_eval(self):
+        output_dir = self.get_auto_remove_tmp_dir()
+        args = f"""
+            --model_name Helsinki-NLP/opus-mt-en-ro
+            --save_dir {output_dir}
+            --data_dir {self.test_file_dir_str}/test_data/wmt_en_ro
+            --num_beams 2
+            --task translation
+        """.split()
+
+        # we want this test to run even if there is only one GPU, but if there are more we use them all
+        n_gpu = get_gpu_count()
+        distributed_args = f"""
+            -m torch.distributed.launch
+            --nproc_per_node={n_gpu}
+            {self.test_file_dir}/run_distributed_eval.py
+        """.split()
+        cmd = [sys.executable] + distributed_args + args
+        execute_subprocess_async(cmd, env=self.get_env())
+
+        metrics_save_path = os.path.join(output_dir, "test_bleu.json")
+        metrics = load_json(metrics_save_path)
+        # print(metrics)
+        self.assertGreaterEqual(metrics["bleu"], 25)

examples/seq2seq/test_tatoeba_conversion.py

@@ -4,7 +4,7 @@ import unittest
 
 from transformers.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
 from transformers.file_utils import cached_property
-from transformers.testing_utils import slow
+from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow
 
 
 @unittest.skipUnless(os.path.exists(DEFAULT_REPO), "Tatoeba directory does not exist.")
@@ -15,10 +15,12 @@ class TatoebaConversionTester(unittest.TestCase):
         return TatoebaConverter(save_dir=tmp_dir)
 
     @slow
+    @require_torch_non_multi_gpu_but_fix_me
     def test_resolver(self):
         self.resolver.convert_models(["heb-eng"])
 
     @slow
+    @require_torch_non_multi_gpu_but_fix_me
     def test_model_card(self):
         content, mmeta = self.resolver.write_model_card("opus-mt-he-en", dry_run=True)
         assert mmeta["long_pair"] == "heb-eng"

examples/seq2seq/train_distilbart_xsum.sh

@@ -2,6 +2,7 @@
 export PYTHONPATH="../":"${PYTHONPATH}"
 python distillation.py \
   --teacher facebook/bart-large-xsum --data_dir xsum \
+  --tokenizer_name facebook/bart-large-xsum \
   --student_decoder_layers 6 --student_encoder_layers 12 \
   --freeze_encoder --freeze_embeds \
   --learning_rate=3e-4 \

examples/seq2seq/utils.py

@@ -269,7 +269,7 @@ class Seq2SeqDataCollator:
         ), f"pad_token_id is not defined for ({self.tokenizer.__class__.__name__}), it must be defined."
         self.data_args = data_args
         self.tpu_num_cores = tpu_num_cores
-        self.dataset_kwargs = {"add_prefix_space": isinstance(tokenizer, BartTokenizer)}
+        self.dataset_kwargs = {"add_prefix_space": True} if isinstance(tokenizer, BartTokenizer) else {}
         if data_args.src_lang is not None:
             self.dataset_kwargs["src_lang"] = data_args.src_lang
         if data_args.tgt_lang is not None:

examples/test_examples.py

@@ -23,12 +23,18 @@ from unittest.mock import patch
 import torch
 
 from transformers.file_utils import is_apex_available
-from transformers.testing_utils import TestCasePlus, torch_device
+from transformers.testing_utils import TestCasePlus, require_torch_non_multi_gpu_but_fix_me, torch_device
 
 
 SRC_DIRS = [
     os.path.join(os.path.dirname(__file__), dirname)
-    for dirname in ["text-generation", "text-classification", "language-modeling", "question-answering"]
+    for dirname in [
+        "text-generation",
+        "text-classification",
+        "token-classification",
+        "language-modeling",
+        "question-answering",
+    ]
 ]
 sys.path.extend(SRC_DIRS)
 
@@ -38,6 +44,7 @@ if SRC_DIRS is not None:
     import run_generation
     import run_glue
     import run_mlm
+    import run_ner
     import run_pl_glue
     import run_squad
 
@@ -60,6 +67,7 @@ def is_cuda_and_apex_available():
 
 
 class ExamplesTests(TestCasePlus):
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -92,6 +100,7 @@ class ExamplesTests(TestCasePlus):
             for value in result.values():
                 self.assertGreaterEqual(value, 0.75)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_pl_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -129,6 +138,7 @@ class ExamplesTests(TestCasePlus):
             #         self.assertGreaterEqual(v, 0.75, f"({k})")
             #
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_clm(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -160,6 +170,7 @@ class ExamplesTests(TestCasePlus):
             result = run_clm.main()
             self.assertLess(result["perplexity"], 100)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_mlm(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -185,6 +196,38 @@ class ExamplesTests(TestCasePlus):
             result = run_mlm.main()
             self.assertLess(result["perplexity"], 42)
 
+    @require_torch_non_multi_gpu_but_fix_me
+    def test_run_ner(self):
+        stream_handler = logging.StreamHandler(sys.stdout)
+        logger.addHandler(stream_handler)
+
+        tmp_dir = self.get_auto_remove_tmp_dir()
+        testargs = f"""
+            run_ner.py
+            --model_name_or_path bert-base-uncased
+            --train_file tests/fixtures/tests_samples/conll/sample.json
+            --validation_file tests/fixtures/tests_samples/conll/sample.json
+            --output_dir {tmp_dir}
+            --overwrite_output_dir
+            --do_train
+            --do_eval
+            --warmup_steps=2
+            --learning_rate=2e-4
+            --per_gpu_train_batch_size=2
+            --per_gpu_eval_batch_size=2
+            --num_train_epochs=2
+        """.split()
+
+        if torch_device != "cuda":
+            testargs.append("--no_cuda")
+
+        with patch.object(sys, "argv", testargs):
+            result = run_ner.main()
+            self.assertGreaterEqual(result["eval_accuracy_score"], 0.75)
+            self.assertGreaterEqual(result["eval_precision"], 0.75)
+            self.assertLess(result["eval_loss"], 0.5)
+
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_squad(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -213,6 +256,7 @@ class ExamplesTests(TestCasePlus):
             self.assertGreaterEqual(result["f1"], 25)
             self.assertGreaterEqual(result["exact"], 21)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_generation(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)

examples/test_xla_examples.py

@@ -20,7 +20,7 @@ import unittest
 from time import time
 from unittest.mock import patch
 
-from transformers.testing_utils import require_torch_tpu
+from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, require_torch_tpu
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -30,6 +30,7 @@ logger = logging.getLogger()
 
 @require_torch_tpu
 class TorchXLAExamplesTests(unittest.TestCase):
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_glue(self):
         import xla_spawn
 
@@ -81,6 +82,7 @@ class TorchXLAExamplesTests(unittest.TestCase):
             # Assert that the script takes less than 300 seconds to make sure it doesn't hang.
             self.assertLess(end - start, 500)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_trainer_tpu(self):
         import xla_spawn
 

examples/text-classification/run_glue.py

@@ -171,6 +171,8 @@ def main():
     # Set the verbosity to info of the Transformers logger (on main process only):
     if is_main_process(training_args.local_rank):
         transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info(f"Training/evaluation parameters {training_args}")
 
     # Set seed before initializing model.

examples/text-classification/run_tf_glue.py

@@ -27,6 +27,12 @@ from transformers import (
     glue_processors,
     glue_tasks_num_labels,
 )
+from transformers.utils import logging as hf_logging
+
+
+hf_logging.set_verbosity_info()
+hf_logging.enable_default_handler()
+hf_logging.enable_explicit_format()
 
 
 class Split(Enum):

examples/text-classification/run_tf_text_classification.py

@@ -21,6 +21,12 @@ from transformers import (
     TFTrainer,
     TFTrainingArguments,
 )
+from transformers.utils import logging as hf_logging
+
+
+hf_logging.set_verbosity_info()
+hf_logging.enable_default_handler()
+hf_logging.enable_explicit_format()
 
 
 def get_tfds(

examples/text-classification/run_xnli.py

@@ -29,6 +29,7 @@ from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Tenso
 from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
+import transformers
 from transformers import (
     WEIGHTS_NAME,
     AdamW,
@@ -41,6 +42,7 @@ from transformers import glue_convert_examples_to_features as convert_examples_t
 from transformers import xnli_compute_metrics as compute_metrics
 from transformers import xnli_output_modes as output_modes
 from transformers import xnli_processors as processors
+from transformers.trainer_utils import is_main_process
 
 
 try:
@@ -526,7 +528,11 @@ def main():
         bool(args.local_rank != -1),
         args.fp16,
     )
-
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     # Set seed
     set_seed(args)
 

examples/token-classification/README.md

@@ -1,6 +1,40 @@
-## Named Entity Recognition
+## Token classification
 
-Based on the scripts [`run_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/token-classification/run_ner.py) for Pytorch and
+Fine-tuning the library models for token classification task such as Named Entity Recognition (NER) or Parts-of-speech
+tagging (POS). The main scrip `run_ner.py` leverages the 🤗 Datasets library and the Trainer API. You can easily
+customize it to your needs if you need extra processing on your datasets.
+
+It will either run on a datasets hosted on our [hub](https://huggingface.co/datasets) or with your own text files for
+training and validation.
+
+The following example fine-tunes BERT on CoNLL-2003:
+
+```bash
+python run_ner.py \
+  --model_name_or_path bert-base-uncased \
+  --dataset_name conll2003 \
+  --output_dir /tmp/test-ner \
+  --do_train \
+  --do_eval
+```
+
+or just can just run the bash script `run.sh`.
+
+To run on your own training and validation files, use the following command:
+
+```bash
+python run_ner.py \
+  --model_name_or_path bert-base-uncased \
+  --train_file path_to_train_file \
+  --validation_file path_to_validation_file \
+  --output_dir /tmp/test-ner \
+  --do_train \
+  --do_eval
+```
+
+## Old version of the script
+
+Based on the scripts [`run_ner_old.py`](https://github.com/huggingface/transformers/blob/master/examples/token-classification/run_ner_old.py) for Pytorch and
 [`run_tf_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/token-classification/run_tf_ner.py) for Tensorflow 2.
 
 The following examples are covered in this section:
@@ -69,7 +103,7 @@ export SEED=1
 To start training, just run:
 
 ```bash
-python3 run_ner.py --data_dir ./ \
+python3 run_ner_old.py --data_dir ./ \
 --labels ./labels.txt \
 --model_name_or_path $BERT_MODEL \
 --output_dir $OUTPUT_DIR \
@@ -87,7 +121,7 @@ If your GPU supports half-precision training, just add the `--fp16` flag. After
 
 #### JSON-based configuration file
 
-Instead of passing all parameters via commandline arguments, the `run_ner.py` script also supports reading parameters from a json-based configuration file:
+Instead of passing all parameters via commandline arguments, the `run_ner_old.py` script also supports reading parameters from a json-based configuration file:
 
 ```json
 {
@@ -106,7 +140,7 @@ Instead of passing all parameters via commandline arguments, the `run_ner.py` sc
 }
 ```
 
-It must be saved with a `.json` extension and can be used by running `python3 run_ner.py config.json`.
+It must be saved with a `.json` extension and can be used by running `python3 run_ner_old.py config.json`.
 
 #### Evaluation
 
@@ -250,7 +284,7 @@ cat data_wnut_17/train.txt data_wnut_17/dev.txt data_wnut_17/test.txt | cut -d "
 
 #### Run the Pytorch version
 
-Fine-tuning with the PyTorch version can be started using the `run_ner.py` script. In this example we use a JSON-based configuration file.
+Fine-tuning with the PyTorch version can be started using the `run_ner_old.py` script. In this example we use a JSON-based configuration file.
 
 This configuration file looks like:
 
@@ -274,7 +308,7 @@ This configuration file looks like:
 
 If your GPU supports half-precision training, please set `fp16` to `true`.
 
-Save this JSON-based configuration under `wnut_17.json`. The fine-tuning can be started with `python3 run_ner.py wnut_17.json`.
+Save this JSON-based configuration under `wnut_17.json`. The fine-tuning can be started with `python3 run_ner_old.py wnut_17.json`.
 
 #### Evaluation
 

examples/token-classification/run.sh

@@ -1,36 +1,6 @@
-## The relevant files are currently on a shared Google
-## drive at https://drive.google.com/drive/folders/1kC0I2UGl2ltrluI9NqDjaQJGw5iliw_J
-## Monitor for changes and eventually migrate to nlp dataset
-curl -L 'https://drive.google.com/uc?export=download&id=1Jjhbal535VVz2ap4v4r_rN1UEHTdLK5P' \
-| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > train.txt.tmp
-curl -L 'https://drive.google.com/uc?export=download&id=1ZfRcQThdtAR5PPRjIDtrVP7BtXSCUBbm' \
-| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > dev.txt.tmp
-curl -L 'https://drive.google.com/uc?export=download&id=1u9mb7kNJHWQCWyweMDRMuTFoOHOfeBTH' \
-| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > test.txt.tmp
-
-export MAX_LENGTH=128
-export BERT_MODEL=bert-base-multilingual-cased
-python3 scripts/preprocess.py train.txt.tmp $BERT_MODEL $MAX_LENGTH > train.txt
-python3 scripts/preprocess.py dev.txt.tmp $BERT_MODEL $MAX_LENGTH > dev.txt
-python3 scripts/preprocess.py test.txt.tmp $BERT_MODEL $MAX_LENGTH > test.txt
-cat train.txt dev.txt test.txt | cut -d " " -f 2 | grep -v "^$"| sort | uniq > labels.txt
-export OUTPUT_DIR=germeval-model
-export BATCH_SIZE=32
-export NUM_EPOCHS=3
-export SAVE_STEPS=750
-export SEED=1
-
 python3 run_ner.py \
---task_type NER \
+  --model_name_or_path bert-base-uncased \
---data_dir . \
+  --dataset_name conll2003 \
---labels ./labels.txt \
+  --output_dir /tmp/test-ner \
---model_name_or_path $BERT_MODEL \
+  --do_train \
---output_dir $OUTPUT_DIR \
+  --do_eval
---max_seq_length  $MAX_LENGTH \
---num_train_epochs $NUM_EPOCHS \
---per_gpu_train_batch_size $BATCH_SIZE \
---save_steps $SAVE_STEPS \
---seed $SEED \
---do_train \
---do_eval \
---do_predict

examples/token-classification/run_chunk.sh

@@ -21,7 +21,7 @@ export NUM_EPOCHS=3
 export SAVE_STEPS=750
 export SEED=1
 
-python3 run_ner.py \
+python3 run_ner_old.py \
 --task_type Chunk \
 --data_dir . \
 --model_name_or_path $BERT_MODEL \

examples/token-classification/run_ner.py

@@ -1,6 +1,5 @@
 # coding=utf-8
-# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright 2020 The HuggingFace Team All rights reserved.
-# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,29 +12,33 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" Fine-tuning the library models for named entity recognition on CoNLL-2003. """
+"""
+Fine-tuning the library models for token classification.
+"""
+# You can also adapt this script on your own token classification task and datasets. Pointers for this are left as comments.
+
 import logging
 import os
 import sys
 from dataclasses import dataclass, field
-from importlib import import_module
+from typing import Optional
-from typing import Dict, List, Optional, Tuple
 
 import numpy as np
+from datasets import load_dataset
 from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
-from torch import nn
 
+import transformers
 from transformers import (
     AutoConfig,
     AutoModelForTokenClassification,
     AutoTokenizer,
-    EvalPrediction,
+    DataCollatorForTokenClassification,
     HfArgumentParser,
     Trainer,
     TrainingArguments,
     set_seed,
 )
-from utils_ner import Split, TokenClassificationDataset, TokenClassificationTask
+from transformers.trainer_utils import is_main_process
 
 
 logger = logging.getLogger(__name__)
@@ -53,15 +56,9 @@ class ModelArguments:
     config_name: Optional[str] = field(
         default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
     )
-    task_type: Optional[str] = field(
-        default="NER", metadata={"help": "Task type to fine tune in training (e.g. NER, POS, etc)"}
-    )
     tokenizer_name: Optional[str] = field(
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
-    use_fast: bool = field(default=False, metadata={"help": "Set this flag to use fast tokenization."})
-    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
-    # or just modify its tokenizer_config.json.
     cache_dir: Optional[str] = field(
         default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
     )
@@ -73,23 +70,58 @@ class DataTrainingArguments:
     Arguments pertaining to what data we are going to input our model for training and eval.
     """
 
-    data_dir: str = field(
+    task_name: Optional[str] = field(default="ner", metadata={"help": "The name of the task (ner, pos...)."})
-        metadata={"help": "The input data dir. Should contain the .txt files for a CoNLL-2003-formatted task."}
+    dataset_name: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
     )
-    labels: Optional[str] = field(
+    dataset_config_name: Optional[str] = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    train_file: Optional[str] = field(
+        default=None, metadata={"help": "The input training data file (a csv or JSON file)."}
+    )
+    validation_file: Optional[str] = field(
         default=None,
-        metadata={"help": "Path to a file containing all labels. If not specified, CoNLL-2003 labels are used."},
+        metadata={"help": "An optional input evaluation data file to evaluate on (a csv or JSON file)."},
     )
-    max_seq_length: int = field(
+    test_file: Optional[str] = field(
-        default=128,
+        default=None,
-        metadata={
+        metadata={"help": "An optional input test data file to predict on (a csv or JSON file)."},
-            "help": "The maximum total input sequence length after tokenization. Sequences longer "
-            "than this will be truncated, sequences shorter will be padded."
-        },
     )
     overwrite_cache: bool = field(
         default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
     )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    pad_to_max_length: bool = field(
+        default=False,
+        metadata={
+            "help": "Whether to pad all samples to model maximum sentence length. "
+            "If False, will pad the samples dynamically when batching to the maximum length in the batch. More "
+            "efficient on GPU but very bad for TPU."
+        },
+    )
+    label_all_tokens: bool = field(
+        default=False,
+        metadata={
+            "help": "Whether to put the label for one word on all tokens of generated by that word or just on the "
+            "one (in which case the other tokens will have a padding index)."
+        },
+    )
+
+    def __post_init__(self):
+        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
+            raise ValueError("Need either a dataset name or a training/validation file.")
+        else:
+            if self.train_file is not None:
+                extension = self.train_file.split(".")[-1]
+                assert extension in ["csv", "json"], "`train_file` should be a csv or a json file."
+            if self.validation_file is not None:
+                extension = self.validation_file.split(".")[-1]
+                assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file."
+        self.task_name = self.task_name.lower()
 
 
 def main():
@@ -112,60 +144,92 @@ def main():
         and not training_args.overwrite_output_dir
     ):
         raise ValueError(
-            f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
+            f"Output directory ({training_args.output_dir}) already exists and is not empty."
-        )
+            "Use --overwrite_output_dir to overcome."
-
-    module = import_module("tasks")
-    try:
-        token_classification_task_clazz = getattr(module, model_args.task_type)
-        token_classification_task: TokenClassificationTask = token_classification_task_clazz()
-    except AttributeError:
-        raise ValueError(
-            f"Task {model_args.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. "
-            f"Available tasks classes are: {TokenClassificationTask.__subclasses__()}"
         )
 
     # Setup logging
     logging.basicConfig(
         format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
         datefmt="%m/%d/%Y %H:%M:%S",
-        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,
+        level=logging.INFO if is_main_process(training_args.local_rank) else logging.WARN,
     )
+
+    # Log on each process the small summary:
     logger.warning(
-        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
-        training_args.local_rank,
+        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
-        training_args.device,
-        training_args.n_gpu,
-        bool(training_args.local_rank != -1),
-        training_args.fp16,
     )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(training_args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
     logger.info("Training/evaluation parameters %s", training_args)
 
-    # Set seed
+    # Set seed before initializing model.
     set_seed(training_args.seed)
 
-    # Prepare CONLL-2003 task
+    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
-    labels = token_classification_task.get_labels(data_args.labels)
+    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
-    label_map: Dict[int, str] = {i: label for i, label in enumerate(labels)}
+    # (the dataset will be downloaded automatically from the datasets Hub).
-    num_labels = len(labels)
+    #
+    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
+    # 'text' is found. You can easily tweak this behavior (see below).
+    #
+    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
+    # download the dataset.
+    if data_args.dataset_name is not None:
+        # Downloading and loading a dataset from the hub.
+        datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
+    else:
+        data_files = {}
+        if data_args.train_file is not None:
+            data_files["train"] = data_args.train_file
+        if data_args.validation_file is not None:
+            data_files["validation"] = data_args.validation_file
+        if data_args.test_file is not None:
+            data_files["test"] = data_args.test_file
+        extension = data_args.train_file.split(".")[-1]
+        datasets = load_dataset(extension, data_files=data_files)
+    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
+    # https://huggingface.co/docs/datasets/loading_datasets.html.
+
+    if training_args.do_train:
+        column_names = datasets["train"].column_names
+    else:
+        column_names = datasets["validation"].column_names
+    text_column_name = "words" if "words" in column_names else column_names[0]
+    label_column_name = data_args.task_name if data_args.task_name in column_names else column_names[1]
+
+    # Labeling (this part will be easier when https://github.com/huggingface/datasets/issues/797 is solved)
+    def get_label_list(labels):
+        unique_labels = set()
+        for label in labels:
+            unique_labels = unique_labels | set(label)
+        label_list = list(unique_labels)
+        label_list.sort()
+        return label_list
+
+    label_list = get_label_list(datasets["train"][label_column_name])
+    label_to_id = {l: i for i, l in enumerate(label_list)}
+    num_labels = len(label_list)
 
     # Load pretrained model and tokenizer
     #
     # Distributed training:
     # The .from_pretrained methods guarantee that only one local process can concurrently
     # download model & vocab.
-
     config = AutoConfig.from_pretrained(
         model_args.config_name if model_args.config_name else model_args.model_name_or_path,
         num_labels=num_labels,
-        id2label=label_map,
+        finetuning_task=data_args.task_name,
-        label2id={label: i for i, label in enumerate(labels)},
         cache_dir=model_args.cache_dir,
     )
     tokenizer = AutoTokenizer.from_pretrained(
         model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
         cache_dir=model_args.cache_dir,
-        use_fast=model_args.use_fast,
+        use_fast=True,
     )
     model = AutoModelForTokenClassification.from_pretrained(
         model_args.model_name_or_path,
@@ -174,67 +238,85 @@ def main():
         cache_dir=model_args.cache_dir,
     )
 
-    # Get datasets
+    # Preprocessing the dataset
-    train_dataset = (
+    # Padding strategy
-        TokenClassificationDataset(
+    padding = "max_length" if data_args.pad_to_max_length else False
-            token_classification_task=token_classification_task,
+
-            data_dir=data_args.data_dir,
+    # Tokenize all texts and align the labels with them.
-            tokenizer=tokenizer,
+    def tokenize_and_align_labels(examples):
-            labels=labels,
+        tokenized_inputs = tokenizer(
-            model_type=config.model_type,
+            examples[text_column_name],
-            max_seq_length=data_args.max_seq_length,
+            padding=padding,
-            overwrite_cache=data_args.overwrite_cache,
+            truncation=True,
-            mode=Split.train,
+            # We use this argument because the texts in our dataset are lists of words (with a label for each word).
+            is_split_into_words=True,
+            return_offsets_mapping=True,
         )
-        if training_args.do_train
+        offset_mappings = tokenized_inputs.pop("offset_mapping")
-        else None
+        labels = []
-    )
+        for label, offset_mapping in zip(examples[label_column_name], offset_mappings):
-    eval_dataset = (
+            label_index = 0
-        TokenClassificationDataset(
+            current_label = -100
-            token_classification_task=token_classification_task,
+            label_ids = []
-            data_dir=data_args.data_dir,
+            for offset in offset_mapping:
-            tokenizer=tokenizer,
+                # We set the label for the first token of each word. Special characters will have an offset of (0, 0)
-            labels=labels,
+                # so the test ignores them.
-            model_type=config.model_type,
+                if offset[0] == 0 and offset[1] != 0:
-            max_seq_length=data_args.max_seq_length,
+                    current_label = label_to_id[label[label_index]]
-            overwrite_cache=data_args.overwrite_cache,
+                    label_index += 1
-            mode=Split.dev,
+                    label_ids.append(current_label)
-        )
+                # For special tokens, we set the label to -100 so it's automatically ignored in the loss function.
-        if training_args.do_eval
+                elif offset[0] == 0 and offset[1] == 0:
-        else None
+                    label_ids.append(-100)
+                # For the other tokens in a word, we set the label to either the current label or -100, depending on
+                # the label_all_tokens flag.
+                else:
+                    label_ids.append(current_label if data_args.label_all_tokens else -100)
+
+            labels.append(label_ids)
+        tokenized_inputs["labels"] = labels
+        return tokenized_inputs
+
+    tokenized_datasets = datasets.map(
+        tokenize_and_align_labels,
+        batched=True,
+        num_proc=data_args.preprocessing_num_workers,
+        load_from_cache_file=not data_args.overwrite_cache,
     )
 
-    def align_predictions(predictions: np.ndarray, label_ids: np.ndarray) -> Tuple[List[int], List[int]]:
+    # Data collator
-        preds = np.argmax(predictions, axis=2)
+    data_collator = DataCollatorForTokenClassification(tokenizer)
 
-        batch_size, seq_len = preds.shape
+    # Metrics
+    def compute_metrics(p):
+        predictions, labels = p
+        predictions = np.argmax(predictions, axis=2)
 
-        out_label_list = [[] for _ in range(batch_size)]
+        # Remove ignored index (special tokens)
-        preds_list = [[] for _ in range(batch_size)]
+        true_predictions = [
+            [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
+            for prediction, label in zip(predictions, labels)
+        ]
+        true_labels = [
+            [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
+            for prediction, label in zip(predictions, labels)
+        ]
 
-        for i in range(batch_size):
-            for j in range(seq_len):
-                if label_ids[i, j] != nn.CrossEntropyLoss().ignore_index:
-                    out_label_list[i].append(label_map[label_ids[i][j]])
-                    preds_list[i].append(label_map[preds[i][j]])
-
-        return preds_list, out_label_list
-
-    def compute_metrics(p: EvalPrediction) -> Dict:
-        preds_list, out_label_list = align_predictions(p.predictions, p.label_ids)
         return {
-            "accuracy_score": accuracy_score(out_label_list, preds_list),
+            "accuracy_score": accuracy_score(true_labels, true_predictions),
-            "precision": precision_score(out_label_list, preds_list),
+            "precision": precision_score(true_labels, true_predictions),
-            "recall": recall_score(out_label_list, preds_list),
+            "recall": recall_score(true_labels, true_predictions),
-            "f1": f1_score(out_label_list, preds_list),
+            "f1": f1_score(true_labels, true_predictions),
         }
 
     # Initialize our Trainer
     trainer = Trainer(
         model=model,
         args=training_args,
-        train_dataset=train_dataset,
+        train_dataset=tokenized_datasets["train"] if training_args.do_train else None,
-        eval_dataset=eval_dataset,
+        eval_dataset=tokenized_datasets["validation"] if training_args.do_eval else None,
+        tokenizer=tokenizer,
+        data_collator=data_collator,
         compute_metrics=compute_metrics,
     )
 
@@ -243,58 +325,50 @@ def main():
         trainer.train(
             model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
         )
-        trainer.save_model()
+        trainer.save_model()  # Saves the tokenizer too for easy upload
-        # For convenience, we also re-save the tokenizer to the same directory,
-        # so that you can share your model easily on huggingface.co/models =)
-        if trainer.is_world_master():
-            tokenizer.save_pretrained(training_args.output_dir)
 
     # Evaluation
     results = {}
     if training_args.do_eval:
         logger.info("*** Evaluate ***")
 
-        result = trainer.evaluate()
+        results = trainer.evaluate()
 
-        output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")
+        output_eval_file = os.path.join(training_args.output_dir, "eval_results_ner.txt")
-        if trainer.is_world_master():
+        if trainer.is_world_process_zero():
             with open(output_eval_file, "w") as writer:
                 logger.info("***** Eval results *****")
-                for key, value in result.items():
+                for key, value in results.items():
-                    logger.info("  %s = %s", key, value)
+                    logger.info(f"  {key} = {value}")
-                    writer.write("%s = %s\n" % (key, value))
+                    writer.write(f"{key} = {value}\n")
-
-            results.update(result)
 
     # Predict
     if training_args.do_predict:
-        test_dataset = TokenClassificationDataset(
+        logger.info("*** Predict ***")
-            token_classification_task=token_classification_task,
-            data_dir=data_args.data_dir,
-            tokenizer=tokenizer,
-            labels=labels,
-            model_type=config.model_type,
-            max_seq_length=data_args.max_seq_length,
-            overwrite_cache=data_args.overwrite_cache,
-            mode=Split.test,
-        )
 
-        predictions, label_ids, metrics = trainer.predict(test_dataset)
+        test_dataset = tokenized_datasets["test"]
-        preds_list, _ = align_predictions(predictions, label_ids)
+        predictions, labels, metrics = trainer.predict(test_dataset)
+        predictions = np.argmax(predictions, axis=2)
+
+        # Remove ignored index (special tokens)
+        true_predictions = [
+            [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
+            for prediction, label in zip(predictions, labels)
+        ]
 
         output_test_results_file = os.path.join(training_args.output_dir, "test_results.txt")
         if trainer.is_world_master():
             with open(output_test_results_file, "w") as writer:
                 for key, value in metrics.items():
-                    logger.info("  %s = %s", key, value)
+                    logger.info(f"  {key} = {value}")
-                    writer.write("%s = %s\n" % (key, value))
+                    writer.write(f"{key} = {value}\n")
 
         # Save predictions
         output_test_predictions_file = os.path.join(training_args.output_dir, "test_predictions.txt")
         if trainer.is_world_master():
             with open(output_test_predictions_file, "w") as writer:
-                with open(os.path.join(data_args.data_dir, "test.txt"), "r") as f:
+                for prediction in true_predictions:
-                    token_classification_task.write_predictions_to_file(writer, f, preds_list)
+                    writer.write(" ".join(prediction) + "\n")
 
     return results
 

examples/token-classification/run_ner_old.py

@@ -0,0 +1,315 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Fine-tuning the library models for named entity recognition on CoNLL-2003. """
+import logging
+import os
+import sys
+from dataclasses import dataclass, field
+from importlib import import_module
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
+from torch import nn
+
+import transformers
+from transformers import (
+    AutoConfig,
+    AutoModelForTokenClassification,
+    AutoTokenizer,
+    EvalPrediction,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    set_seed,
+)
+from transformers.trainer_utils import is_main_process
+from utils_ner import Split, TokenClassificationDataset, TokenClassificationTask
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
+    """
+
+    model_name_or_path: str = field(
+        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    task_type: Optional[str] = field(
+        default="NER", metadata={"help": "Task type to fine tune in training (e.g. NER, POS, etc)"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    use_fast: bool = field(default=False, metadata={"help": "Set this flag to use fast tokenization."})
+    # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
+    # or just modify its tokenizer_config.json.
+    cache_dir: Optional[str] = field(
+        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
+    )
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+
+    data_dir: str = field(
+        metadata={"help": "The input data dir. Should contain the .txt files for a CoNLL-2003-formatted task."}
+    )
+    labels: Optional[str] = field(
+        default=None,
+        metadata={"help": "Path to a file containing all labels. If not specified, CoNLL-2003 labels are used."},
+    )
+    max_seq_length: int = field(
+        default=128,
+        metadata={
+            "help": "The maximum total input sequence length after tokenization. Sequences longer "
+            "than this will be truncated, sequences shorter will be padded."
+        },
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
+    )
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    if (
+        os.path.exists(training_args.output_dir)
+        and os.listdir(training_args.output_dir)
+        and training_args.do_train
+        and not training_args.overwrite_output_dir
+    ):
+        raise ValueError(
+            f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
+        )
+
+    module = import_module("tasks")
+    try:
+        token_classification_task_clazz = getattr(module, model_args.task_type)
+        token_classification_task: TokenClassificationTask = token_classification_task_clazz()
+    except AttributeError:
+        raise ValueError(
+            f"Task {model_args.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. "
+            f"Available tasks classes are: {TokenClassificationTask.__subclasses__()}"
+        )
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,
+    )
+    logger.warning(
+        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+        training_args.local_rank,
+        training_args.device,
+        training_args.n_gpu,
+        bool(training_args.local_rank != -1),
+        training_args.fp16,
+    )
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    if is_main_process(training_args.local_rank):
+        transformers.utils.logging.set_verbosity_info()
+        transformers.utils.logging.enable_default_handler()
+        transformers.utils.logging.enable_explicit_format()
+    logger.info("Training/evaluation parameters %s", training_args)
+
+    # Set seed
+    set_seed(training_args.seed)
+
+    # Prepare CONLL-2003 task
+    labels = token_classification_task.get_labels(data_args.labels)
+    label_map: Dict[int, str] = {i: label for i, label in enumerate(labels)}
+    num_labels = len(labels)
+
+    # Load pretrained model and tokenizer
+    #
+    # Distributed training:
+    # The .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+
+    config = AutoConfig.from_pretrained(
+        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
+        num_labels=num_labels,
+        id2label=label_map,
+        label2id={label: i for i, label in enumerate(labels)},
+        cache_dir=model_args.cache_dir,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
+        cache_dir=model_args.cache_dir,
+        use_fast=model_args.use_fast,
+    )
+    model = AutoModelForTokenClassification.from_pretrained(
+        model_args.model_name_or_path,
+        from_tf=bool(".ckpt" in model_args.model_name_or_path),
+        config=config,
+        cache_dir=model_args.cache_dir,
+    )
+
+    # Get datasets
+    train_dataset = (
+        TokenClassificationDataset(
+            token_classification_task=token_classification_task,
+            data_dir=data_args.data_dir,
+            tokenizer=tokenizer,
+            labels=labels,
+            model_type=config.model_type,
+            max_seq_length=data_args.max_seq_length,
+            overwrite_cache=data_args.overwrite_cache,
+            mode=Split.train,
+        )
+        if training_args.do_train
+        else None
+    )
+    eval_dataset = (
+        TokenClassificationDataset(
+            token_classification_task=token_classification_task,
+            data_dir=data_args.data_dir,
+            tokenizer=tokenizer,
+            labels=labels,
+            model_type=config.model_type,
+            max_seq_length=data_args.max_seq_length,
+            overwrite_cache=data_args.overwrite_cache,
+            mode=Split.dev,
+        )
+        if training_args.do_eval
+        else None
+    )
+
+    def align_predictions(predictions: np.ndarray, label_ids: np.ndarray) -> Tuple[List[int], List[int]]:
+        preds = np.argmax(predictions, axis=2)
+
+        batch_size, seq_len = preds.shape
+
+        out_label_list = [[] for _ in range(batch_size)]
+        preds_list = [[] for _ in range(batch_size)]
+
+        for i in range(batch_size):
+            for j in range(seq_len):
+                if label_ids[i, j] != nn.CrossEntropyLoss().ignore_index:
+                    out_label_list[i].append(label_map[label_ids[i][j]])
+                    preds_list[i].append(label_map[preds[i][j]])
+
+        return preds_list, out_label_list
+
+    def compute_metrics(p: EvalPrediction) -> Dict:
+        preds_list, out_label_list = align_predictions(p.predictions, p.label_ids)
+        return {
+            "accuracy_score": accuracy_score(out_label_list, preds_list),
+            "precision": precision_score(out_label_list, preds_list),
+            "recall": recall_score(out_label_list, preds_list),
+            "f1": f1_score(out_label_list, preds_list),
+        }
+
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+        compute_metrics=compute_metrics,
+    )
+
+    # Training
+    if training_args.do_train:
+        trainer.train(
+            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
+        )
+        trainer.save_model()
+        # For convenience, we also re-save the tokenizer to the same directory,
+        # so that you can share your model easily on huggingface.co/models =)
+        if trainer.is_world_master():
+            tokenizer.save_pretrained(training_args.output_dir)
+
+    # Evaluation
+    results = {}
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        result = trainer.evaluate()
+
+        output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")
+        if trainer.is_world_master():
+            with open(output_eval_file, "w") as writer:
+                logger.info("***** Eval results *****")
+                for key, value in result.items():
+                    logger.info("  %s = %s", key, value)
+                    writer.write("%s = %s\n" % (key, value))
+
+            results.update(result)
+
+    # Predict
+    if training_args.do_predict:
+        test_dataset = TokenClassificationDataset(
+            token_classification_task=token_classification_task,
+            data_dir=data_args.data_dir,
+            tokenizer=tokenizer,
+            labels=labels,
+            model_type=config.model_type,
+            max_seq_length=data_args.max_seq_length,
+            overwrite_cache=data_args.overwrite_cache,
+            mode=Split.test,
+        )
+
+        predictions, label_ids, metrics = trainer.predict(test_dataset)
+        preds_list, _ = align_predictions(predictions, label_ids)
+
+        output_test_results_file = os.path.join(training_args.output_dir, "test_results.txt")
+        if trainer.is_world_master():
+            with open(output_test_results_file, "w") as writer:
+                for key, value in metrics.items():
+                    logger.info("  %s = %s", key, value)
+                    writer.write("%s = %s\n" % (key, value))
+
+        # Save predictions
+        output_test_predictions_file = os.path.join(training_args.output_dir, "test_predictions.txt")
+        if trainer.is_world_master():
+            with open(output_test_predictions_file, "w") as writer:
+                with open(os.path.join(data_args.data_dir, "test.txt"), "r") as f:
+                    token_classification_task.write_predictions_to_file(writer, f, preds_list)
+
+    return results
+
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+
+if __name__ == "__main__":
+    main()

examples/token-classification/run_old.sh

@@ -0,0 +1,36 @@
+## The relevant files are currently on a shared Google
+## drive at https://drive.google.com/drive/folders/1kC0I2UGl2ltrluI9NqDjaQJGw5iliw_J
+## Monitor for changes and eventually migrate to nlp dataset
+curl -L 'https://drive.google.com/uc?export=download&id=1Jjhbal535VVz2ap4v4r_rN1UEHTdLK5P' \
+| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > train.txt.tmp
+curl -L 'https://drive.google.com/uc?export=download&id=1ZfRcQThdtAR5PPRjIDtrVP7BtXSCUBbm' \
+| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > dev.txt.tmp
+curl -L 'https://drive.google.com/uc?export=download&id=1u9mb7kNJHWQCWyweMDRMuTFoOHOfeBTH' \
+| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > test.txt.tmp
+
+export MAX_LENGTH=128
+export BERT_MODEL=bert-base-multilingual-cased
+python3 scripts/preprocess.py train.txt.tmp $BERT_MODEL $MAX_LENGTH > train.txt
+python3 scripts/preprocess.py dev.txt.tmp $BERT_MODEL $MAX_LENGTH > dev.txt
+python3 scripts/preprocess.py test.txt.tmp $BERT_MODEL $MAX_LENGTH > test.txt
+cat train.txt dev.txt test.txt | cut -d " " -f 2 | grep -v "^$"| sort | uniq > labels.txt
+export OUTPUT_DIR=germeval-model
+export BATCH_SIZE=32
+export NUM_EPOCHS=3
+export SAVE_STEPS=750
+export SEED=1
+
+python3 run_ner_old.py \
+--task_type NER \
+--data_dir . \
+--labels ./labels.txt \
+--model_name_or_path $BERT_MODEL \
+--output_dir $OUTPUT_DIR \
+--max_seq_length  $MAX_LENGTH \
+--num_train_epochs $NUM_EPOCHS \
+--per_gpu_train_batch_size $BATCH_SIZE \
+--save_steps $SAVE_STEPS \
+--seed $SEED \
+--do_train \
+--do_eval \
+--do_predict

examples/token-classification/run_pos.sh

@@ -21,7 +21,7 @@ export NUM_EPOCHS=3
 export SAVE_STEPS=750
 export SEED=1
 
-python3 run_ner.py \
+python3 run_ner_old.py \
 --task_type POS \
 --data_dir . \
 --model_name_or_path $BERT_MODEL \

examples/token-classification/run_tf_ner.py

@@ -33,9 +33,15 @@ from transformers import (
     TFTrainer,
     TFTrainingArguments,
 )
+from transformers.utils import logging as hf_logging
 from utils_ner import Split, TFTokenClassificationDataset, TokenClassificationTask
 
 
+hf_logging.set_verbosity_info()
+hf_logging.enable_default_handler()
+hf_logging.enable_explicit_format()
+
+
 logger = logging.getLogger(__name__)
 
 

examples/token-classification/test_ner_examples.py

@@ -3,8 +3,8 @@ import sys
 import unittest
 from unittest.mock import patch
 
-import run_ner
+import run_ner_old as run_ner
-from transformers.testing_utils import slow
+from transformers.testing_utils import require_torch_non_multi_gpu_but_fix_me, slow
 
 
 logging.basicConfig(level=logging.INFO)
@@ -14,6 +14,7 @@ logger = logging.getLogger()
 
 class ExamplesTests(unittest.TestCase):
     @slow
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_ner(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -34,6 +35,7 @@ class ExamplesTests(unittest.TestCase):
             result = run_ner.main()
             self.assertLess(result["eval_loss"], 1.5)
 
+    @require_torch_non_multi_gpu_but_fix_me
     def test_run_ner_pl(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)

examples/token-classification/utils_ner.py

@@ -66,14 +66,16 @@ class Split(Enum):
 
 
 class TokenClassificationTask:
-    def read_examples_from_file(self, data_dir, mode: Union[Split, str]) -> List[InputExample]:
+    @staticmethod
+    def read_examples_from_file(data_dir, mode: Union[Split, str]) -> List[InputExample]:
         raise NotImplementedError
 
-    def get_labels(self, path: str) -> List[str]:
+    @staticmethod
+    def get_labels(path: str) -> List[str]:
         raise NotImplementedError
 
+    @staticmethod
     def convert_examples_to_features(
-        self,
         examples: List[InputExample],
         label_list: List[str],
         max_seq_length: int,

model_cards/TypicaAI/magbert-ner/README.md

@@ -10,11 +10,12 @@ widget:
 
 [MagBERT-NER] is a state-of-the-art NER model for Moroccan French language (Maghreb). The MagBERT-NER model was fine-tuned for NER Task based the language model for French Camembert (based on the RoBERTa architecture).
 
-For further information or requests, please go to [Typica.AI Website](https://typicasoft.io/)
+For further information or requests, please visite our website at [typica.ai Website](https://typica.ai/) or send us an email at contactus@typica.ai
 
 ## How to use MagBERT-NER with HuggingFace
 
 ##### Load MagBERT-NER and its sub-word tokenizer :
+
 ```python
 from transformers import AutoTokenizer, AutoModelForTokenClassification
 
@@ -45,11 +46,14 @@ nlp("Saad Dine El Otmani, né le 16 janvier 1956 à Inezgane, est un homme d'Ét
 
 ```
 
-```
-
 
 ## Authors 
 
-MagBert-NER was trained and evaluated by Hicham Assoudi, Ph.D.
+MagBert-NER Model was trained by Hicham Assoudi, Ph.D. 
+For any questions, comments you can contact me at assoudi@typica.ai
 
 
+## Citation
+
+If you use our work, please cite:
+Hicham Assoudi, Ph.D., MagBERT-NER: a state-of-the-art NER model for Moroccan French language (Maghreb), (2020)

model_cards/allenai/wmt19-de-en-6-6-base/README.md

@@ -61,7 +61,7 @@ Pretrained weights were left identical to the original model released by allenai
 Here are the BLEU scores:
 
 model   |  transformers
--------|---------|----------
+-------|---------
 wmt19-de-en-6-6-base  |  38.37
 
 The score was calculated using this code:

model_cards/allenai/wmt19-de-en-6-6-big/README.md

@@ -61,7 +61,7 @@ Pretrained weights were left identical to the original model released by allenai
 Here are the BLEU scores:
 
 model   |  transformers
--------|---------|----------
+-------|---------
 wmt19-de-en-6-6-big  |  39.9
 
 The score was calculated using this code:

model_cards/bionlp/bluebert_pubmed_mimic_uncased_L-12_H-768_A-12/README.md

@@ -0,0 +1,80 @@
+---
+language: 
+- en
+tags:
+- bert
+- bluebert
+license: 
+- PUBLIC DOMAIN NOTICE
+datasets:
+- PubMed
+- MIMIC-III
+
+---
+
+# BlueBert-Base, Uncased, PubMed and MIMIC-III
+
+## Model description
+
+A BERT model pre-trained on PubMed abstracts and clinical notes ([MIMIC-III](https://mimic.physionet.org/)).
+
+## Intended uses & limitations
+
+#### How to use
+
+Please see https://github.com/ncbi-nlp/bluebert
+
+## Training data
+
+We provide [preprocessed PubMed texts](https://ftp.ncbi.nlm.nih.gov/pub/lu/Suppl/NCBI-BERT/pubmed_uncased_sentence_nltk.txt.tar.gz) that were used to pre-train the BlueBERT models. 
+The corpus contains ~4000M words extracted from the [PubMed ASCII code version](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PubMed/). 
+
+Pre-trained model: https://huggingface.co/bert-base-uncased
+
+## Training procedure
+
+*  lowercasing the text
+*  removing speical chars `\x00`-`\x7F`
+*  tokenizing the text using the [NLTK Treebank tokenizer](https://www.nltk.org/_modules/nltk/tokenize/treebank.html)
+
+Below is a code snippet for more details.
+
+```python
+value = value.lower()
+value = re.sub(r'[\r\n]+', ' ', value)
+value = re.sub(r'[^\x00-\x7F]+', ' ', value)
+
+tokenized = TreebankWordTokenizer().tokenize(value)
+sentence = ' '.join(tokenized)
+sentence = re.sub(r"\s's\b", "'s", sentence)
+```
+
+### BibTeX entry and citation info
+
+```bibtex
+@InProceedings{peng2019transfer,
+  author    = {Yifan Peng and Shankai Yan and Zhiyong Lu},
+  title     = {Transfer Learning in Biomedical Natural Language Processing: An Evaluation of BERT and ELMo on Ten Benchmarking Datasets},
+  booktitle = {Proceedings of the 2019 Workshop on Biomedical Natural Language Processing (BioNLP 2019)},
+  year      = {2019},
+  pages     = {58--65},
+}
+```
+
+### Acknowledgments
+
+This work was supported by the Intramural Research Programs of the National Institutes of Health, National Library of
+Medicine and Clinical Center. This work was supported by the National Library of Medicine of the National Institutes of Health under award number 4R00LM013001-01.
+
+We are also grateful to the authors of BERT and ELMo to make the data and codes publicly available.
+
+We would like to thank Dr Sun Kim for processing the PubMed texts.
+
+### Disclaimer
+
+This tool shows the results of research conducted in the Computational Biology Branch, NCBI. The information produced
+on this website is not intended for direct diagnostic use or medical decision-making without review and oversight
+by a clinical professional. Individuals should not change their health behavior solely on the basis of information
+produced on this website. NIH does not independently verify the validity or utility of the information produced
+by this tool. If you have questions about the information produced on this website, please see a health care
+professional. More information about NCBI's disclaimer policy is available.

model_cards/bionlp/bluebert_pubmed_uncased_L-12_H-768_A-12/README.md

@@ -0,0 +1,60 @@
+---
+language: 
+- en
+tags:
+- bluebert
+license: 
+- PUBLIC DOMAIN NOTICE
+datasets:
+- pubmed
+
+---
+
+# BlueBert-Base, Uncased, PubMed
+
+## Model description
+
+A BERT model pre-trained on PubMed abstracts
+
+## Intended uses & limitations
+
+#### How to use
+
+Please see https://github.com/ncbi-nlp/bluebert
+
+## Training data
+
+We provide [preprocessed PubMed texts](https://ftp.ncbi.nlm.nih.gov/pub/lu/Suppl/NCBI-BERT/pubmed_uncased_sentence_nltk.txt.tar.gz) that were used to pre-train the BlueBERT models. 
+The corpus contains ~4000M words extracted from the [PubMed ASCII code version](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PubMed/). 
+
+Pre-trained model: https://huggingface.co/bert-base-uncased
+
+## Training procedure
+
+*  lowercasing the text
+*  removing speical chars `\x00`-`\x7F`
+*  tokenizing the text using the [NLTK Treebank tokenizer](https://www.nltk.org/_modules/nltk/tokenize/treebank.html)
+
+Below is a code snippet for more details.
+
+```python
+value = value.lower()
+value = re.sub(r'[\r\n]+', ' ', value)
+value = re.sub(r'[^\x00-\x7F]+', ' ', value)
+
+tokenized = TreebankWordTokenizer().tokenize(value)
+sentence = ' '.join(tokenized)
+sentence = re.sub(r"\s's\b", "'s", sentence)
+```
+
+### BibTeX entry and citation info
+
+```bibtex
+@InProceedings{peng2019transfer,
+  author    = {Yifan Peng and Shankai Yan and Zhiyong Lu},
+  title     = {Transfer Learning in Biomedical Natural Language Processing: An Evaluation of BERT and ELMo on Ten Benchmarking Datasets},
+  booktitle = {Proceedings of the 2019 Workshop on Biomedical Natural Language Processing (BioNLP 2019)},
+  year      = {2019},
+  pages     = {58--65},
+}
+```

model_cards/cimm-kzn/enrudr-bert/README.md

@@ -3,9 +3,9 @@ language:
 - ru
 - en
 ---
-## RuDR-BERT
+## EnRuDR-BERT
 
-EnRuDR-BERT - Multilingual, Cased, which pretrained on the raw part of the RuDReC corpus (1.4M reviews) and collecting of consumer comments on drug administration from [2]. Pre-training was based on the [original BERT code](https://github.com/google-research/bert) provided by Google. In particular, Multi-BERT was for used for initialization; vocabulary of Russian subtokens and parameters are the same as in Multi-BERT. Training details are described in our paper. \
+EnRuDR-BERT - Multilingual, Cased, which pretrained on the raw part of the RuDReC corpus (1.4M reviews) and english collection of consumer comments on drug administration from [2]. Pre-training was based on the [original BERT code](https://github.com/google-research/bert) provided by Google. In particular, Multi-BERT was for used for initialization; vocabulary of Russian subtokens and parameters are the same as in Multi-BERT. Training details are described in our paper. \
    link: https://yadi.sk/d/-PTn0xhk1PqvgQ
    
 

model_cards/dbmdz/bert-base-italian-cased/README.md

@@ -1,12 +1,14 @@
 ---
 language: it
 license: mit
+datasets:
+- wikipedia
 ---
 
-# 🤗 + 📚 dbmdz BERT models
+# 🤗 + 📚 dbmdz BERT and ELECTRA models
 
 In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
-Library open sources Italian BERT models 🎉
+Library open sources Italian BERT and ELECTRA models 🎉
 
 # Italian BERT
 
@@ -22,23 +24,35 @@ For the XXL Italian models, we use the same training data from OPUS and extend
 it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/).
 Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens.
 
+Note: Unfortunately, a wrong vocab size was used when training the XXL models.
+This explains the mismatch of the "real" vocab size of 31102, compared to the
+vocab size specified in `config.json`. However, the model is working and all
+evaluations were done under those circumstances.
+See [this issue](https://github.com/dbmdz/berts/issues/7) for more information.
+
+The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch
+size of 128. We pretty much following the ELECTRA training procedure as used for
+[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra).
+
 ## Model weights
 
 Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
 compatible weights are available. If you need access to TensorFlow checkpoints,
 please raise an issue!
 
-| Model                                   | Downloads
+| Model                                                | Downloads
-| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------
-| `dbmdz/bert-base-italian-cased`         | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)       • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)       • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
+| `dbmdz/bert-base-italian-cased`                      | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)                                               • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)                      • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
-| `dbmdz/bert-base-italian-uncased`       | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-uncased`                    | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)                                             • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)                    • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
-| `dbmdz/bert-base-italian-xxl-cased`     | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)   • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)   • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-cased`                  | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)                                           • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)                  • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
-| `dbmdz/bert-base-italian-xxl-uncased`   | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-uncased`                | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json)                                         • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin)                • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-generator`     | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt)
 
 ## Results
 
 For results on downstream tasks like NER or PoS tagging, please refer to
-[this repository](https://github.com/stefan-it/fine-tuned-berts-seq).
+[this repository](https://github.com/stefan-it/italian-bertelectra).
 
 ## Usage
 
@@ -47,8 +61,11 @@ With Transformers >= 2.3 our Italian BERT models can be loaded like:
 ```python
 from transformers import AutoModel, AutoTokenizer
 
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-italian-cased")
+model_name = "dbmdz/bert-base-italian-cased"
-model = AutoModel.from_pretrained("dbmdz/bert-base-italian-cased")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
 ```
 
 To load the (recommended) Italian XXL BERT models, just use:
@@ -56,8 +73,23 @@ To load the (recommended) Italian XXL BERT models, just use:
 ```python
 from transformers import AutoModel, AutoTokenizer
 
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-italian-xxl-cased")
+model_name = "dbmdz/bert-base-italian-xxl-cased"
-model = AutoModel.from_pretrained("dbmdz/bert-base-italian-xxl-cased")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
+```
+
+To load the Italian XXL ELECTRA model (discriminator), just use:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModelWithLMHead.from_pretrained(model_name)
 ```
 
 # Huggingface model hub
@@ -66,7 +98,7 @@ All models are available on the [Huggingface model hub](https://huggingface.co/d
 
 # Contact (Bugs, Feedback, Contribution and more)
 
-For questions about our BERT models just open an issue
+For questions about our BERT/ELECTRA models just open an issue
 [here](https://github.com/dbmdz/berts/issues/new) 🤗
 
 # Acknowledgments

model_cards/dbmdz/bert-base-italian-uncased/README.md

@@ -1,12 +1,14 @@
 ---
 language: it
 license: mit
+datasets:
+- wikipedia
 ---
 
-# 🤗 + 📚 dbmdz BERT models
+# 🤗 + 📚 dbmdz BERT and ELECTRA models
 
 In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
-Library open sources Italian BERT models 🎉
+Library open sources Italian BERT and ELECTRA models 🎉
 
 # Italian BERT
 
@@ -22,23 +24,35 @@ For the XXL Italian models, we use the same training data from OPUS and extend
 it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/).
 Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens.
 
+Note: Unfortunately, a wrong vocab size was used when training the XXL models.
+This explains the mismatch of the "real" vocab size of 31102, compared to the
+vocab size specified in `config.json`. However, the model is working and all
+evaluations were done under those circumstances.
+See [this issue](https://github.com/dbmdz/berts/issues/7) for more information.
+
+The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch
+size of 128. We pretty much following the ELECTRA training procedure as used for
+[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra).
+
 ## Model weights
 
 Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
 compatible weights are available. If you need access to TensorFlow checkpoints,
 please raise an issue!
 
-| Model                                   | Downloads
+| Model                                                | Downloads
-| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------
-| `dbmdz/bert-base-italian-cased`         | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)       • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)       • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
+| `dbmdz/bert-base-italian-cased`                      | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)                                               • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)                      • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
-| `dbmdz/bert-base-italian-uncased`       | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-uncased`                    | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)                                             • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)                    • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
-| `dbmdz/bert-base-italian-xxl-cased`     | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)   • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)   • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-cased`                  | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)                                           • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)                  • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
-| `dbmdz/bert-base-italian-xxl-uncased`   | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-uncased`                | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json)                                         • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin)                • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-generator`     | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt)
 
 ## Results
 
 For results on downstream tasks like NER or PoS tagging, please refer to
-[this repository](https://github.com/stefan-it/fine-tuned-berts-seq).
+[this repository](https://github.com/stefan-it/italian-bertelectra).
 
 ## Usage
 
@@ -47,8 +61,11 @@ With Transformers >= 2.3 our Italian BERT models can be loaded like:
 ```python
 from transformers import AutoModel, AutoTokenizer
 
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-italian-cased")
+model_name = "dbmdz/bert-base-italian-cased"
-model = AutoModel.from_pretrained("dbmdz/bert-base-italian-cased")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
 ```
 
 To load the (recommended) Italian XXL BERT models, just use:
@@ -56,8 +73,23 @@ To load the (recommended) Italian XXL BERT models, just use:
 ```python
 from transformers import AutoModel, AutoTokenizer
 
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-italian-xxl-cased")
+model_name = "dbmdz/bert-base-italian-xxl-cased"
-model = AutoModel.from_pretrained("dbmdz/bert-base-italian-xxl-cased")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
+```
+
+To load the Italian XXL ELECTRA model (discriminator), just use:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModelWithLMHead.from_pretrained(model_name)
 ```
 
 # Huggingface model hub
@@ -66,7 +98,7 @@ All models are available on the [Huggingface model hub](https://huggingface.co/d
 
 # Contact (Bugs, Feedback, Contribution and more)
 
-For questions about our BERT models just open an issue
+For questions about our BERT/ELECTRA models just open an issue
 [here](https://github.com/dbmdz/berts/issues/new) 🤗
 
 # Acknowledgments

model_cards/dbmdz/bert-base-italian-xxl-cased/README.md

@@ -1,12 +1,14 @@
 ---
 language: it
 license: mit
+datasets:
+- wikipedia
 ---
 
-# 🤗 + 📚 dbmdz BERT models
+# 🤗 + 📚 dbmdz BERT and ELECTRA models
 
 In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
-Library open sources Italian BERT models 🎉
+Library open sources Italian BERT and ELECTRA models 🎉
 
 # Italian BERT
 
@@ -22,23 +24,35 @@ For the XXL Italian models, we use the same training data from OPUS and extend
 it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/).
 Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens.
 
+Note: Unfortunately, a wrong vocab size was used when training the XXL models.
+This explains the mismatch of the "real" vocab size of 31102, compared to the
+vocab size specified in `config.json`. However, the model is working and all
+evaluations were done under those circumstances.
+See [this issue](https://github.com/dbmdz/berts/issues/7) for more information.
+
+The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch
+size of 128. We pretty much following the ELECTRA training procedure as used for
+[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra).
+
 ## Model weights
 
 Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
 compatible weights are available. If you need access to TensorFlow checkpoints,
 please raise an issue!
 
-| Model                                   | Downloads
+| Model                                                | Downloads
-| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------
-| `dbmdz/bert-base-italian-cased`         | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)       • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)       • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
+| `dbmdz/bert-base-italian-cased`                      | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)                                               • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)                      • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
-| `dbmdz/bert-base-italian-uncased`       | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-uncased`                    | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)                                             • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)                    • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
-| `dbmdz/bert-base-italian-xxl-cased`     | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)   • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)   • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-cased`                  | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)                                           • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)                  • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
-| `dbmdz/bert-base-italian-xxl-uncased`   | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-uncased`                | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json)                                         • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin)                • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-generator`     | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt)
 
 ## Results
 
 For results on downstream tasks like NER or PoS tagging, please refer to
-[this repository](https://github.com/stefan-it/fine-tuned-berts-seq).
+[this repository](https://github.com/stefan-it/italian-bertelectra).
 
 ## Usage
 
@@ -47,8 +61,11 @@ With Transformers >= 2.3 our Italian BERT models can be loaded like:
 ```python
 from transformers import AutoModel, AutoTokenizer
 
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-italian-cased")
+model_name = "dbmdz/bert-base-italian-cased"
-model = AutoModel.from_pretrained("dbmdz/bert-base-italian-cased")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
 ```
 
 To load the (recommended) Italian XXL BERT models, just use:
@@ -56,8 +73,23 @@ To load the (recommended) Italian XXL BERT models, just use:
 ```python
 from transformers import AutoModel, AutoTokenizer
 
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-italian-xxl-cased")
+model_name = "dbmdz/bert-base-italian-xxl-cased"
-model = AutoModel.from_pretrained("dbmdz/bert-base-italian-xxl-cased")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
+```
+
+To load the Italian XXL ELECTRA model (discriminator), just use:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModelWithLMHead.from_pretrained(model_name)
 ```
 
 # Huggingface model hub
@@ -66,7 +98,7 @@ All models are available on the [Huggingface model hub](https://huggingface.co/d
 
 # Contact (Bugs, Feedback, Contribution and more)
 
-For questions about our BERT models just open an issue
+For questions about our BERT/ELECTRA models just open an issue
 [here](https://github.com/dbmdz/berts/issues/new) 🤗
 
 # Acknowledgments

model_cards/dbmdz/bert-base-italian-xxl-uncased/README.md

@@ -1,12 +1,14 @@
 ---
 language: it
 license: mit
+datasets:
+- wikipedia
 ---
 
-# 🤗 + 📚 dbmdz BERT models
+# 🤗 + 📚 dbmdz BERT and ELECTRA models
 
 In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
-Library open sources Italian BERT models 🎉
+Library open sources Italian BERT and ELECTRA models 🎉
 
 # Italian BERT
 
@@ -22,23 +24,35 @@ For the XXL Italian models, we use the same training data from OPUS and extend
 it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/).
 Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens.
 
+Note: Unfortunately, a wrong vocab size was used when training the XXL models.
+This explains the mismatch of the "real" vocab size of 31102, compared to the
+vocab size specified in `config.json`. However, the model is working and all
+evaluations were done under those circumstances.
+See [this issue](https://github.com/dbmdz/berts/issues/7) for more information.
+
+The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch
+size of 128. We pretty much following the ELECTRA training procedure as used for
+[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra).
+
 ## Model weights
 
 Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
 compatible weights are available. If you need access to TensorFlow checkpoints,
 please raise an issue!
 
-| Model                                   | Downloads
+| Model                                                | Downloads
-| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------
-| `dbmdz/bert-base-italian-cased`         | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)       • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)       • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
+| `dbmdz/bert-base-italian-cased`                      | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)                                               • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)                      • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
-| `dbmdz/bert-base-italian-uncased`       | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-uncased`                    | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)                                             • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)                    • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
-| `dbmdz/bert-base-italian-xxl-cased`     | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)   • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)   • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-cased`                  | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)                                           • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)                  • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
-| `dbmdz/bert-base-italian-xxl-uncased`   | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-uncased`                | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json)                                         • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin)                • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-generator`     | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt)
 
 ## Results
 
 For results on downstream tasks like NER or PoS tagging, please refer to
-[this repository](https://github.com/stefan-it/fine-tuned-berts-seq).
+[this repository](https://github.com/stefan-it/italian-bertelectra).
 
 ## Usage
 
@@ -47,8 +61,11 @@ With Transformers >= 2.3 our Italian BERT models can be loaded like:
 ```python
 from transformers import AutoModel, AutoTokenizer
 
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-italian-cased")
+model_name = "dbmdz/bert-base-italian-cased"
-model = AutoModel.from_pretrained("dbmdz/bert-base-italian-cased")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
 ```
 
 To load the (recommended) Italian XXL BERT models, just use:
@@ -56,8 +73,23 @@ To load the (recommended) Italian XXL BERT models, just use:
 ```python
 from transformers import AutoModel, AutoTokenizer
 
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-italian-xxl-cased")
+model_name = "dbmdz/bert-base-italian-xxl-cased"
-model = AutoModel.from_pretrained("dbmdz/bert-base-italian-xxl-cased")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
+```
+
+To load the Italian XXL ELECTRA model (discriminator), just use:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModelWithLMHead.from_pretrained(model_name)
 ```
 
 # Huggingface model hub
@@ -66,7 +98,7 @@ All models are available on the [Huggingface model hub](https://huggingface.co/d
 
 # Contact (Bugs, Feedback, Contribution and more)
 
-For questions about our BERT models just open an issue
+For questions about our BERT/ELECTRA models just open an issue
 [here](https://github.com/dbmdz/berts/issues/new) 🤗
 
 # Acknowledgments

model_cards/dbmdz/electra-base-italian-xxl-cased-discriminator/README.md

@@ -0,0 +1,110 @@
+---
+language: it
+license: mit
+datasets:
+- wikipedia
+---
+
+# 🤗 + 📚 dbmdz BERT and ELECTRA models
+
+In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
+Library open sources Italian BERT and ELECTRA models 🎉
+
+# Italian BERT
+
+The source data for the Italian BERT model consists of a recent Wikipedia dump and
+various texts from the [OPUS corpora](http://opus.nlpl.eu/) collection. The final
+training corpus has a size of 13GB and 2,050,057,573 tokens.
+
+For sentence splitting, we use NLTK (faster compared to spacy).
+Our cased and uncased models are training with an initial sequence length of 512
+subwords for ~2-3M steps.
+
+For the XXL Italian models, we use the same training data from OPUS and extend
+it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/).
+Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens.
+
+Note: Unfortunately, a wrong vocab size was used when training the XXL models.
+This explains the mismatch of the "real" vocab size of 31102, compared to the
+vocab size specified in `config.json`. However, the model is working and all
+evaluations were done under those circumstances.
+See [this issue](https://github.com/dbmdz/berts/issues/7) for more information.
+
+The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch
+size of 128. We pretty much following the ELECTRA training procedure as used for
+[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra).
+
+## Model weights
+
+Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
+compatible weights are available. If you need access to TensorFlow checkpoints,
+please raise an issue!
+
+| Model                                                | Downloads
+| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| `dbmdz/bert-base-italian-cased`                      | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)                                               • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)                      • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
+| `dbmdz/bert-base-italian-uncased`                    | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)                                             • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)                    • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-cased`                  | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)                                           • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)                  • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-uncased`                | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json)                                         • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin)                • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-generator`     | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt)
+
+## Results
+
+For results on downstream tasks like NER or PoS tagging, please refer to
+[this repository](https://github.com/stefan-it/italian-bertelectra).
+
+## Usage
+
+With Transformers >= 2.3 our Italian BERT models can be loaded like:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/bert-base-italian-cased"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
+```
+
+To load the (recommended) Italian XXL BERT models, just use:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/bert-base-italian-xxl-cased"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
+```
+
+To load the Italian XXL ELECTRA model (discriminator), just use:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModelWithLMHead.from_pretrained(model_name)
+```
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
+
+# Contact (Bugs, Feedback, Contribution and more)
+
+For questions about our BERT/ELECTRA models just open an issue
+[here](https://github.com/dbmdz/berts/issues/new) 🤗
+
+# Acknowledgments
+
+Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
+Thanks for providing access to the TFRC ❤️
+
+Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
+it is possible to download both cased and uncased models from their S3 storage 🤗

model_cards/dbmdz/electra-base-italian-xxl-cased-generator/README.md

@@ -0,0 +1,110 @@
+---
+language: it
+license: mit
+datasets:
+- wikipedia
+---
+
+# 🤗 + 📚 dbmdz BERT and ELECTRA models
+
+In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
+Library open sources Italian BERT and ELECTRA models 🎉
+
+# Italian BERT
+
+The source data for the Italian BERT model consists of a recent Wikipedia dump and
+various texts from the [OPUS corpora](http://opus.nlpl.eu/) collection. The final
+training corpus has a size of 13GB and 2,050,057,573 tokens.
+
+For sentence splitting, we use NLTK (faster compared to spacy).
+Our cased and uncased models are training with an initial sequence length of 512
+subwords for ~2-3M steps.
+
+For the XXL Italian models, we use the same training data from OPUS and extend
+it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/).
+Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens.
+
+Note: Unfortunately, a wrong vocab size was used when training the XXL models.
+This explains the mismatch of the "real" vocab size of 31102, compared to the
+vocab size specified in `config.json`. However, the model is working and all
+evaluations were done under those circumstances.
+See [this issue](https://github.com/dbmdz/berts/issues/7) for more information.
+
+The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch
+size of 128. We pretty much following the ELECTRA training procedure as used for
+[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra).
+
+## Model weights
+
+Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
+compatible weights are available. If you need access to TensorFlow checkpoints,
+please raise an issue!
+
+| Model                                                | Downloads
+| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| `dbmdz/bert-base-italian-cased`                      | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json)                                               • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin)                      • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt)
+| `dbmdz/bert-base-italian-uncased`                    | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json)                                             • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin)                    • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-cased`                  | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json)                                           • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin)                  • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt)
+| `dbmdz/bert-base-italian-xxl-uncased`                | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json)                                         • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin)                • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt)
+| `dbmdz/electra-base-italian-xxl-cased-generator`     | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json)     • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin)     • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt)
+
+## Results
+
+For results on downstream tasks like NER or PoS tagging, please refer to
+[this repository](https://github.com/stefan-it/italian-bertelectra).
+
+## Usage
+
+With Transformers >= 2.3 our Italian BERT models can be loaded like:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/bert-base-italian-cased"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
+```
+
+To load the (recommended) Italian XXL BERT models, just use:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/bert-base-italian-xxl-cased"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModel.from_pretrained(model_name)
+```
+
+To load the Italian XXL ELECTRA model (discriminator), just use:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+model = AutoModelWithLMHead.from_pretrained(model_name)
+```
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
+
+# Contact (Bugs, Feedback, Contribution and more)
+
+For questions about our BERT/ELECTRA models just open an issue
+[here](https://github.com/dbmdz/berts/issues/new) 🤗
+
+# Acknowledgments
+
+Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
+Thanks for providing access to the TFRC ❤️
+
+Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
+it is possible to download both cased and uncased models from their S3 storage 🤗