Remove use of deprected method in Trainer HP search (#8996)

* Remove deprecated `evalutate_during_training`

* Update src/transformers/training_args_tf.py

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

.circleci/config.yml

@@ -77,7 +77,7 @@ jobs:
                       - v0.4-torch_and_tf-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[sklearn,tf-cpu,torch,testing]
+            - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece]
             - save_cache:
                 key: v0.4-{{ checksum "setup.py" }}
                 paths:
@@ -103,7 +103,7 @@ jobs:
                       - v0.4-torch-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[sklearn,torch,testing]
+            - run: pip install .[sklearn,torch,testing,sentencepiece]
             - save_cache:
                   key: v0.4-torch-{{ checksum "setup.py" }}
                   paths:
@@ -129,7 +129,7 @@ jobs:
                       - v0.4-tf-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[sklearn,tf-cpu,testing]
+            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
             - save_cache:
                   key: v0.4-tf-{{ checksum "setup.py" }}
                   paths:
@@ -155,7 +155,7 @@ jobs:
                     - v0.4-flax-{{ checksum "setup.py" }}
                     - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: sudo pip install .[flax,sklearn,torch,testing]
+            - run: sudo pip install .[flax,sklearn,torch,testing,sentencepiece]
             - save_cache:
                   key: v0.4-flax-{{ checksum "setup.py" }}
                   paths:
@@ -181,7 +181,7 @@ jobs:
                       - v0.4-torch-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[sklearn,torch,testing]
+            - run: pip install .[sklearn,torch,testing,sentencepiece]
             - save_cache:
                   key: v0.4-torch-{{ checksum "setup.py" }}
                   paths:
@@ -207,7 +207,7 @@ jobs:
                       - v0.4-tf-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[sklearn,tf-cpu,testing]
+            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
             - save_cache:
                   key: v0.4-tf-{{ checksum "setup.py" }}
                   paths:
@@ -231,7 +231,7 @@ jobs:
                       - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[ja,testing]
+            - run: pip install .[ja,testing,sentencepiece]
             - run: python -m unidic download
             - save_cache:
                   key: v0.4-custom_tokenizers-{{ checksum "setup.py" }}
@@ -258,7 +258,7 @@ jobs:
                       - v0.4-torch_examples-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[sklearn,torch,testing]
+            - run: pip install .[sklearn,torch,sentencepiece,testing]
             - run: pip install -r examples/requirements.txt
             - save_cache:
                   key: v0.4-torch_examples-{{ checksum "setup.py" }}
@@ -324,7 +324,7 @@ jobs:
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
             - run: pip install isort
-            - run: pip install .[tf,torch,flax,quality]
+            - run: pip install .[all,quality]
             - save_cache:
                   key: v0.4-code_quality-{{ checksum "setup.py" }}
                   paths:

.github/PULL_REQUEST_TEMPLATE.md

@@ -20,7 +20,7 @@ Fixes # (issue)
 - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/master/CONTRIBUTING.md#start-contributing-pull-requests),
       Pull Request section?
 - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
-      to the it if that's the case.
+      to it if that's the case.
 - [ ] Did you make sure to update the documentation with your changes? Here are the
       [documentation guidelines](https://github.com/huggingface/transformers/tree/master/docs), and
       [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/master/docs#writing-source-documentation).

.github/workflows/self-push.yml

@@ -48,7 +48,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime]
+          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
 
       - name: Are GPUs recognized by our DL frameworks
@@ -117,7 +117,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime]
+          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
 
       - name: Are GPUs recognized by our DL frameworks
@@ -185,7 +185,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime]
+          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
 
       - name: Are GPUs recognized by our DL frameworks
@@ -244,7 +244,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime]
+          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
 
       - name: Are GPUs recognized by our DL frameworks

.github/workflows/self-scheduled.yml

@@ -9,7 +9,6 @@ on:
   push:
     branches:
       - ci_*
-      - framework-agnostic-tokenizers
   repository_dispatch:
   schedule:
     - cron: "0 0 * * *"
@@ -50,7 +49,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime]
+          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
           pip list
 
@@ -144,7 +143,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime]
+          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
           pip list
 
@@ -223,7 +222,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[torch,sklearn,testing,onnxruntime]
+          pip install .[torch,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
           pip list
 
@@ -251,11 +250,11 @@ jobs:
           RUN_SLOW: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s --make-reports=examples_torch_multi_gpu examples
+          python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_torch_examples_multi_gpu examples
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/examples_torch_multi_gpu_failures_short.txt
+        run: cat reports/tests_torch_examples_multi_gpu_failures_short.txt
 
       - name: Run all pipeline tests on multi-GPU
         if: ${{ always() }}
@@ -314,7 +313,7 @@ jobs:
         run: |
           source .env/bin/activate
           pip install --upgrade pip
-          pip install .[tf,sklearn,testing,onnxruntime]
+          pip install .[tf,sklearn,testing,onnxruntime,sentencepiece]
           pip install git+https://github.com/huggingface/datasets
           pip list
 
@@ -345,11 +344,11 @@ jobs:
           RUN_PIPELINE_TESTS: yes
         run: |
           source .env/bin/activate
-          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipelines_multi_gpu tests
+          python -m pytest -n 1 --dist=loadfile -s -m is_pipeline_test --make-reports=tests_tf_pipeline_multi_gpu tests
           
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_tf_multi_gpu_pipelines_failures_short.txt
+        run: cat reports/tests_tf_pipeline_multi_gpu_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}

README.md

@@ -197,6 +197,8 @@ ultilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/
 1. **[Other community models](https://huggingface.co/models)**, contributed by the [community](https://huggingface.co/users).
 1. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
 
+To cehck if each model has an implementation in PyTorch/TensorFlow/Flax or has an associated tokenizer backed by the 🤗 Tokenizers library, refer to [this table](https://huggingface.co/transformers/index.html#bigtable)
+
 These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations. You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html).
 
 

docs/source/_static/css/huggingface.css

@@ -2,6 +2,15 @@
 
 /* Colab dropdown */
 
+table.center-aligned-table td {
+    text-align: center;
+}
+
+table.center-aligned-table th {
+    text-align: center;
+    vertical-align: middle;
+}
+
 .colab-dropdown {
     position: relative;
     display: inline-block;

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'3.5.0'
+release = u'4.0.0'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/index.rst

@@ -35,6 +35,8 @@ Choose the right framework for every part of a model's lifetime:
 - Move a single model between TF2.0/PyTorch frameworks at will
 - Seamlessly pick the right framework for training, evaluation, production
 
+Experimental support for Flax with a few models right now, expected to grow in the coming months.
+
 Contents
 -----------------------------------------------------------------------------------------------------------------------
 
@@ -44,7 +46,7 @@ The documentation is organized in five parts:
   and a glossary.
 - **USING 🤗 TRANSFORMERS** contains general tutorials on how to use the library.
 - **ADVANCED GUIDES** contains more advanced guides that are more specific to a given script or part of the library.
-- **RESEARCH** focuses on tutorials that have less to do with how to use the library but more about general resarch in
+- **RESEARCH** focuses on tutorials that have less to do with how to use the library but more about general research in
   transformers model
 - The three last section contain the documentation of each public class and function, grouped in:
 
@@ -52,8 +54,8 @@ The documentation is organized in five parts:
     - **MODELS** for the classes and functions related to each model implemented in the library.
     - **INTERNAL HELPERS** for the classes and functions we use internally.
 
-The library currently contains PyTorch and Tensorflow implementations, pre-trained model weights, usage scripts and
-conversion utilities for the following models:
+The library currently contains PyTorch, Tensorflow and Flax implementations, pretrained model weights, usage scripts
+and conversion utilities for the following models:
 
 ..
     This list is updated automatically from the README with `make fix-copies`. Do not update manually!
@@ -166,6 +168,97 @@ conversion utilities for the following models:
 34. `Other community models <https://huggingface.co/models>`__, contributed by the `community
     <https://huggingface.co/users>`__.
 
+
+.. _bigtable:
+
+The table below represents the current support in the library for each of those models, whether they have a Python
+tokenizer (called "slow"). A "fast" tokenizer backed by the 🤗 Tokenizers library, whether they have support in PyTorch,
+TensorFlow and/or Flax.
+
+..
+    This table is updated automatically from the auto modules with `make fix-copies`. Do not update manually!
+
+.. rst-class:: center-aligned-table
+
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|            Model            | Tokenizer slow | Tokenizer fast | PyTorch support | TensorFlow support | Flax Support |
++=============================+================+================+=================+====================+==============+
+|           ALBERT            |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|            BART             |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|            BERT             |       ✅       |       ✅       |       ✅        |         ✅         |      ✅      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|       Bert Generation       |       ✅       |       ❌       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|         Blenderbot          |       ✅       |       ❌       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|            CTRL             |       ✅       |       ❌       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|          CamemBERT          |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|             DPR             |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|           DeBERTa           |       ✅       |       ❌       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|         DistilBERT          |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|           ELECTRA           |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|       Encoder decoder       |       ❌       |       ❌       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+| FairSeq Machine-Translation |       ✅       |       ❌       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|          FlauBERT           |       ✅       |       ❌       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|     Funnel Transformer      |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|           LXMERT            |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|          LayoutLM           |       ✅       |       ✅       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|         Longformer          |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|           Marian            |       ✅       |       ❌       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|         MobileBERT          |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|         OpenAI GPT          |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|        OpenAI GPT-2         |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|           Pegasus           |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|         ProphetNet          |       ✅       |       ❌       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|             RAG             |       ✅       |       ❌       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|          Reformer           |       ✅       |       ✅       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|          RetriBERT          |       ✅       |       ✅       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|           RoBERTa           |       ✅       |       ✅       |       ✅        |         ✅         |      ✅      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|         SqueezeBERT         |       ✅       |       ✅       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|             T5              |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|       Transformer-XL        |       ✅       |       ❌       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|             XLM             |       ✅       |       ❌       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|         XLM-RoBERTa         |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|        XLMProphetNet        |       ✅       |       ❌       |       ✅        |         ❌         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|            XLNet            |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|            mBART            |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+|             mT5             |       ✅       |       ✅       |       ✅        |         ✅         |      ❌      |
++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
+
+
 .. toctree::
     :maxdepth: 2
     :caption: Get started

docs/source/installation.md

@@ -70,15 +70,15 @@ to check 🤗 Transformers is properly installed.
 
 This library provides pretrained models that will be downloaded and cached locally. Unless you specify a location with
 `cache_dir=...` when you use methods like `from_pretrained`, these models will automatically be downloaded in the
-folder given by the shell environment variable ``TRANSFORMERS_CACHE``. The default value for it will be the PyTorch
-cache home followed by ``/transformers/`` (even if you don't have PyTorch installed). This is (by order of priority):
+folder given by the shell environment variable ``TRANSFORMERS_CACHE``. The default value for it will be the Hugging
+Face cache home followed by ``/transformers/``. This is (by order of priority):
 
-  * shell environment variable ``TORCH_HOME``
-  * shell environment variable ``XDG_CACHE_HOME`` + ``/torch/``
-  * default: ``~/.cache/torch/``
+  * shell environment variable ``HF_HOME`` 
+  * shell environment variable ``XDG_CACHE_HOME`` + ``/huggingface/``
+  * default: ``~/.cache/huggingface/``
 
 So if you don't have any specific environment variable set, the cache directory will be at
-``~/.cache/torch/transformers/``.
+``~/.cache/huggingface/transformers/``.
 
 **Note:** If you have set a shell environment variable for one of the predecessors of this library
 (``PYTORCH_TRANSFORMERS_CACHE`` or ``PYTORCH_PRETRAINED_BERT_CACHE``), those will be used if there is no shell
@@ -97,6 +97,6 @@ You should check out our [swift-coreml-transformers](https://github.com/huggingf
 It contains a set of tools to convert PyTorch or TensorFlow 2.0 trained Transformer models (currently contains `GPT-2`, 
 `DistilGPT-2`, `BERT`, and `DistilBERT`) to CoreML models that run on iOS devices.
 
-At some point in the future, you'll be able to seamlessly move from pre-training or fine-tuning models in PyTorch or
+At some point in the future, you'll be able to seamlessly move from pretraining or fine-tuning models in PyTorch or
 TensorFlow 2.0 to productizing them in CoreML, or prototype a model or an app in CoreML then research its
 hyperparameters or architecture from PyTorch or TensorFlow 2.0. Super exciting!

docs/source/migration.md

@@ -1,5 +1,170 @@
 # Migrating from previous packages
 
+## Migrating from transformers `v3.x` to `v4.x`
+
+A couple of changes were introduced when the switch from version 3 to version 4 was done. Below is a summary of the
+expected changes:
+
+#### 1. AutoTokenizers and pipelines now use fast (rust) tokenizers by default.
+
+The python and rust tokenizers have roughly the same API, but the rust tokenizers have a more complete feature set. 
+
+This introduces two breaking changes:
+- The handling of overflowing tokens between the python and rust tokenizers is different.
+- The rust tokenizers do not accept integers in the encoding methods.
+
+##### How to obtain the same behavior as v3.x in v4.x
+
+- The pipelines now contain additional features out of the box. See the [token-classification pipeline with the `grouped_entities` flag](https://huggingface.co/transformers/main_classes/pipelines.html?highlight=textclassification#tokenclassificationpipeline).
+- The auto-tokenizers now return rust tokenizers. In order to obtain the python tokenizers instead, the user may use the `use_fast` flag by setting it to `False`:
+
+In version `v3.x`:
+```py
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+```
+to obtain the same in version `v4.x`:
+```py
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("bert-base-cased", use_fast=False)
+```
+
+#### 2. SentencePiece is removed from the required dependencies
+
+The requirement on the SentencePiece dependency has been lifted from the `setup.py`. This is done so that we may have a channel on anaconda cloud without relying on `conda-forge`. This means that the tokenizers that depend on the SentencePiece library will not be available with a standard `transformers` installation.
+
+This includes the **slow** versions of:
+- `XLNetTokenizer`
+- `AlbertTokenizer`
+- `CamembertTokenizer`
+- `MBartTokenizer`
+- `PegasusTokenizer`
+- `T5Tokenizer`
+- `ReformerTokenizer`
+- `XLMRobertaTokenizer`
+
+##### How to obtain the same behavior as v3.x in v4.x
+
+In order to obtain the same behavior as version `v3.x`, you should install `sentencepiece` additionally:
+
+In version `v3.x`:
+```bash
+pip install transformers
+```
+to obtain the same in version `v4.x`:
+```bash
+pip install transformers[sentencepiece]
+```
+or
+```bash
+pip install transformers sentencepiece
+```
+#### 3. The architecture of the repo has been updated so that each model resides in its folder
+
+The past and foreseeable addition of new models means that the number of files in the directory `src/transformers` keeps growing and becomes harder to navigate and understand. We made the choice to put each model and the files accompanying it in their own sub-directories.
+
+This is a breaking change as importing intermediary layers using a model's module directly needs to be done via a different path.
+
+##### How to obtain the same behavior as v3.x in v4.x
+
+In order to obtain the same behavior as version `v3.x`, you should update the path used to access the layers. 
+
+In version `v3.x`:
+```bash
+from transformers.modeling_bert import BertLayer
+```
+to obtain the same in version `v4.x`:
+```bash
+from transformers.models.bert.modeling_bert import BertLayer
+```
+
+#### 4. Switching the `return_dict` argument to `True` by default
+
+The [`return_dict` argument](https://huggingface.co/transformers/main_classes/output.html) enables the return of dict-like python objects containing the model outputs, instead of the standard tuples. This object is self-documented as keys can be used to retrieve values, while also behaving as a tuple as users may retrieve objects by index or by slice.
+
+This is a breaking change as the limitation of that tuple is that it cannot be unpacked: `value0, value1 = outputs` will not work.
+
+##### How to obtain the same behavior as v3.x in v4.x
+
+In order to obtain the same behavior as version `v3.x`, you should specify the `return_dict` argument to `False`, either in the model configuration or during the forward pass.
+
+In version `v3.x`:
+```bash
+model = BertModel.from_pretrained("bert-base-cased")
+outputs = model(**inputs)
+```
+to obtain the same in version `v4.x`:
+```bash
+model = BertModel.from_pretrained("bert-base-cased")
+outputs = model(**inputs, return_dict=False)
+```
+or
+```bash
+model = BertModel.from_pretrained("bert-base-cased", return_dict=False)
+outputs = model(**inputs)
+```
+
+#### 5. Removed some deprecated attributes
+
+Attributes that were deprecated have been removed if they had been deprecated for at least a month. The full list of deprecated attributes can be found in [#8604](https://github.com/huggingface/transformers/pull/8604).
+
+Here is a list of these attributes/methods/arguments and what their replacements should be:
+
+In several models, the labels become consistent with the other models:
+- `masked_lm_labels` becomes `labels` in `AlbertForMaskedLM` and `AlbertForPreTraining`.
+- `masked_lm_labels` becomes `labels` in `BertForMaskedLM` and `BertForPreTraining`.
+- `masked_lm_labels` becomes `labels` in `DistilBertForMaskedLM`.
+- `masked_lm_labels` becomes `labels` in `ElectraForMaskedLM`.
+- `masked_lm_labels` becomes `labels` in `LongformerForMaskedLM`.
+- `masked_lm_labels` becomes `labels` in `MobileBertForMaskedLM`.
+- `masked_lm_labels` becomes `labels` in `RobertaForMaskedLM`.
+- `lm_labels` becomes `labels` in `BartForConditionalGeneration`.
+- `lm_labels` becomes `labels` in `GPT2DoubleHeadsModel`.
+- `lm_labels` becomes `labels` in `OpenAIGPTDoubleHeadsModel`.
+- `lm_labels` becomes `labels` in `T5ForConditionalGeneration`.
+
+In several models, the caching mechanism becomes consistent with the other models:
+- `decoder_cached_states` becomes `past_key_values` in all BART-like, FSMT and T5 models.
+- `decoder_past_key_values` becomes `past_key_values` in all BART-like, FSMT and T5 models.
+- `past` becomes `past_key_values` in all CTRL models.
+- `past` becomes `past_key_values` in all GPT-2 models.
+
+Regarding the tokenizer classes:
+- The tokenizer attribute `max_len` becomes `model_max_length`.
+- The tokenizer attribute `return_lengths` becomes `return_length`.
+- The tokenizer encoding argument `is_pretokenized` becomes `is_split_into_words`.
+
+Regarding the `Trainer` class:
+- The `Trainer` argument `tb_writer` is removed in favor of the callback `TensorBoardCallback(tb_writer=...)`.
+- The `Trainer` argument `prediction_loss_only` is removed in favor of the class argument `args.prediction_loss_only`.
+- The `Trainer` attribute `data_collator` should be a callable.
+- The `Trainer` method `_log` is deprecated in favor of `log`.
+- The `Trainer` method `_training_step` is deprecated in favor of `training_step`.
+- The `Trainer` method `_prediction_loop` is deprecated in favor of `prediction_loop`.
+- The `Trainer` method `is_local_master` is deprecated in favor of `is_local_process_zero`.
+- The `Trainer` method `is_world_master` is deprecated in favor of `is_world_process_zero`.
+
+Regarding the `TFTrainer` class:
+- The `TFTrainer` argument `prediction_loss_only` is removed in favor of the class argument `args.prediction_loss_only`.
+- The `Trainer` method `_log` is deprecated in favor of `log`.
+- The `TFTrainer` method `_prediction_loop` is deprecated in favor of `prediction_loop`.
+- The `TFTrainer` method `_setup_wandb` is deprecated in favor of `setup_wandb`.
+- The `TFTrainer` method `_run_model` is deprecated in favor of `run_model`.
+
+Regarding the `TrainerArgument` class:
+- The `TrainerArgument` argument `evaluate_during_training` is deprecated in favor of `evaluation_strategy`.
+
+Regarding the Transfo-XL model:
+- The Transfo-XL configuration attribute `tie_weight` becomes `tie_words_embeddings`.
+- The Transfo-XL modeling method `reset_length` becomes `reset_memory_length`.
+
+Regarding pipelines:
+- The `FillMaskPipeline` argument `topk` becomes `top_k`.
+
+
+
 ## Migrating from pytorch-transformers to 🤗 Transformers
 
 Here is a quick summary of what you should take care of when migrating from `pytorch-transformers` to 🤗 Transformers.

docs/source/model_doc/bertgeneration.rst

@@ -10,7 +10,7 @@ Tasks <https://arxiv.org/abs/1907.12461>`__ by Sascha Rothe, Shashi Narayan, Ali
 
 The abstract from the paper is the following:
 
-*Unsupervised pre-training of large neural models has recently revolutionized Natural Language Processing. By
+*Unsupervised pretraining of large neural models has recently revolutionized Natural Language Processing. By
 warm-starting from the publicly released checkpoints, NLP practitioners have pushed the state-of-the-art on multiple
 benchmarks while saving significant amounts of compute time. So far the focus has been mainly on the Natural Language
 Understanding tasks. In this paper, we demonstrate the efficacy of pre-trained checkpoints for Sequence Generation. We

docs/source/model_doc/deberta.rst

@@ -20,8 +20,8 @@ disentangled attention mechanism, where each word is represented using two vecto
 position, respectively, and the attention weights among words are computed using disentangled matrices on their
 contents and relative positions. Second, an enhanced mask decoder is used to replace the output softmax layer to
 predict the masked tokens for model pretraining. We show that these two techniques significantly improve the efficiency
-of model pre-training and performance of downstream tasks. Compared to RoBERTa-Large, a DeBERTa model trained on half
-of the training data performs consistently better on a wide range of NLP tasks, achieving improvements on MNLI by +0.9%
+of model pretraining and performance of downstream tasks. Compared to RoBERTa-Large, a DeBERTa model trained on half of
+the training data performs consistently better on a wide range of NLP tasks, achieving improvements on MNLI by +0.9%
 (90.2% vs. 91.1%), on SQuAD v2.0 by +2.3% (88.4% vs. 90.7%) and RACE by +3.6% (83.2% vs. 86.8%). The DeBERTa code and
 pre-trained models will be made publicly available at https://github.com/microsoft/DeBERTa.*
 

docs/source/model_doc/distilbert.rst

@@ -18,9 +18,9 @@ operating these large models in on-the-edge and/or under constrained computation
 remains challenging. In this work, we propose a method to pre-train a smaller general-purpose language representation
 model, called DistilBERT, which can then be fine-tuned with good performances on a wide range of tasks like its larger
 counterparts. While most prior work investigated the use of distillation for building task-specific models, we leverage
-knowledge distillation during the pre-training phase and show that it is possible to reduce the size of a BERT model by
+knowledge distillation during the pretraining phase and show that it is possible to reduce the size of a BERT model by
 40%, while retaining 97% of its language understanding capabilities and being 60% faster. To leverage the inductive
-biases learned by larger models during pre-training, we introduce a triple loss combining language modeling,
+biases learned by larger models during pretraining, we introduce a triple loss combining language modeling,
 distillation and cosine-distance losses. Our smaller, faster and lighter model is cheaper to pre-train and we
 demonstrate its capabilities for on-device computations in a proof-of-concept experiment and a comparative on-device
 study.*

docs/source/model_doc/dpr.rst

@@ -5,7 +5,7 @@ Overview
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Dense Passage Retrieval (DPR) is a set of tools and models for state-of-the-art open-domain Q&A research. It was
-intorduced in `Dense Passage Retrieval for Open-Domain Question Answering <https://arxiv.org/abs/2004.04906>`__ by
+introduced in `Dense Passage Retrieval for Open-Domain Question Answering <https://arxiv.org/abs/2004.04906>`__ by
 Vladimir Karpukhin, Barlas Oğuz, Sewon Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, Wen-tau Yih.
 
 The abstract from the paper is the following:

docs/source/model_doc/electra.rst

@@ -12,14 +12,14 @@ identify which tokens were replaced by the generator in the sequence.
 
 The abstract from the paper is the following:
 
-*Masked language modeling (MLM) pre-training methods such as BERT corrupt the input by replacing some tokens with
-[MASK] and then train a model to reconstruct the original tokens. While they produce good results when transferred to
+*Masked language modeling (MLM) pretraining methods such as BERT corrupt the input by replacing some tokens with [MASK]
+and then train a model to reconstruct the original tokens. While they produce good results when transferred to
 downstream NLP tasks, they generally require large amounts of compute to be effective. As an alternative, we propose a
-more sample-efficient pre-training task called replaced token detection. Instead of masking the input, our approach
+more sample-efficient pretraining task called replaced token detection. Instead of masking the input, our approach
 corrupts it by replacing some tokens with plausible alternatives sampled from a small generator network. Then, instead
 of training a model that predicts the original identities of the corrupted tokens, we train a discriminative model that
 predicts whether each token in the corrupted input was replaced by a generator sample or not. Thorough experiments
-demonstrate this new pre-training task is more efficient than MLM because the task is defined over all input tokens
+demonstrate this new pretraining task is more efficient than MLM because the task is defined over all input tokens
 rather than just the small subset that was masked out. As a result, the contextual representations learned by our
 approach substantially outperform the ones learned by BERT given the same model size, data, and compute. The gains are
 particularly strong for small models; for example, we train a model on one GPU for 4 days that outperforms GPT (trained

docs/source/model_doc/flaubert.rst

@@ -19,7 +19,7 @@ representations (Dai and Le, 2015; Peters et al., 2018; Howard and Ruder, 2018;
 heterogeneous French corpus. Models of different sizes are trained using the new CNRS (French National Centre for
 Scientific Research) Jean Zay supercomputer. We apply our French language models to diverse NLP tasks (text
 classification, paraphrasing, natural language inference, parsing, word sense disambiguation) and show that most of the
-time they outperform other pre-training approaches. Different versions of FlauBERT as well as a unified evaluation
+time they outperform other pretraining approaches. Different versions of FlauBERT as well as a unified evaluation
 protocol for the downstream tasks, called FLUE (French Language Understanding Evaluation), are shared to the research
 community for further reproducible experiments in French NLP.*
 

docs/source/model_doc/gpt.rst

@@ -14,7 +14,7 @@ The abstract from the paper is the following:
 *Natural language understanding comprises a wide range of diverse tasks such as textual entailment, question answering,
 semantic similarity assessment, and document classification. Although large unlabeled text corpora are abundant,
 labeled data for learning these specific tasks is scarce, making it challenging for discriminatively trained models to
-perform adequately. We demonstrate that large gains on these tasks can be realized by generative pre-training of a
+perform adequately. We demonstrate that large gains on these tasks can be realized by generative pretraining of a
 language model on a diverse corpus of unlabeled text, followed by discriminative fine-tuning on each specific task. In
 contrast to previous approaches, we make use of task-aware input transformations during fine-tuning to achieve
 effective transfer while requiring minimal changes to the model architecture. We demonstrate the effectiveness of our

docs/source/model_doc/layoutlm.rst

@@ -6,19 +6,19 @@ Overview
 
 The LayoutLM model was proposed in the paper `LayoutLM: Pre-training of Text and Layout for Document Image
 Understanding <https://arxiv.org/abs/1912.13318>`__ by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei, and
-Ming Zhou. It's a simple but effective pre-training method of text and layout for document image understanding and
+Ming Zhou. It's a simple but effective pretraining method of text and layout for document image understanding and
 information extraction tasks, such as form understanding and receipt understanding.
 
 The abstract from the paper is the following:
 
 *Pre-training techniques have been verified successfully in a variety of NLP tasks in recent years. Despite the
-widespread use of pre-training models for NLP applications, they almost exclusively focus on text-level manipulation,
+widespread use of pretraining models for NLP applications, they almost exclusively focus on text-level manipulation,
 while neglecting layout and style information that is vital for document image understanding. In this paper, we propose
 the \textbf{LayoutLM} to jointly model interactions between text and layout information across scanned document images,
 which is beneficial for a great number of real-world document image understanding tasks such as information extraction
 from scanned documents. Furthermore, we also leverage image features to incorporate words' visual information into
 LayoutLM. To the best of our knowledge, this is the first time that text and layout are jointly learned in a single
-framework for document-level pre-training. It achieves new state-of-the-art results in several downstream tasks,
+framework for document-level pretraining. It achieves new state-of-the-art results in several downstream tasks,
 including form understanding (from 70.72 to 79.27), receipt understanding (from 94.02 to 95.24) and document image
 classification (from 93.07 to 94.42).*
 

docs/source/model_doc/longformer.rst

@@ -99,21 +99,41 @@ Longformer specific outputs
 .. autoclass:: transformers.models.longformer.modeling_longformer.LongformerBaseModelOutputWithPooling
     :members: 
 
-.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMultipleChoiceModelOutput
+.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMaskedLMOutput
     :members: 
 
 .. autoclass:: transformers.models.longformer.modeling_longformer.LongformerQuestionAnsweringModelOutput
     :members: 
 
+.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerSequenceClassifierOutput
+    :members: 
+
+.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerMultipleChoiceModelOutput
+    :members: 
+
+.. autoclass:: transformers.models.longformer.modeling_longformer.LongformerTokenClassifierOutput
+    :members: 
+
 .. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutput
     :members: 
 
 .. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerBaseModelOutputWithPooling
     :members: 
 
+.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerMaskedLMOutput
+    :members: 
+
 .. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerQuestionAnsweringModelOutput
     :members: 
 
+.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerSequenceClassifierOutput
+    :members: 
+
+.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerMultipleChoiceModelOutput
+    :members: 
+
+.. autoclass:: transformers.models.longformer.modeling_tf_longformer.TFLongformerTokenClassifierOutput
+    :members: 
 
 LongformerModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -177,3 +197,24 @@ TFLongformerForQuestionAnswering
 .. autoclass:: transformers.TFLongformerForQuestionAnswering
     :members: call
 
+
+TFLongformerForSequenceClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFLongformerForSequenceClassification
+    :members: call
+
+
+TFLongformerForTokenClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFLongformerForTokenClassification
+    :members: call
+
+
+TFLongformerForMultipleChoice
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFLongformerForMultipleChoice
+    :members: call
+

docs/source/model_doc/lxmert.rst

@@ -19,7 +19,7 @@ Encoder Representations from Transformers) framework to learn these vision-and-l
 build a large-scale Transformer model that consists of three encoders: an object relationship encoder, a language
 encoder, and a cross-modality encoder. Next, to endow our model with the capability of connecting vision and language
 semantics, we pre-train the model with large amounts of image-and-sentence pairs, via five diverse representative
-pre-training tasks: masked language modeling, masked object prediction (feature regression and label classification),
+pretraining tasks: masked language modeling, masked object prediction (feature regression and label classification),
 cross-modality matching, and image question answering. These tasks help in learning both intra-modality and
 cross-modality relationships. After fine-tuning from our pretrained parameters, our model achieves the state-of-the-art
 results on two visual question answering datasets (i.e., VQA and GQA). We also show the generalizability of our

docs/source/model_doc/mbart.rst

@@ -13,7 +13,7 @@ The MBart model was presented in `Multilingual Denoising Pre-training for Neural
 Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
 
 According to the abstract, MBART is a sequence-to-sequence denoising auto-encoder pretrained on large-scale monolingual
-corpora in many languages using the BART objective. mBART is one of the first methods for pre-training a complete
+corpora in many languages using the BART objective. mBART is one of the first methods for pretraining a complete
 sequence-to-sequence model by denoising full texts in multiple languages, while previous approaches have focused only
 on the encoder, decoder, or reconstructing parts of the text.
 

docs/source/model_doc/prophetnet.rst

@@ -17,7 +17,7 @@ the next token.
 
 The abstract from the paper is the following:
 
-*In this paper, we present a new sequence-to-sequence pre-training model called ProphetNet, which introduces a novel
+*In this paper, we present a new sequence-to-sequence pretraining model called ProphetNet, which introduces a novel
 self-supervised objective named future n-gram prediction and the proposed n-stream self-attention mechanism. Instead of
 the optimization of one-step ahead prediction in traditional sequence-to-sequence model, the ProphetNet is optimized by
 n-step ahead prediction which predicts the next n tokens simultaneously based on previous context tokens at each time
@@ -25,7 +25,7 @@ step. The future n-gram prediction explicitly encourages the model to plan for t
 overfitting on strong local correlations. We pre-train ProphetNet using a base scale dataset (16GB) and a large scale
 dataset (160GB) respectively. Then we conduct experiments on CNN/DailyMail, Gigaword, and SQuAD 1.1 benchmarks for
 abstractive summarization and question generation tasks. Experimental results show that ProphetNet achieves new
-state-of-the-art results on all these datasets compared to the models using the same scale pre-training corpus.*
+state-of-the-art results on all these datasets compared to the models using the same scale pretraining corpus.*
 
 The Authors' code can be found `here <https://github.com/microsoft/ProphetNet>`__.
 

docs/source/model_doc/t5.rst

@@ -17,7 +17,7 @@ The abstract from the paper is the following:
 task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning
 has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of
 transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a
-text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer
+text-to-text format. Our systematic study compares pretraining objectives, architectures, unlabeled datasets, transfer
 approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration
 with scale and our new "Colossal Clean Crawled Corpus", we achieve state-of-the-art results on many benchmarks covering
 summarization, question answering, text classification, and more. To facilitate future work on transfer learning for

docs/source/model_doc/xlmprophetnet.rst

@@ -19,7 +19,7 @@ just the next token. Its architecture is identical to ProhpetNet, but the model
 
 The abstract from the paper is the following:
 
-*In this paper, we present a new sequence-to-sequence pre-training model called ProphetNet, which introduces a novel
+*In this paper, we present a new sequence-to-sequence pretraining model called ProphetNet, which introduces a novel
 self-supervised objective named future n-gram prediction and the proposed n-stream self-attention mechanism. Instead of
 the optimization of one-step ahead prediction in traditional sequence-to-sequence model, the ProphetNet is optimized by
 n-step ahead prediction which predicts the next n tokens simultaneously based on previous context tokens at each time
@@ -27,7 +27,7 @@ step. The future n-gram prediction explicitly encourages the model to plan for t
 overfitting on strong local correlations. We pre-train ProphetNet using a base scale dataset (16GB) and a large scale
 dataset (160GB) respectively. Then we conduct experiments on CNN/DailyMail, Gigaword, and SQuAD 1.1 benchmarks for
 abstractive summarization and question generation tasks. Experimental results show that ProphetNet achieves new
-state-of-the-art results on all these datasets compared to the models using the same scale pre-training corpus.*
+state-of-the-art results on all these datasets compared to the models using the same scale pretraining corpus.*
 
 The Authors' code can be found `here <https://github.com/microsoft/ProphetNet>`__.
 

docs/source/model_summary.rst

@@ -527,10 +527,10 @@ Pegasus
 <https://arxiv.org/pdf/1912.08777.pdf>`_, Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu on Dec 18, 2019.
 
 Sequence-to-sequence model with the same encoder-decoder model architecture as BART. Pegasus is pre-trained jointly on
-two self-supervised objective functions: Masked Language Modeling (MLM) and a novel summarization specific pre-training
+two self-supervised objective functions: Masked Language Modeling (MLM) and a novel summarization specific pretraining
 objective, called Gap Sentence Generation (GSG).
 
-  * MLM: encoder input tokens are randomely replaced by a mask tokens and have to be predicted by the encoder (like in
+  * MLM: encoder input tokens are randomly replaced by a mask tokens and have to be predicted by the encoder (like in
     BERT)
   * GSG: whole encoder input sentences are replaced by a second mask token and fed to the decoder, but which has a
     causal mask to hide the future words like a regular auto-regressive transformer decoder.
@@ -609,7 +609,7 @@ MT5
 `mT5: A massively multilingual pre-trained text-to-text transformer <https://arxiv.org/abs/2010.11934>`_, Linting Xue
 et al.
 
-The model architecture is same as T5. mT5's pre-training objective includes T5's self-supervised training, but not T5's
+The model architecture is same as T5. mT5's pretraining objective includes T5's self-supervised training, but not T5's
 supervised training. mT5 is trained on 101 languages.
 
 The library provides a version of this model for conditional generation.
@@ -630,8 +630,8 @@ MBart
 `Multilingual Denoising Pre-training for Neural Machine Translation <https://arxiv.org/abs/2001.08210>`_ by Yinhan Liu,
 Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
 
-The model architecture and pre-training objective is same as BART, but MBart is trained on 25 languages and is intended
-for supervised and unsupervised machine translation. MBart is one of the first methods for pre-training a complete
+The model architecture and pretraining objective is same as BART, but MBart is trained on 25 languages and is intended
+for supervised and unsupervised machine translation. MBart is one of the first methods for pretraining a complete
 sequence-to-sequence model by denoising full texts in multiple languages,
 
 The library provides a version of this model for conditional generation.
@@ -658,7 +658,7 @@ ProphetNet
 `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by
 Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.
 
-ProphetNet introduces a novel *sequence-to-sequence* pre-training objective, called *future n-gram prediction*. In
+ProphetNet introduces a novel *sequence-to-sequence* pretraining objective, called *future n-gram prediction*. In
 future n-gram prediction, the model predicts the next n tokens simultaneously based on previous context tokens at each
 time step instead instead of just the single next token. The future n-gram prediction explicitly encourages the model
 to plan for the future tokens and prevent overfitting on strong local correlations. The model architecture is based on
@@ -683,8 +683,8 @@ XLM-ProphetNet
 `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training, <https://arxiv.org/abs/2001.04063>`__ by
 Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.
 
-XLM-ProphetNet's model architecture and pre-training objective is same as ProphetNet, but XLM-ProphetNet was
-pre-trained on the cross-lingual dataset `XGLUE <https://arxiv.org/abs/2004.01401>`__.
+XLM-ProphetNet's model architecture and pretraining objective is same as ProphetNet, but XLM-ProphetNet was pre-trained
+on the cross-lingual dataset `XGLUE <https://arxiv.org/abs/2004.01401>`__.
 
 The library provides a pre-trained version of this model for multi-lingual conditional generation and fine-tuned
 versions for headline generation and question generation, respectively.

docs/source/multilingual.rst

@@ -109,7 +109,7 @@ XLM-RoBERTa
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 XLM-RoBERTa was trained on 2.5TB of newly created clean CommonCrawl data in 100 languages. It provides strong gains
-over previously released multi-lingual models like mBERT or XLM on downstream taks like classification, sequence
+over previously released multi-lingual models like mBERT or XLM on downstream tasks like classification, sequence
 labeling and question answering.
 
 Two XLM-RoBERTa checkpoints can be used for multi-lingual tasks:

docs/source/perplexity.rst

@@ -62,7 +62,7 @@ sliding the context window so that the model has more context when making each p
 This is a closer approximation to the true decomposition of the sequence probability and will typically yield a more
 favorable score. The downside is that it requires a separate forward pass for each token in the corpus. A good
 practical compromise is to employ a strided sliding window, moving the context by larger strides rather than sliding by
-1 token a time. This allows computation to procede much faster while still giving the model a large context to make
+1 token a time. This allows computation to proceed much faster while still giving the model a large context to make
 predictions at each step.
 
 Example: Calculating perplexity with GPT-2 in 🤗 Transformers

docs/source/task_summary.rst

@@ -231,7 +231,9 @@ Here is an example of question answering using a model and a tokenizer. The proc
     ...     input_ids = inputs["input_ids"].tolist()[0]
     ...
     ...     text_tokens = tokenizer.convert_ids_to_tokens(input_ids)
-    ...     answer_start_scores, answer_end_scores = model(**inputs)
+    ...     outputs = model(**inputs)
+    ...     answer_start_scores = outputs.start_logits
+    ...     answer_end_scores = outputs.end_logits
     ...
     ...     answer_start = torch.argmax(
     ...         answer_start_scores
@@ -273,7 +275,9 @@ Here is an example of question answering using a model and a tokenizer. The proc
     ...     input_ids = inputs["input_ids"].numpy()[0]
     ...
     ...     text_tokens = tokenizer.convert_ids_to_tokens(input_ids)
-    ...     answer_start_scores, answer_end_scores = model(inputs)
+    ...     outputs = model(inputs)
+    ...     answer_start_scores = outputs.start_logits
+    ...     answer_end_scores = outputs.end_logits
     ...
     ...     answer_start = tf.argmax(
     ...         answer_start_scores, axis=1
@@ -301,7 +305,7 @@ Language modeling is the task of fitting a model to a corpus, which can be domai
 transformer-based models are trained using a variant of language modeling, e.g. BERT with masked language modeling,
 GPT-2 with causal language modeling.
 
-Language modeling can be useful outside of pre-training as well, for example to shift the model distribution to be
+Language modeling can be useful outside of pretraining as well, for example to shift the model distribution to be
 domain-specific: using a language model trained over a very large corpus, and then fine-tuning it to a news dataset or
 on scientific papers e.g. `LysandreJik/arxiv-nlp <https://huggingface.co/lysandre/arxiv-nlp>`__.
 

docs/source/testing.rst

@@ -1055,7 +1055,7 @@ If you need to validate the output of a logger, you can use :obj:`CaptureLogger`
 
     msg = "Testing 1, 2, 3"
     logging.set_verbosity_info()
-    logger = logging.get_logger("transformers.tokenization_bart")
+    logger = logging.get_logger("transformers.models.bart.tokenization_bart")
     with CaptureLogger(logger) as cl:
         logger.info(msg)
     assert cl.out, msg+"\n"

examples/benchmarking/plot_csv_file.py

@@ -25,7 +25,7 @@ class PlotArguments:
     )
     plot_along_batch: bool = field(
         default=False,
-        metadata={"help": "Whether to plot along batch size or sequence lengh. Defaults to sequence length."},
+        metadata={"help": "Whether to plot along batch size or sequence length. Defaults to sequence length."},
     )
     is_time: bool = field(
         default=False,

examples/contrib/run_chinese_ref.py

@@ -3,7 +3,7 @@ import json
 from typing import List
 
 from ltp import LTP
-from transformers.tokenization_bert import BertTokenizer
+from transformers import BertTokenizer
 
 
 def _is_chinese_char(cp):

examples/distillation/README.md

@@ -17,7 +17,7 @@ This folder contains the original code used to train Distil* as well as examples
 
 ## What is Distil*
 
-Distil* is a class of compressed models that started with DistilBERT. DistilBERT stands for Distillated-BERT. DistilBERT is a small, fast, cheap and light Transformer model based on Bert architecture. It has 40% less parameters than `bert-base-uncased`, runs 60% faster while preserving 97% of BERT's performances as measured on the GLUE language understanding benchmark. DistilBERT is trained using knowledge distillation, a technique to compress a large model called the teacher into a smaller model called the student. By distillating Bert, we obtain a smaller Transformer model that bears a lot of similarities with the original BERT model while being lighter, smaller and faster to run. DistilBERT is thus an interesting option to put large-scaled trained Transformer model into production.
+Distil* is a class of compressed models that started with DistilBERT. DistilBERT stands for Distilled-BERT. DistilBERT is a small, fast, cheap and light Transformer model based on Bert architecture. It has 40% less parameters than `bert-base-uncased`, runs 60% faster while preserving 97% of BERT's performances as measured on the GLUE language understanding benchmark. DistilBERT is trained using knowledge distillation, a technique to compress a large model called the teacher into a smaller model called the student. By distillating Bert, we obtain a smaller Transformer model that bears a lot of similarities with the original BERT model while being lighter, smaller and faster to run. DistilBERT is thus an interesting option to put large-scaled trained Transformer model into production.
 
 We have applied the same method to other Transformer architectures and released the weights:
 - GPT2: on the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test set of 16.3 compared to 21.1 for **DistilGPT2** (after fine-tuning on the train set).
@@ -57,7 +57,7 @@ Here are the results on the *test* sets for 6 of the languages available in XNLI
 
 This part of the library has only be tested with Python3.6+. There are few specific dependencies to install before launching a distillation, you can install them with the command `pip install -r requirements.txt`.
 
-**Important note:** The training scripts have been updated to support PyTorch v1.2.0 (there are breakings changes compared to v1.1.0).
+**Important note:** The training scripts have been updated to support PyTorch v1.2.0 (there are breaking changes compared to v1.1.0).
 
 
 ## How to use DistilBERT
@@ -111,7 +111,7 @@ python scripts/binarized_data.py \
     --dump_file data/binarized_text
 ```
 
-Our implementation of masked language modeling loss follows [XLM](https://github.com/facebookresearch/XLM)'s one and smoothes the probability of masking with a factor that put more emphasis on rare words. Thus we count the occurrences of each tokens in the data:
+Our implementation of masked language modeling loss follows [XLM](https://github.com/facebookresearch/XLM)'s one and smooths the probability of masking with a factor that put more emphasis on rare words. Thus we count the occurrences of each tokens in the data:
 
 ```bash
 python scripts/token_counts.py \
@@ -173,7 +173,7 @@ python -m torch.distributed.launch \
         --token_counts data/token_counts.bert-base-uncased.pickle
 ```
 
-**Tips:** Starting distillated training with good initialization of the model weights is crucial to reach decent performance. In our experiments, we initialized our model from a few layers of the teacher (Bert) itself! Please refer to `scripts/extract.py` and `scripts/extract_distilbert.py` to create a valid initialization checkpoint and use `--student_pretrained_weights` argument to use this initialization for the distilled training!
+**Tips:** Starting distilled training with good initialization of the model weights is crucial to reach decent performance. In our experiments, we initialized our model from a few layers of the teacher (Bert) itself! Please refer to `scripts/extract.py` and `scripts/extract_distilbert.py` to create a valid initialization checkpoint and use `--student_pretrained_weights` argument to use this initialization for the distilled training!
 
 Happy distillation!
 

examples/distillation/distiller.py

@@ -188,7 +188,7 @@ class Distiller:
 
     def prepare_batch_mlm(self, batch):
         """
-        Prepare the batch: from the token_ids and the lenghts, compute the attention mask and the masked label for MLM.
+        Prepare the batch: from the token_ids and the lengths, compute the attention mask and the masked label for MLM.
 
         Input:
         ------
@@ -200,7 +200,7 @@ class Distiller:
         -------
             token_ids: `torch.tensor(bs, seq_length)` - The token ids after the modifications for MLM.
             attn_mask: `torch.tensor(bs, seq_length)` - The attention mask for the self-attention.
-            mlm_labels: `torch.tensor(bs, seq_length)` - The masked languge modeling labels. There is a -100 where there is nothing to predict.
+            mlm_labels: `torch.tensor(bs, seq_length)` - The masked language modeling labels. There is a -100 where there is nothing to predict.
         """
         token_ids, lengths = batch
         token_ids, lengths = self.round_batch(x=token_ids, lengths=lengths)
@@ -253,7 +253,7 @@ class Distiller:
 
     def prepare_batch_clm(self, batch):
         """
-        Prepare the batch: from the token_ids and the lenghts, compute the attention mask and the labels for CLM.
+        Prepare the batch: from the token_ids and the lengths, compute the attention mask and the labels for CLM.
 
         Input:
         ------

examples/distillation/scripts/extract_distilbert.py

@@ -86,7 +86,7 @@ if __name__ == "__main__":
             compressed_sd[f"vocab_layer_norm.{w}"] = state_dict[f"cls.predictions.transform.LayerNorm.{w}"]
 
     print(f"N layers selected for distillation: {std_idx}")
-    print(f"Number of params transfered for distillation: {len(compressed_sd.keys())}")
+    print(f"Number of params transferred for distillation: {len(compressed_sd.keys())}")
 
-    print(f"Save transfered checkpoint to {args.dump_checkpoint}.")
+    print(f"Save transferred checkpoint to {args.dump_checkpoint}.")
     torch.save(compressed_sd, args.dump_checkpoint)

examples/language-modeling/README.md

@@ -90,7 +90,7 @@ selected tokens (which may be part of words), they mask randomly selected words
 to that word). This technique has been refined for Chinese in [this paper](https://arxiv.org/abs/1906.08101).
 
 To fine-tune a model using whole word masking, use the following script:
-
+```bash
 python run_mlm_wwm.py \
     --model_name_or_path roberta-base \
     --dataset_name wikitext \
@@ -164,7 +164,7 @@ context length for permutation language modeling.
 The `--max_span_length` flag may also be used to limit the length of a span of masked tokens used 
 for permutation language modeling.
 
-Here is how to fine-tun XLNet on wikitext-2:
+Here is how to fine-tune XLNet on wikitext-2:
 
 ```bash
 python run_plm.py \

examples/longform-qa/eli5_app.py

@@ -107,7 +107,12 @@ def make_support(question, source="wiki40b", method="dense", n_results=10):
     return question_doc, support_list
 
 
-@st.cache(hash_funcs={torch.Tensor: (lambda _: None), transformers.tokenization_bart.BartTokenizer: (lambda _: None)})
+@st.cache(
+    hash_funcs={
+        torch.Tensor: (lambda _: None),
+        transformers.models.bart.tokenization_bart.BartTokenizer: (lambda _: None),
+    }
+)
 def answer_question(
     question_doc, s2s_model, s2s_tokenizer, min_len=64, max_len=256, sampling=False, n_beams=2, top_p=0.95, temp=0.8
 ):

examples/movement-pruning/README.md

@@ -21,7 +21,7 @@ You can also have a look at this fun *Explain Like I'm Five* introductory [slide
 
 One promise of extreme pruning is to obtain extremely small models that can be easily sent (and stored) on edge devices. By setting weights to 0., we reduce the amount of information we need to store, and thus decreasing the memory size. We are able to obtain extremely sparse fine-pruned models with movement pruning: ~95% of the dense performance with ~5% of total remaining weights in the BERT encoder.
 
-In [this notebook](https://github.com/huggingface/transformers/blob/master/examples/movement-pruning/Saving_PruneBERT.ipynb), we showcase how we can leverage standard tools that exist out-of-the-box to efficiently store an extremely sparse question answering model (only 6% of total remaining weights in the encoder). We are able to reduce the memory size of the encoder **from the 340MB (the orignal dense BERT) to 11MB**, without any additional training of the model (every operation is performed *post fine-pruning*). It is sufficiently small to store it on a [91' floppy disk](https://en.wikipedia.org/wiki/Floptical) 📎!
+In [this notebook](https://github.com/huggingface/transformers/blob/master/examples/movement-pruning/Saving_PruneBERT.ipynb), we showcase how we can leverage standard tools that exist out-of-the-box to efficiently store an extremely sparse question answering model (only 6% of total remaining weights in the encoder). We are able to reduce the memory size of the encoder **from the 340MB (the original dense BERT) to 11MB**, without any additional training of the model (every operation is performed *post fine-pruning*). It is sufficiently small to store it on a [91' floppy disk](https://en.wikipedia.org/wiki/Floptical) 📎!
 
 While movement pruning does not directly optimize for memory footprint (but rather the number of non-null weights), we hypothetize that further memory compression ratios can be achieved with specific quantization aware trainings (see for instance [Q8BERT](https://arxiv.org/abs/1910.06188), [And the Bit Goes Down](https://arxiv.org/abs/1907.05686) or [Quant-Noise](https://arxiv.org/abs/2004.07320)).
 

examples/movement-pruning/emmental/modules/binarizer.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-Binarizers take a (real value) matrice as input and produce a binary (values in {0,1}) mask of the same shape.
+Binarizers take a (real value) matrix as input and produce a binary (values in {0,1}) mask of the same shape.
 """
 
 import torch

examples/rag/use_own_knowledge_dataset.py

@@ -47,7 +47,7 @@ def embed(documents: dict, ctx_encoder: DPRContextEncoder, ctx_tokenizer: DPRCon
     input_ids = ctx_tokenizer(
         documents["title"], documents["text"], truncation=True, padding="longest", return_tensors="pt"
     )["input_ids"]
-    embeddings = ctx_encoder(input_ids.to(device=device)).pooler_output
+    embeddings = ctx_encoder(input_ids.to(device=device), return_dict=True).pooler_output
     return {"embeddings": embeddings.detach().cpu().numpy()}
 
 

examples/seq2seq/builtin_trainer/finetune.sh

@@ -3,7 +3,8 @@
 python finetune_trainer.py \
     --learning_rate=3e-5 \
     --fp16 \
-    --do_train --do_eval --do_predict --evaluate_during_training \
+    --do_train --do_eval --do_predict \
+    --evaluation_strategy steps \
     --predict_with_generate \
     --n_val 1000 \
     "$@"

examples/seq2seq/builtin_trainer/finetune_tpu.sh

@@ -5,7 +5,8 @@ export TPU_NUM_CORES=8
 python xla_spawn.py --num_cores $TPU_NUM_CORES \
     finetune_trainer.py \
     --learning_rate=3e-5 \
-    --do_train --do_eval --evaluate_during_training \
+    --do_train --do_eval \
+    --evaluation_strategy steps \
     --prediction_loss_only \
     --n_val 1000 \
     "$@"

examples/seq2seq/builtin_trainer/train_distil_marian_enro.sh

@@ -16,7 +16,8 @@ python finetune_trainer.py \
     --num_train_epochs=6 \
     --save_steps 3000 --eval_steps 3000 \
     --max_source_length $MAX_LEN --max_target_length $MAX_LEN --val_max_target_length $MAX_LEN --test_max_target_length $MAX_LEN \
-    --do_train --do_eval --do_predict --evaluate_during_training\
+    --do_train --do_eval --do_predict \
+    --evaluation_strategy steps \
     --predict_with_generate --logging_first_step \
     --task translation --label_smoothing 0.1 \
     "$@"

examples/seq2seq/builtin_trainer/train_distil_marian_enro_tpu.sh

@@ -17,7 +17,8 @@ python xla_spawn.py --num_cores $TPU_NUM_CORES \
     --save_steps 500 --eval_steps 500 \
     --logging_first_step --logging_steps 200 \
     --max_source_length $MAX_LEN --max_target_length $MAX_LEN --val_max_target_length $MAX_LEN --test_max_target_length $MAX_LEN \
-    --do_train --do_eval --evaluate_during_training \
+    --do_train --do_eval \
+    --evaluation_strategy steps \
     --prediction_loss_only \
     --task translation --label_smoothing 0.1 \
     "$@"

examples/seq2seq/builtin_trainer/train_distilbart_cnn.sh

@@ -19,6 +19,7 @@ python finetune_trainer.py \
     --save_steps 3000 --eval_steps 3000 \
     --logging_first_step \
     --max_target_length 56 --val_max_target_length $MAX_TGT_LEN --test_max_target_length $MAX_TGT_LEN \
-    --do_train --do_eval --do_predict --evaluate_during_training \
+    --do_train --do_eval --do_predict \
+    --evaluation_strategy steps \
     --predict_with_generate --sortish_sampler \
     "$@"

examples/seq2seq/builtin_trainer/train_mbart_cc25_enro.sh

@@ -15,7 +15,8 @@ python finetune_trainer.py \
     --sortish_sampler \
     --num_train_epochs 6 \
     --save_steps 25000 --eval_steps 25000 --logging_steps 1000 \
-    --do_train --do_eval --do_predict --evaluate_during_training \
-    --predict_with_generate --logging_first_step 
+    --do_train --do_eval --do_predict \
+    --evaluation_strategy steps \
+    --predict_with_generate --logging_first_step \
     --task translation \
     "$@"

examples/seq2seq/seq2seq_trainer.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import torch
 from torch import nn
@@ -153,7 +153,11 @@ class Seq2SeqTrainer(Trainer):
         return loss
 
     def prediction_step(
-        self, model: nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]], prediction_loss_only: bool
+        self,
+        model: nn.Module,
+        inputs: Dict[str, Union[torch.Tensor, Any]],
+        prediction_loss_only: bool,
+        ignore_keys: Optional[List[str]] = None,
     ) -> Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
         """
         Perform an evaluation step on :obj:`model` using obj:`inputs`.

model_cards/Geotrend/bert-base-15lang-cased/README.md

@@ -0,0 +1,49 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-15lang-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+The measurements below have been computed on a [Google Cloud n1-standard-1 machine (1 vCPU, 3.75 GB)](https://cloud.google.com/compute/docs/machine-types\#n1_machine_type):
+
+|             Model               | Num parameters |   Size   |  Memory  | Loading time |
+| ------------------------------- | -------------- | -------- | -------- | ------------ |
+| bert-base-multilingual-cased    |   178 million  |  714 MB  | 1400 MB  |    4.2 sec   |
+| Geotrend/bert-base-15lang-cased |   141 million  |  564 MB  | 1098 MB  |    3.1 sec   |
+
+Handled languages: en, fr, es, de, zh, ar, ru, vi, el, bg, th, tr, hi, ur and sw.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-15lang-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-15lang-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-ar-cased/README.md

@@ -0,0 +1,41 @@
+---
+language: ar
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-ar-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-ar-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-ar-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-bg-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: bg
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-bg-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-bg-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-bg-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-de-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: de
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-de-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-de-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-de-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-el-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: el
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-el-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-el-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-el-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-ar-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-ar-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-ar-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-ar-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-bg-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-bg-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-bg-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-bg-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: en
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-de-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-de-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-de-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-de-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-el-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-el-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-el-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-el-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-es-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-es-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-es-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-es-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-fr-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-fr-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-fr-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-fr-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-hi-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-hi-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-hi-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-hi-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-ru-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-ru-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-ru-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-ru-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-sw-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-sw-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-sw-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-sw-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-th-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-th-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-th-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-th-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-tr-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-tr-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-tr-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-tr-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-ur-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-ur-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-ur-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-ur-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-vi-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-vi-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-vi-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-vi-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-en-zh-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: multilingual
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-en-zh-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-zh-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-en-zh-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-es-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: es
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-es-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-es-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-es-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-fr-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: fr
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-fr-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-fr-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-fr-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-hi-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: hi
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-hi-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-hi-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-hi-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-ru-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: ru
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-ru-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-ru-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-ru-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-sw-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: sw
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-sw-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-sw-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-sw-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-th-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: th
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-th-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-th-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-th-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-tr-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: tr
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-tr-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-tr-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-tr-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-ur-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: ur
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-ur-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-ur-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-ur-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-vi-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: vi
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-vi-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-vi-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-vi-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/Geotrend/bert-base-zh-cased/README.md

@@ -0,0 +1,40 @@
+---
+language: zh
+
+datasets: wikipedia
+
+license: apache-2.0
+---
+
+# bert-base-zh-cased
+
+We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages.
+
+Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy.
+
+For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf).
+
+## How to use
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-zh-cased")
+model = AutoModel.from_pretrained("Geotrend/bert-base-zh-cased")
+
+```
+
+### How to cite
+
+```bibtex
+@inproceedings{smallermbert,
+  title={Load What You Need: Smaller Versions of Mutlilingual BERT},
+  author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire},
+  booktitle={SustaiNLP / EMNLP},
+  year={2020}
+}
+```
+
+## Contact 
+
+Please contact amine@geotrend.fr for any question, feedback or request.

model_cards/KB/albert-base-swedish-cased-alpha/README.md

@@ -4,7 +4,7 @@ language: sv
 
 # Swedish BERT Models
 
-The National Library of Sweden / KBLab releases three pretrained language models based on BERT and ALBERT. The models are trained on aproximately 15-20GB of text (200M sentences, 3000M tokens) from various sources (books, news, government publications, swedish wikipedia and internet forums) aiming to provide a representative BERT model for Swedish text. A more complete description will be published later on.
+The National Library of Sweden / KBLab releases three pretrained language models based on BERT and ALBERT. The models are trained on approximately 15-20GB of text (200M sentences, 3000M tokens) from various sources (books, news, government publications, swedish wikipedia and internet forums) aiming to provide a representative BERT model for Swedish text. A more complete description will be published later on.
 
 The following three models are currently available:
 
@@ -86,7 +86,7 @@ for token in nlp(text):
 print(l)
 ```
 
-Which should result in the following (though less cleanly formated):
+Which should result in the following (though less cleanly formatted):
 
 ```python
 [ { 'word': 'Engelbert',     'score': 0.99..., 'entity': 'PRS'},
@@ -104,7 +104,7 @@ Which should result in the following (though less cleanly formated):
 
 ### ALBERT base
 
-The easisest way to do this is, again, using Huggingface Transformers:
+The easiest way to do this is, again, using Huggingface Transformers:
 
 ```python
 from transformers import AutoModel,AutoTokenizer

model_cards/KB/bert-base-swedish-cased-ner/README.md

@@ -4,7 +4,7 @@ language: sv
 
 # Swedish BERT Models
 
-The National Library of Sweden / KBLab releases three pretrained language models based on BERT and ALBERT. The models are trained on aproximately 15-20GB of text (200M sentences, 3000M tokens) from various sources (books, news, government publications, swedish wikipedia and internet forums) aiming to provide a representative BERT model for Swedish text. A more complete description will be published later on.
+The National Library of Sweden / KBLab releases three pretrained language models based on BERT and ALBERT. The models are trained on approximately 15-20GB of text (200M sentences, 3000M tokens) from various sources (books, news, government publications, swedish wikipedia and internet forums) aiming to provide a representative BERT model for Swedish text. A more complete description will be published later on.
 
 The following three models are currently available:
 
@@ -86,7 +86,7 @@ for token in nlp(text):
 print(l)
 ```
 
-Which should result in the following (though less cleanly formated):
+Which should result in the following (though less cleanly formatted):
 
 ```python
 [ { 'word': 'Engelbert',     'score': 0.99..., 'entity': 'PRS'},
@@ -104,7 +104,7 @@ Which should result in the following (though less cleanly formated):
 
 ### ALBERT base
 
-The easisest way to do this is, again, using Huggingface Transformers:
+The easiest way to do this is, again, using Huggingface Transformers:
 
 ```python
 from transformers import AutoModel,AutoTokenizer

model_cards/elgeish/cs224n-squad2.0-albert-base-v2/README.md

@@ -4,7 +4,7 @@ tags:
 ---
 
 ## CS224n SQuAD2.0 Project Dataset
-The goal of this model is to save CS224n students GPU time when establising
+The goal of this model is to save CS224n students GPU time when establishing
 baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
 The training set used to fine-tune this model is the same as
 the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,

model_cards/elgeish/cs224n-squad2.0-albert-xxlarge-v1/README.md

@@ -4,7 +4,7 @@ tags:
 ---
 
 ## CS224n SQuAD2.0 Project Dataset
-The goal of this model is to save CS224n students GPU time when establising
+The goal of this model is to save CS224n students GPU time when establishing
 baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
 The training set used to fine-tune this model is the same as
 the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,

model_cards/elgeish/cs224n-squad2.0-distilbert-base-uncased/README.md

@@ -1,5 +1,5 @@
 ## CS224n SQuAD2.0 Project Dataset
-The goal of this model is to save CS224n students GPU time when establising
+The goal of this model is to save CS224n students GPU time when establishing
 baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
 The training set used to fine-tune this model is the same as
 the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,

model_cards/elgeish/cs224n-squad2.0-roberta-base/README.md

@@ -1,5 +1,5 @@
 ## CS224n SQuAD2.0 Project Dataset
-The goal of this model is to save CS224n students GPU time when establising
+The goal of this model is to save CS224n students GPU time when establishing
 baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
 The training set used to fine-tune this model is the same as
 the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,

model_cards/julien-c/dummy-unknown/README.md

@@ -9,8 +9,7 @@ tags:
 ```python
 import json
 import os
-from transformers.configuration_roberta import RobertaConfig
-from transformers import RobertaForMaskedLM, TFRobertaForMaskedLM
+from transformers import RobertaConfig, RobertaForMaskedLM, TFRobertaForMaskedLM
 
 DIRNAME = "./dummy-unknown"
 

model_cards/mrm8488/t5-base-finetuned-e2m-intent/README.md

@@ -19,7 +19,7 @@ Transfer learning, where a model is first pre-trained on a data-rich task before
 
 ## Details of the downstream task (Intent Prediction) - Dataset 📚 
 
-Dataset ID: ```event2Mind``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
+Dataset ID: ```event2Mind``` from  [Huggingface/NLP](https://github.com/huggingface/nlp)
 
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |

model_cards/mrm8488/t5-base-finetuned-question-generation-ap/README.md

@@ -19,7 +19,7 @@ Transfer learning, where a model is first pre-trained on a data-rich task before
 
 ## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓
 
-Dataset ID: ```squad``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
+Dataset ID: ```squad``` from  [Huggingface/NLP](https://github.com/huggingface/nlp)
 
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |

model_cards/mrm8488/t5-base-finetuned-squadv2/README.md

@@ -19,7 +19,7 @@ Transfer learning, where a model is first pre-trained on a data-rich task before
 
 ## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓
 
-Dataset ID: ```squad_v2``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
+Dataset ID: ```squad_v2``` from  [Huggingface/NLP](https://github.com/huggingface/nlp)
 
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |

model_cards/mrm8488/t5-base-finetuned-wikiSQL-sql-to-en/README.md

@@ -19,7 +19,7 @@ Transfer learning, where a model is first pre-trained on a data-rich task before
 
 ## Details of the Dataset 📚 
 
-Dataset ID: ```wikisql``` from  [HugginFace/NLP](https://huggingface.co/nlp/viewer/?dataset=wikisql)
+Dataset ID: ```wikisql``` from  [Huggingface/NLP](https://huggingface.co/nlp/viewer/?dataset=wikisql)
 
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |

model_cards/mrm8488/t5-base-finetuned-wikiSQL/README.md

@@ -19,7 +19,7 @@ Transfer learning, where a model is first pre-trained on a data-rich task before
 
 ## Details of the Dataset 📚 
 
-Dataset ID: ```wikisql``` from  [HugginFace/NLP](https://huggingface.co/nlp/viewer/?dataset=wikisql)
+Dataset ID: ```wikisql``` from  [Huggingface/NLP](https://huggingface.co/nlp/viewer/?dataset=wikisql)
 
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |

model_cards/mrm8488/t5-small-finetuned-quora-for-paraphrasing/README.md

@@ -19,7 +19,7 @@ Transfer learning, where a model is first pre-trained on a data-rich task before
 
 ## Details of the downstream task (Question Paraphrasing) - Dataset 📚❓↔️❓
 
-Dataset ID: ```quora``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
+Dataset ID: ```quora``` from  [Huggingface/NLP](https://github.com/huggingface/nlp)
 
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |

model_cards/mrm8488/t5-small-finetuned-squadv1/README.md

@@ -19,7 +19,7 @@ Transfer learning, where a model is first pre-trained on a data-rich task before
 
 ## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓
 
-Dataset ID: ```squad``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
+Dataset ID: ```squad``` from  [Huggingface/NLP](https://github.com/huggingface/nlp)
 
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |

model_cards/mrm8488/t5-small-finetuned-squadv2/README.md

@@ -19,7 +19,7 @@ Transfer learning, where a model is first pre-trained on a data-rich task before
 
 ## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓
 
-Dataset ID: ```squad_v2``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
+Dataset ID: ```squad_v2``` from  [Huggingface/NLP](https://github.com/huggingface/nlp)
 
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |

model_cards/mrm8488/t5-small-finetuned-wikiSQL/README.md

@@ -19,7 +19,7 @@ Transfer learning, where a model is first pre-trained on a data-rich task before
 
 ## Details of the Dataset 📚 
 
-Dataset ID: ```wikisql``` from  [HugginFace/NLP](https://huggingface.co/nlp/viewer/?dataset=wikisql)
+Dataset ID: ```wikisql``` from  [Huggingface/NLP](https://huggingface.co/nlp/viewer/?dataset=wikisql)
 
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |

scripts/fsmt/fsmt-make-super-tiny-model.py

@@ -16,7 +16,7 @@ import json
 import tempfile
 
 from transformers import FSMTTokenizer, FSMTConfig, FSMTForConditionalGeneration
-from transformers.tokenization_fsmt import VOCAB_FILES_NAMES
+from transformers.models.fsmt.tokenization_fsmt import VOCAB_FILES_NAMES
 
 mname_tiny = "tiny-wmt19-en-ru"
 

setup.py

@@ -119,7 +119,7 @@ extras["dev"] = extras["all"] + extras["testing"] + extras["quality"] + extras["
 
 setup(
     name="transformers",
-    version="4.0.0-dev",
+    version="4.0.0",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Sylvain Gugger, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",

src/transformers/__init__.py

@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-__version__ = "4.0.0-dev"
+__version__ = "4.0.0"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -98,6 +98,7 @@ from .models.albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
 from .models.auto import (
     ALL_PRETRAINED_CONFIG_ARCHIVE_MAP,
     CONFIG_MAPPING,
+    MODEL_NAMES_MAPPING,
     TOKENIZER_MAPPING,
     AutoConfig,
     AutoTokenizer,
@@ -766,7 +767,10 @@ if is_tf_available():
     from .models.longformer import (
         TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFLongformerForMaskedLM,
+        TFLongformerForMultipleChoice,
         TFLongformerForQuestionAnswering,
+        TFLongformerForSequenceClassification,
+        TFLongformerForTokenClassification,
         TFLongformerModel,
         TFLongformerSelfAttention,
     )
@@ -873,6 +877,7 @@ else:
 
 
 if is_flax_available():
+    from .models.auto import FLAX_MODEL_MAPPING, FlaxAutoModel
     from .models.bert import FlaxBertModel
     from .models.roberta import FlaxRobertaModel
 else:

src/transformers/configuration_utils.py

@@ -43,6 +43,8 @@ class PretrainedConfig(object):
         - **is_composition** (:obj:`bool`): Whether the config class is composed of multiple sub-configs. In this case
           the config has to be initialized from two or more configs of type :class:`~transformers.PretrainedConfig`
           like: :class:`~transformers.EncoderDecoderConfig` or :class:`~RagConfig`.
+        - **keys_to_ignore_at_inference** (:obj:`List[str]`): A list of keys to ignore by default when looking at
+          dictionary outputs of the model during inference.
 
     Args:
         name_or_path (:obj:`str`, `optional`, defaults to :obj:`""`):
@@ -53,8 +55,6 @@ class PretrainedConfig(object):
             Whether or not the model should return all hidden-states.
         output_attentions (:obj:`bool`, `optional`, defaults to :obj:`False`):
             Whether or not the model should returns all attentions.
-        use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`):
-            Whether or not the model should return the last key/values attentions (not used by all models).
         return_dict (:obj:`bool`, `optional`, defaults to :obj:`True`):
             Whether or not the model should return a :class:`~transformers.file_utils.ModelOutput` instead of a plain
             tuple.
@@ -166,7 +166,6 @@ class PretrainedConfig(object):
         self.return_dict = kwargs.pop("return_dict", True)
         self.output_hidden_states = kwargs.pop("output_hidden_states", False)
         self.output_attentions = kwargs.pop("output_attentions", False)
-        self.use_cache = kwargs.pop("use_cache", True)  # Not used by all models
         self.torchscript = kwargs.pop("torchscript", False)  # Only used by PyTorch models
         self.use_bfloat16 = kwargs.pop("use_bfloat16", False)
         self.pruned_heads = kwargs.pop("pruned_heads", {})

src/transformers/data/datasets/language_modeling.py

@@ -229,7 +229,7 @@ class LineByLineWithSOPTextDataset(Dataset):
         # to `block_size` anyways, so short sequences are generally wasted
         # computation. However, we *sometimes*
         # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter
-        # sequences to minimize the mismatch between pre-training and fine-tuning.
+        # sequences to minimize the mismatch between pretraining and fine-tuning.
         # The `target_seq_length` is just a rough target however, whereas
         # `block_size` is a hard limit.
         target_seq_length = max_num_tokens
@@ -425,7 +425,7 @@ class TextDatasetForNextSentencePrediction(Dataset):
         # to `block_size` anyways, so short sequences are generally wasted
         # computation. However, we *sometimes*
         # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter
-        # sequences to minimize the mismatch between pre-training and fine-tuning.
+        # sequences to minimize the mismatch between pretraining and fine-tuning.
         # The `target_seq_length` is just a rough target however, whereas
         # `block_size` is a hard limit.
         target_seq_length = max_num_tokens

src/transformers/file_utils.py

@@ -203,8 +203,28 @@ except ImportError:
     _tokenizers_available = False
 
 
-default_cache_path = os.path.join(torch_cache_home, "transformers")
+old_default_cache_path = os.path.join(torch_cache_home, "transformers")
+# New default cache, shared with the Datasets library
+hf_cache_home = os.path.expanduser(
+    os.getenv("HF_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "huggingface"))
+)
+default_cache_path = os.path.join(hf_cache_home, "transformers")
 
+# Onetime move from the old location to the new one if no ENV variable has been set.
+if (
+    os.path.isdir(old_default_cache_path)
+    and "PYTORCH_PRETRAINED_BERT_CACHE" not in os.environ
+    and "PYTORCH_TRANSFORMERS_CACHE" not in os.environ
+    and "TRANSFORMERS_CACHE" not in os.environ
+):
+    logger.warn(
+        "In Transformers v4.0.0, the default path to cache downloaded models changed from "
+        "'~/.cache/torch/transformers' to '~/.cache/huggingface/transformers'. Since you don't seem to have overridden "
+        "and '~/.cache/torch/transformers' is a directory that exists, we're moving it to "
+        "'~/.cache/huggingface/transformers' to avoid redownloading models you have already in the cache. You should "
+        "only see this message once."
+    )
+    shutil.move(old_default_cache_path, default_cache_path)
 
 PYTORCH_PRETRAINED_BERT_CACHE = os.getenv("PYTORCH_PRETRAINED_BERT_CACHE", default_cache_path)
 PYTORCH_TRANSFORMERS_CACHE = os.getenv("PYTORCH_TRANSFORMERS_CACHE", PYTORCH_PRETRAINED_BERT_CACHE)

src/transformers/generation_tf_utils.py

@@ -38,6 +38,7 @@ class TFGenerationMixin:
 
     def _use_cache(self, outputs, use_cache):
         """During generation, decide whether to pass the `past` variable to the next forward pass."""
+        use_cache = getattr(self.config, "use_cache", False)
         if len(outputs) <= 1 or use_cache is False:
             return False
         if hasattr(self.config, "mem_len") and self.config.mem_len == 0:
@@ -194,7 +195,6 @@ class TFGenerationMixin:
         min_length = min_length if min_length is not None else self.config.min_length
         do_sample = do_sample if do_sample is not None else self.config.do_sample
         early_stopping = early_stopping if early_stopping is not None else self.config.early_stopping
-        use_cache = use_cache if use_cache is not None else self.config.use_cache
         num_beams = num_beams if num_beams is not None else self.config.num_beams
         temperature = temperature if temperature is not None else self.config.temperature
         top_k = top_k if top_k is not None else self.config.top_k
@@ -224,7 +224,6 @@ class TFGenerationMixin:
         assert isinstance(min_length, int) and min_length >= 0, "`min_length` should be a positive integer."
         assert isinstance(do_sample, bool), "`do_sample` should be a boolean."
         assert isinstance(early_stopping, bool), "`early_stopping` should be a boolean."
-        assert isinstance(use_cache, bool), "`use_cache` should be a boolean."
         assert isinstance(num_beams, int) and num_beams > 0, "`num_beams` should be a strictly positive integer."
         assert temperature > 0, "`temperature` should be strictly positive."
         assert isinstance(top_k, int) and top_k >= 0, "`top_k` should be a positive integer."

src/transformers/generation_utils.py

@@ -462,7 +462,6 @@ class GenerationMixin:
         pad_token_id = pad_token_id if pad_token_id is not None else self.config.pad_token_id
         bos_token_id = bos_token_id if bos_token_id is not None else self.config.bos_token_id
         eos_token_id = eos_token_id if eos_token_id is not None else self.config.eos_token_id
-        use_cache = use_cache if use_cache is not None else self.config.use_cache
 
         if input_ids is None:
             # init `input_ids` with bos_token_id