Release: v3.0.2

* BertTokenizerFast - Do not specify strip_accents by default

* Bump tokenizers to new version

* Add test for AddedToken serialization

.circleci/config.yml

@@ -12,9 +12,11 @@ jobs:
             - checkout
             - run: sudo pip install .[sklearn,tf-cpu,torch,testing]
             - run: sudo pip install codecov pytest-cov
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
+            - run: python -m pytest -n 8 --dist=loadfile -s ./tests/ --cov  | tee output.txt
             - run: codecov
-
+            - store_artifacts:
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
     run_tests_torch:
         working_directory: ~/transformers
         docker:
@@ -26,9 +28,11 @@ jobs:
         steps:
             - checkout
             - run: sudo pip install .[sklearn,torch,testing]
-            - run: sudo pip install codecov pytest-cov
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
-            - run: codecov
+            - run: python -m pytest -n 8 --dist=loadfile -s ./tests/ | tee output.txt
+            - store_artifacts:
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
+
     run_tests_tf:
         working_directory: ~/transformers
         docker:
@@ -40,9 +44,10 @@ jobs:
         steps:
             - checkout
             - run: sudo pip install .[sklearn,tf-cpu,testing]
-            - run: sudo pip install codecov pytest-cov
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
-            - run: codecov
+            - run: python -m pytest -n 8 --dist=loadfile -s ./tests/ | tee output.txt
+            - store_artifacts:
+               path: ~/transformers/output.txt
+               destination: test_output.txt
     run_tests_custom_tokenizers:
         working_directory: ~/transformers
         docker:
@@ -52,7 +57,10 @@ jobs:
         steps:
             - checkout
             - run: sudo pip install .[mecab,testing]
-            - run: python -m pytest -sv ./tests/test_tokenization_bert_japanese.py
+            - run: python -m pytest -s ./tests/test_tokenization_bert_japanese.py | tee output.txt
+            - store_artifacts:
+                path: ~/transformers/output.txt
+                destination: test_output.txt
     run_examples_torch:
         working_directory: ~/transformers
         docker:
@@ -65,7 +73,10 @@ jobs:
             - checkout
             - run: sudo pip install .[sklearn,torch,testing]
             - run: sudo pip install -r examples/requirements.txt
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./examples/
+            - run: python -m pytest -n 8 --dist=loadfile -s ./examples/ | tee output.txt
+            - store_artifacts:
+                  path: ~/transformers/output.txt
+                  destination: test_output.txt
     build_doc:
         working_directory: ~/transformers
         docker:

.circleci/deploy.sh

@@ -46,4 +46,5 @@ deploy_doc "11c3257" v2.8.0
 deploy_doc "e7cfc1a" v2.9.0
 deploy_doc "7cb203f" v2.9.1
 deploy_doc "10d7239" v2.10.0 
-deploy_doc "b42586e" #v2.11.0 Latest stable release
+deploy_doc "b42586e" v2.11.0
+deploy_doc "b62ca59" #v3.0.0 Latest stable release

.github/workflows/self-push.yml

@@ -51,4 +51,11 @@ jobs:
         USE_CUDA: yes
       run: |
         source .env/bin/activate
-        python -m pytest -n 2 --dist=loadfile -s -v ./tests/
+        python -m pytest -n 2 --dist=loadfile -s ./tests/ | tee output.txt
+    - name: cat output.txt
+      run: cat output.txt
+    - name: Upload output.txt
+      uses: actions/upload-artifact@v1
+      with:
+        name: pytest_output
+        path: output.txt

.github/workflows/self-scheduled.yml

@@ -46,5 +46,11 @@ jobs:
         USE_CUDA: yes
       run: |
         source .env/bin/activate
-        python -m pytest -n 1 --dist=loadfile -s -v ./tests/
-        
+        python -m pytest -n 1 --dist=loadfile -s ./tests/  | tee output.txt
+    - name: cat output.txt
+      run: cat output.txt
+    - name: Upload output.txt
+      uses: actions/upload-artifact@v1
+      with:
+        name: pytest_output
+        path: output.txt

README.md

@@ -24,6 +24,7 @@
 
 🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) provides state-of-the-art general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet, T5, CTRL...) for Natural Language Understanding (NLU) and Natural Language Generation (NLG) with over thousands of pretrained models in 100+ languages and deep interoperability between PyTorch & TensorFlow 2.0.
 
+### Recent contributors
 [![](https://sourcerer.io/fame/clmnt/huggingface/transformers/images/0)](https://sourcerer.io/fame/clmnt/huggingface/transformers/links/0)[![](https://sourcerer.io/fame/clmnt/huggingface/transformers/images/1)](https://sourcerer.io/fame/clmnt/huggingface/transformers/links/1)[![](https://sourcerer.io/fame/clmnt/huggingface/transformers/images/2)](https://sourcerer.io/fame/clmnt/huggingface/transformers/links/2)[![](https://sourcerer.io/fame/clmnt/huggingface/transformers/images/3)](https://sourcerer.io/fame/clmnt/huggingface/transformers/links/3)[![](https://sourcerer.io/fame/clmnt/huggingface/transformers/images/4)](https://sourcerer.io/fame/clmnt/huggingface/transformers/links/4)[![](https://sourcerer.io/fame/clmnt/huggingface/transformers/images/5)](https://sourcerer.io/fame/clmnt/huggingface/transformers/links/5)[![](https://sourcerer.io/fame/clmnt/huggingface/transformers/images/6)](https://sourcerer.io/fame/clmnt/huggingface/transformers/links/6)[![](https://sourcerer.io/fame/clmnt/huggingface/transformers/images/7)](https://sourcerer.io/fame/clmnt/huggingface/transformers/links/7)
 
 ### Features
@@ -287,8 +288,8 @@ pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf
 sentence_0 = "This research was consistent with his findings."
 sentence_1 = "His findings were compatible with this research."
 sentence_2 = "His findings were not compatible with this research."
-inputs_1 = tokenizer.encode_plus(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt')
-inputs_2 = tokenizer.encode_plus(sentence_0, sentence_2, add_special_tokens=True, return_tensors='pt')
+inputs_1 = tokenizer(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt')
+inputs_2 = tokenizer(sentence_0, sentence_2, add_special_tokens=True, return_tensors='pt')
 
 pred_1 = pytorch_model(inputs_1['input_ids'], token_type_ids=inputs_1['token_type_ids'])[0].argmax().item()
 pred_2 = pytorch_model(inputs_2['input_ids'], token_type_ids=inputs_2['token_type_ids'])[0].argmax().item()

docs/README.md

@@ -167,7 +167,7 @@ Here's an example showcasing everything so far:
 
             Indices can be obtained using :class:`transformers.AlbertTokenizer`.
             See :func:`transformers.PreTrainedTokenizer.encode` and
-            :func:`transformers.PreTrainedTokenizer.encode_plus` for details.
+            :func:`transformers.PreTrainedTokenizer.__call__` for details.
 
             `What are input IDs? <../glossary.html#input-ids>`__
 ```

docs/source/_static/js/custom.js

@@ -1,10 +1,11 @@
 // These two things need to be updated at each release for the version selector.
 // Last stable version
-const stableVersion = "v2.11.0"
+const stableVersion = "v3.0.0"
 // Dictionary doc folder to label
 const versionMapping = {
     "master": "master",
-    "": "v2.11.0 (stable)",
+    "": "v3.0.0 (stable)",
+    "v2.11.0": "v2.11.0",
     "v2.10.0": "v2.10.0",
     "v2.9.1": "v2.9.0/v2.9.1",
     "v2.8.0": "v2.8.0",
@@ -86,7 +87,7 @@ function addVersionControl() {
     const parts = location.toString().split('/');
     let versionIndex = parts.length - 2;
     // Index page may not have a last part with filename.html so we need to go up
-    if (parts[parts.length - 1] != "" && ! parts[parts.length - 1].match(/\.html$/)) {
+    if (parts[parts.length - 1] != "" && ! parts[parts.length - 1].match(/\.html$|^search.html?/)) {
         versionIndex = parts.length - 1;
     }
     // Main classes and models are nested so we need to go deeper

docs/source/benchmarks.md

@@ -1,54 +0,0 @@
-# Benchmarks
-
-This section is dedicated to the Benchmarks done by the library, both by maintainers, contributors and users. These 
-benchmark will help keep track of the preformance improvements that are brought to our models across versions.
-
-## Benchmarking all models for inference
-
-As of version 2.1 we have benchmarked all models for inference, across many different settings: using PyTorch, with
-and without TorchScript, using TensorFlow, with and without XLA. All of those tests were done across CPUs (except for
-TensorFlow XLA) and GPUs.
-
-The approach is detailed in the [following blogpost](https://medium.com/huggingface/benchmarking-transformers-pytorch-and-tensorflow-e2917fb891c2)
-
-The results are available [here](https://docs.google.com/spreadsheets/d/1sryqufw2D0XlUH4sq3e9Wnxu5EAQkaohzrJbd5HdQ_w/edit?usp=sharing).
-
-## TF2 with mixed precision, XLA, Distribution (@tlkh)
-
-This work was done by [Timothy Liu](https://github.com/tlkh).
-
-There are very positive results to be gained from the various TensorFlow 2.0 features:
-
-- Automatic Mixed Precision (AMP)
-- XLA compiler
-- Distribution strategies (multi-GPU)
-
-The benefits are listed here (tested on CoLA, MRPC, SST-2):
-
-- AMP: Between 1.4x to 1.6x decrease in overall time without change in batch size
-- AMP+XLA: Up to 2.5x decrease in overall time on SST-2 (larger dataset)
-- Distribution: Between 1.4x to 3.4x decrease in overall time on 4xV100
-- Combined: Up to 5.7x decrease in overall training time, or 9.1x training throughput
-
-The model quality (measured by the validation accuracy) fluctuates slightly. Taking an average of 4 training runs 
-on a single GPU gives the following results:
-
-- CoLA: AMP results in slighter lower acc (0.820 vs 0.824)
-- MRPC: AMP results in lower acc (0.823 vs 0.835)
-- SST-2: AMP results in slighter lower acc (0.918 vs 0.922)
-
-However, in a distributed setting with 4xV100 (4x batch size), AMP can yield in better results:
-
-CoLA: AMP results in higher acc (0.828 vs 0.812)
-MRPC: AMP results in lower acc (0.817 vs 0.827)
-SST-2: AMP results in slightly lower acc (0.926 vs 0.929)
-
-The benchmark script is available [here](https://github.com/NVAITC/benchmarking/blob/master/tf2/bert_dist.py).
-
-Note: on some tasks (e.g. MRPC), the dataset is too small. The overhead due to the model compilation with XLA as well
-as the distribution strategy setup does not speed things up. The XLA compile time is also the reason why although throughput 
-can increase a lot (e.g. 2.7x for single GPU), overall (end-to-end) training speed-up is not as fast (as low as 1.4x)
-
-The benefits as seen on SST-2 (larger dataset) is much clear.
-
-All results can be seen on this [Google Sheet](https://docs.google.com/spreadsheets/d/1538MN224EzjbRL239sqSiUy6YY-rAjHyXhTzz_Zptls/edit#gid=960868445).

docs/source/benchmarks.rst

@@ -0,0 +1,322 @@
+Benchmarks
+==========
+
+Let's take a look at how 🤗 Transformer models can be benchmarked, best practices, and already available benchmarks.
+
+A notebook explaining in more detail how to benchmark 🤗 Transformer models can be found `here <https://github.com/huggingface/transformers/blob/master/notebooks/05-benchmark.ipynb>`__.
+
+How to benchmark 🤗 Transformer models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The classes :class:`~transformers.PyTorchBenchmark` and :class:`~transformers.TensorFlowBenchmark` allow to flexibly benchmark 🤗 Transformer models.
+The benchmark classes allow us to measure the `peak memory usage` and `required time` for both 
+`inference` and `training`. 
+
+.. note::
+
+  Hereby, `inference` is defined by a single forward pass, and `training` is defined by a single forward pass and backward pass.
+
+The benchmark classes :class:`~transformers.PyTorchBenchmark` and :class:`~transformers.TensorFlowBenchmark` expect an object of type :class:`~transformers.PyTorchBenchmarkArguments` and :class:`~transformers.TensorFlowBenchmarkArguments`, respectively, for instantiation. :class:`~transformers.PyTorchBenchmarkArguments` and :class:`~transformers.TensorFlowBenchmarkArguments` are data classes and contain all relevant configurations for their corresponding benchmark class.
+In the following example, it is shown how a BERT model of type `bert-base-cased` can be benchmarked.
+
+.. code-block::
+
+    >>> ## PYTORCH CODE
+    >>> from transformers import PyTorchBenchmark, PyTorchBenchmarkArguments
+
+    >>> args = PyTorchBenchmarkArguments(models=["bert-base-uncased"], batch_sizes=[8], sequence_lengths=[8, 32, 128, 512])
+    >>> benchmark = PyTorchBenchmark(args)
+
+    >>> ## TENSORFLOW CODE
+    >>> from transformers import TensorFlowBenchmark, TensorFlowBenchmarkArguments
+
+    >>> args = TensorFlowBenchmarkArguments(models=["bert-base-uncased"], batch_sizes=[8], sequence_lengths=[8, 32, 128, 512])
+    >>> benchmark = TensorFlowBenchmark(args)
+
+
+Here, three arguments are given to the benchmark argument data classes, namely ``models``, ``batch_sizes``, and ``sequence_lengths``. The argument ``models`` is required and expects a :obj:`list` of model identifiers from the `model hub <https://huggingface.co/models>`__
+The :obj:`list` arguments ``batch_sizes`` and ``sequence_lengths`` define the size of the ``input_ids`` on which the model is benchmarked. 
+There are many more parameters that can be configured via the benchmark argument data classes. For more detail on these one can either directly consult the files 
+``src/transformers/benchmark/benchmark_args_utils.py``, ``src/transformers/benchmark/benchmark_args.py`` (for PyTorch) and ``src/transformers/benchmark/benchmark_args_tf.py`` (for Tensorflow). 
+Alternatively, running the following shell commands from root will print out a descriptive list of all configurable parameters for PyTorch and Tensorflow respectively.
+
+.. code-block::
+
+    >>> ## PYTORCH CODE
+    python examples/benchmarking/run_benchmark.py --help
+
+    >>> ## TENSORFLOW CODE
+    python examples/benchmarking/run_benchmark_tf.py --help
+
+
+An instantiated benchmark object can then simply be run by calling ``benchmark.run()``.
+
+.. code-block::
+
+    >>> ## PYTORCH CODE
+    >>> results = benchmark.run()
+    >>> print(results)
+    ====================       INFERENCE - SPEED - RESULT       ====================
+    --------------------------------------------------------------------------------
+    Model Name             Batch Size     Seq Length     Time in s                  
+    --------------------------------------------------------------------------------
+    bert-base-uncased          8               8             0.006     
+    bert-base-uncased          8               32            0.006     
+    bert-base-uncased          8              128            0.018     
+    bert-base-uncased          8              512            0.088     
+    --------------------------------------------------------------------------------
+    
+    ====================      INFERENCE - MEMORY - RESULT       ====================
+    --------------------------------------------------------------------------------
+    Model Name             Batch Size     Seq Length    Memory in MB 
+    --------------------------------------------------------------------------------
+    bert-base-uncased          8               8             1227
+    bert-base-uncased          8               32            1281
+    bert-base-uncased          8              128            1307
+    bert-base-uncased          8              512            1539
+    --------------------------------------------------------------------------------
+    
+    ====================        ENVIRONMENT INFORMATION         ====================
+    - transformers_version: 2.11.0
+    - framework: PyTorch
+    - use_torchscript: False
+    - framework_version: 1.4.0
+    - python_version: 3.6.10
+    - system: Linux
+    - cpu: x86_64
+    - architecture: 64bit
+    - date: 2020-06-29
+    - time: 08:58:43.371351
+    - fp16: False
+    - use_multiprocessing: True
+    - only_pretrain_model: False
+    - cpu_ram_mb: 32088
+    - use_gpu: True
+    - num_gpus: 1
+    - gpu: TITAN RTX
+    - gpu_ram_mb: 24217
+    - gpu_power_watts: 280.0
+    - gpu_performance_state: 2
+    - use_tpu: False
+    
+    >>> ## TENSORFLOW CODE
+    >>> results = benchmark.run()
+    >>> print(results)
+    ====================       INFERENCE - SPEED - RESULT       ====================
+    --------------------------------------------------------------------------------
+    Model Name             Batch Size     Seq Length     Time in s                  
+    --------------------------------------------------------------------------------
+    bert-base-uncased          8               8             0.005
+    bert-base-uncased          8               32            0.008
+    bert-base-uncased          8              128            0.022
+    bert-base-uncased          8              512            0.105
+    --------------------------------------------------------------------------------
+    
+    ====================      INFERENCE - MEMORY - RESULT       ====================
+    --------------------------------------------------------------------------------
+    Model Name             Batch Size     Seq Length    Memory in MB 
+    --------------------------------------------------------------------------------
+    bert-base-uncased          8               8             1330
+    bert-base-uncased          8               32            1330
+    bert-base-uncased          8              128            1330
+    bert-base-uncased          8              512            1770
+    --------------------------------------------------------------------------------
+    
+    ====================        ENVIRONMENT INFORMATION         ====================
+    - transformers_version: 2.11.0
+    - framework: Tensorflow
+    - use_xla: False
+    - framework_version: 2.2.0
+    - python_version: 3.6.10
+    - system: Linux
+    - cpu: x86_64
+    - architecture: 64bit
+    - date: 2020-06-29
+    - time: 09:26:35.617317
+    - fp16: False
+    - use_multiprocessing: True
+    - only_pretrain_model: False
+    - cpu_ram_mb: 32088
+    - use_gpu: True
+    - num_gpus: 1
+    - gpu: TITAN RTX
+    - gpu_ram_mb: 24217
+    - gpu_power_watts: 280.0
+    - gpu_performance_state: 2
+    - use_tpu: False
+
+By default, the `time` and the `required memory` for `inference` are benchmarked. 
+In the example output above the first two sections show the result corresponding to `inference time` and `inference memory`. 
+In addition, all relevant information about the computing environment, `e.g.` the GPU type, the system, the library versions, etc... are printed out in the third section under `ENVIRONMENT INFORMATION`.
+This information can optionally be saved in a `.csv` file when adding the argument :obj:`save_to_csv=True` to :class:`~transformers.PyTorchBenchmarkArguments` and :class:`~transformers.TensorFlowBenchmarkArguments` respectively.
+In this case, every section is saved in a separate `.csv` file. The path to each `.csv` file can optionally be defined via the argument data classes.
+
+Instead of benchmarking pre-trained models via their model identifier, `e.g.` `bert-base-uncased`, the user can alternatively benchmark an arbitrary configuration of any available model class. 
+In this case, a :obj:`list` of configurations must be inserted with the benchmark args as follows.
+
+.. code-block::
+
+    >>> ## PYTORCH CODE
+    >>> from transformers import PyTorchBenchmark, PyTorchBenchmarkArguments, BertConfig
+
+    >>> args = PyTorchBenchmarkArguments(models=["bert-base", "bert-384-hid", "bert-6-lay"], batch_sizes=[8], sequence_lengths=[8, 32, 128, 512])
+    >>> config_base = BertConfig()
+    >>> config_384_hid = BertConfig(hidden_size=384)
+    >>> config_6_lay = BertConfig(num_hidden_layers=6)
+
+    >>> benchmark = PyTorchBenchmark(args, configs=[config_base, config_384_hid, config_6_lay])
+    >>> benchmark.run()
+    ====================       INFERENCE - SPEED - RESULT       ====================
+    --------------------------------------------------------------------------------
+    Model Name             Batch Size     Seq Length       Time in s                  
+    --------------------------------------------------------------------------------
+    bert-base                  8              128            0.006
+    bert-base                  8              512            0.006
+    bert-base                  8              128            0.018     
+    bert-base                  8              512            0.088     
+    bert-384-hid              8               8             0.006     
+    bert-384-hid              8               32            0.006     
+    bert-384-hid              8              128            0.011     
+    bert-384-hid              8              512            0.054     
+    bert-6-lay                 8               8             0.003     
+    bert-6-lay                 8               32            0.004     
+    bert-6-lay                 8              128            0.009     
+    bert-6-lay                 8              512            0.044
+    --------------------------------------------------------------------------------
+    
+    ====================      INFERENCE - MEMORY - RESULT       ====================
+    --------------------------------------------------------------------------------
+    Model Name             Batch Size     Seq Length      Memory in MB 
+    --------------------------------------------------------------------------------
+    bert-base                  8               8             1277
+    bert-base                  8               32            1281
+    bert-base                  8              128            1307     
+    bert-base                  8              512            1539     
+    bert-384-hid              8               8             1005     
+    bert-384-hid              8               32            1027     
+    bert-384-hid              8              128            1035     
+    bert-384-hid              8              512            1255     
+    bert-6-lay                 8               8             1097     
+    bert-6-lay                 8               32            1101     
+    bert-6-lay                 8              128            1127     
+    bert-6-lay                 8              512            1359
+    --------------------------------------------------------------------------------
+    
+    ====================        ENVIRONMENT INFORMATION         ====================
+    - transformers_version: 2.11.0
+    - framework: PyTorch
+    - use_torchscript: False
+    - framework_version: 1.4.0
+    - python_version: 3.6.10
+    - system: Linux
+    - cpu: x86_64
+    - architecture: 64bit
+    - date: 2020-06-29
+    - time: 09:35:25.143267
+    - fp16: False
+    - use_multiprocessing: True
+    - only_pretrain_model: False
+    - cpu_ram_mb: 32088
+    - use_gpu: True
+    - num_gpus: 1
+    - gpu: TITAN RTX
+    - gpu_ram_mb: 24217
+    - gpu_power_watts: 280.0
+    - gpu_performance_state: 2
+    - use_tpu: False
+
+    >>> ## TENSORFLOW CODE
+    >>> from transformers import TensorFlowBenchmark, TensorFlowBenchmarkArguments, BertConfig
+
+    >>> args = TensorFlowBenchmarkArguments(models=["bert-base", "bert-384-hid", "bert-6-lay"], batch_sizes=[8], sequence_lengths=[8, 32, 128, 512])
+    >>> config_base = BertConfig()
+    >>> config_384_hid = BertConfig(hidden_size=384)
+    >>> config_6_lay = BertConfig(num_hidden_layers=6)
+
+    >>> benchmark = TensorFlowBenchmark(args, configs=[config_base, config_384_hid, config_6_lay])
+    >>> benchmark.run()
+    ====================       INFERENCE - SPEED - RESULT       ====================
+    --------------------------------------------------------------------------------
+    Model Name             Batch Size     Seq Length       Time in s                  
+    --------------------------------------------------------------------------------
+    bert-base                  8               8             0.005
+    bert-base                  8               32            0.008
+    bert-base                  8              128            0.022
+    bert-base                  8              512            0.106
+    bert-384-hid              8               8             0.005
+    bert-384-hid              8               32            0.007
+    bert-384-hid              8              128            0.018
+    bert-384-hid              8              512            0.064
+    bert-6-lay                 8               8             0.002
+    bert-6-lay                 8               32            0.003
+    bert-6-lay                 8              128            0.0011
+    bert-6-lay                 8              512            0.074
+    --------------------------------------------------------------------------------
+    
+    ====================      INFERENCE - MEMORY - RESULT       ====================
+    --------------------------------------------------------------------------------
+    Model Name             Batch Size     Seq Length      Memory in MB 
+    --------------------------------------------------------------------------------
+    bert-base                  8               8             1330
+    bert-base                  8               32            1330
+    bert-base                  8              128            1330
+    bert-base                  8              512            1770
+    bert-384-hid              8               8             1330
+    bert-384-hid              8               32            1330
+    bert-384-hid              8              128            1330
+    bert-384-hid              8              512            1540
+    bert-6-lay                 8               8             1330
+    bert-6-lay                 8               32            1330
+    bert-6-lay                 8              128            1330
+    bert-6-lay                 8              512            1540
+    --------------------------------------------------------------------------------
+    
+    ====================        ENVIRONMENT INFORMATION         ====================
+    - transformers_version: 2.11.0
+    - framework: Tensorflow
+    - use_xla: False
+    - framework_version: 2.2.0
+    - python_version: 3.6.10
+    - system: Linux
+    - cpu: x86_64
+    - architecture: 64bit
+    - date: 2020-06-29
+    - time: 09:38:15.487125
+    - fp16: False
+    - use_multiprocessing: True
+    - only_pretrain_model: False
+    - cpu_ram_mb: 32088
+    - use_gpu: True
+    - num_gpus: 1
+    - gpu: TITAN RTX
+    - gpu_ram_mb: 24217
+    - gpu_power_watts: 280.0
+    - gpu_performance_state: 2
+    - use_tpu: False
+
+
+Again, `inference time` and `required memory` for `inference` are measured, but this time for customized configurations of the :obj:`BertModel` class. This feature can especially be helpful when 
+deciding for which configuration the model should be trained.
+
+
+Benchmark best practices
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This section lists a couple of best practices one should be aware of when benchmarking a model.
+
+- Currently, only single device benchmarking is supported. When benchmarking on GPU, it is recommended that the user 
+  specifies on which device the code should be run by setting the ``CUDA_VISIBLE_DEVICES`` environment variable in the shell, `e.g.` ``export CUDA_VISIBLE_DEVICES=0`` before running the code.
+- The option :obj:`no_multi_processing` should only be set to :obj:`True` for testing and debugging. To ensure accurate memory measurement it is recommended to run each memory benchmark in a separate process by making sure :obj:`no_multi_processing` is set to :obj:`True`.
+- One should always state the environment information when sharing the results of a model benchmark. Results can vary heavily between different GPU devices, library versions, etc., so that benchmark results on their own are not very useful for the community.
+
+
+Sharing your benchmark
+~~~~~~~~~~~~~~~~~~~~~~
+
+Previously all available core models (10 at the time) have been benchmarked for `inference time`, across many different settings: using PyTorch, with
+and without TorchScript, using TensorFlow, with and without XLA. All of those tests were done across CPUs (except for
+TensorFlow XLA) and GPUs.
+
+The approach is detailed in the `following blogpost <https://medium.com/huggingface/benchmarking-transformers-pytorch-and-tensorflow-e2917fb891c2>`__ and the results are available `here <https://docs.google.com/spreadsheets/d/1sryqufw2D0XlUH4sq3e9Wnxu5EAQkaohzrJbd5HdQ_w/edit?usp=sharing>`__.
+
+With the new `benchmark` tools, it is easier than ever to share your benchmark results with the community `here <https://github.com/huggingface/transformers/blob/master/examples/benchmarking/README.md>`__.

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'2.11.0'
+release = u'3.0.2'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/glossary.rst

@@ -11,7 +11,7 @@ General terms
   tokens at a certain timestep.
 - MLM: masked language modeling, a pretraining task where the model sees a corrupted version of the texts, usually done
   by masking some tokens randomly, and has to predict the original text.
-- multimodal: a task taht combines texts with another kind of inputs (for instance images).
+- multimodal: a task that combines texts with another kind of inputs (for instance images).
 - NLG: natural language generation, all tasks related to generating text ( for instance talk with transformers,
   translation)
 - NLP: natural language processing, a generic way to say "deal with texts".

docs/source/index.rst

@@ -139,10 +139,10 @@ conversion utilities for the following models:
 
     task_summary
     model_summary
-    training
     preprocessing
-    serialization
+    training
     model_sharing
+    tokenizer_summary
     multilingual
 
 .. toctree::
@@ -174,6 +174,7 @@ conversion utilities for the following models:
     main_classes/pipelines
     main_classes/optimizer_schedules
     main_classes/processors
+    main_classes/trainer
     model_doc/auto
     model_doc/encoderdecoder
     model_doc/bert

docs/source/main_classes/optimizer_schedules.rst

@@ -1,4 +1,4 @@
-Optimizer
+Optimization
 ----------------------------------------------------
 
 The ``.optimization`` module provides:
@@ -7,24 +7,25 @@ The ``.optimization`` module provides:
 - several schedules in the form of schedule objects that inherit from ``_LRSchedule``:
 - a gradient accumulation class to accumulate the gradients of multiple batches
 
-``AdamW``
-~~~~~~~~~~~~~~~~
+``AdamW`` (PyTorch)
+~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.AdamW
     :members:
 
-``AdamWeightDecay``
-~~~~~~~~~~~~~~~~~~~
+``AdamWeightDecay`` (TensorFlow)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.AdamWeightDecay
 
 .. autofunction:: transformers.create_optimizer
 
 Schedules
-----------------------------------------------------
+~~~~~~~~~~~~~~~~~~~
+
+Learning Rate Schedules (Pytorch)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Learning Rate Schedules
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autofunction:: transformers.get_constant_schedule
 
 
@@ -56,16 +57,16 @@ Learning Rate Schedules
     :target: /imgs/warmup_linear_schedule.png
     :alt:
 
-``Warmup``
-~~~~~~~~~~~~~~~~
+``Warmup`` (TensorFlow)
+^^^^^^^^^^^^^^^^^^^^^^^
 
 .. autoclass:: transformers.WarmUp
     :members:
 
 Gradient Strategies
-----------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~
 
-``GradientAccumulator``
-~~~~~~~~~~~~~~~~~~~~~~~
+``GradientAccumulator`` (TensorFlow)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. autoclass:: transformers.GradientAccumulator

docs/source/main_classes/tokenizer.rst

@@ -11,7 +11,7 @@ The base classes ``PreTrainedTokenizer`` and ``PreTrainedTokenizerFast`` impleme
 - adding new tokens to the vocabulary in a way that is independant of the underlying structure (BPE, SentencePiece...),
 - managing special tokens like mask, beginning-of-sentence, etc tokens (adding them, assigning them to attributes in the tokenizer for easy access and making sure they are not split during tokenization)
 
-``BatchEncoding`` holds the output of the tokenizer's encoding methods (``encode_plus`` and ``batch_encode_plus``) and is derived from a Python dictionary. When the tokenizer is a pure python tokenizer, this class behave just like a standard python dictionary and hold the various model inputs computed by these methodes (``input_ids``, ``attention_mask``...). When the tokenizer is a "Fast" tokenizer (i.e. backed by HuggingFace tokenizers library), this class provides in addition several advanced alignement methods which can be used to map between the original string (character and words) and the token space (e.g. getting the index of the token comprising a given character or the span of characters corresponding to a given token).
+``BatchEncoding`` holds the output of the tokenizer's encoding methods (``__call__``, ``encode_plus`` and ``batch_encode_plus``) and is derived from a Python dictionary. When the tokenizer is a pure python tokenizer, this class behave just like a standard python dictionary and hold the various model inputs computed by these methodes (``input_ids``, ``attention_mask``...). When the tokenizer is a "Fast" tokenizer (i.e. backed by HuggingFace tokenizers library), this class provides in addition several advanced alignement methods which can be used to map between the original string (character and words) and the token space (e.g. getting the index of the token comprising a given character or the span of characters corresponding to a given token).
 
 ``PreTrainedTokenizer``
 ~~~~~~~~~~~~~~~~~~~~~~~~

docs/source/main_classes/trainer.rst

@@ -0,0 +1,45 @@
+Trainer
+----------
+
+The :class:`~transformers.Trainer` and :class:`~transformers.TFTrainer` classes provide an API for feature-complete
+training in most standard use cases. It's used in most of the :doc:`example scripts <../examples>`.
+
+Before instantiating your :class:`~transformers.Trainer`/:class:`~transformers.TFTrainer`, create a 
+:class:`~transformers.TrainingArguments`/:class:`~transformers.TFTrainingArguments` to access all the points of
+customization during training.
+
+The API supports distributed training on multiple GPUs/TPUs, mixed precision through `NVIDIA Apex
+<https://github.com/NVIDIA/apex>`__ for PyTorch and :obj:`tf.keras.mixed_precision` for TensorFlow.
+
+``Trainer`` 
+~~~~~~~~~~~
+
+.. autoclass:: transformers.Trainer
+    :members:
+
+``TFTrainer`` 
+~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFTrainer
+    :members:
+
+``TrainingArguments``
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TrainingArguments
+    :members:
+
+``TFTrainingArguments``
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.TFTrainingArguments
+    :members:
+
+Utilities
+~~~~~~~~~
+
+.. autoclass:: transformers.EvalPrediction
+
+.. autofunction:: transformers.set_seed
+
+.. autofunction:: transformers.torch_distributed_zero_first

docs/source/model_doc/reformer.rst

@@ -112,3 +112,17 @@ ReformerModelWithLMHead
 
 .. autoclass:: transformers.ReformerModelWithLMHead
     :members:
+
+
+ReformerForMaskedLM
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ReformerForMaskedLM
+    :members:
+
+
+ReformerForQuestionAnswering
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ReformerForQuestionAnswering
+    :members:

docs/source/model_sharing.md

@@ -1,55 +0,0 @@
-# Model upload and sharing
-
-Starting with `v2.2.2`, you can now upload and share your fine-tuned models with the community, using the <abbr title="Command-line interface">CLI</abbr> that's built-in to the library.
-
-**First, create an account on [https://huggingface.co/join](https://huggingface.co/join)**. Optionally, join an existing organization or create a new one. Then:
-
-```shell
-transformers-cli login
-# log in using the same credentials as on huggingface.co
-```
-Upload your model:
-```shell
-transformers-cli upload ./path/to/pretrained_model/
-
-# ^^ Upload folder containing weights/tokenizer/config
-# saved via `.save_pretrained()`
-
-transformers-cli upload ./config.json [--filename folder/foobar.json]
-
-# ^^ Upload a single file
-# (you can optionally override its filename, which can be nested inside a folder)
-```
-
-If you want your model to be namespaced by your organization name rather than your username, add the following flag to any command:
-```shell
---organization organization_name
-```
-
-Your model will then be accessible through its identifier, a concatenation of your username (or organization name) and the folder name above:
-```python
-"username/pretrained_model"
-# or if an org:
-"organization_name/pretrained_model"
-```
-
-**Please add a README.md model card** to the repo under `model_cards/` with: model description, training params (dataset, preprocessing, hardware used, hyperparameters), evaluation results, intended uses & limitations, etc.
-
-Your model now has a page on huggingface.co/models 🔥
-
-Anyone can load it from code:
-```python
-tokenizer = AutoTokenizer.from_pretrained("namespace/pretrained_model")
-model = AutoModel.from_pretrained("namespace/pretrained_model")
-```
-
-List all your files on S3:
-```shell
-transformers-cli s3 ls
-```
-
-You can also delete unneeded files:
-
-```shell
-transformers-cli s3 rm …
-```

docs/source/model_sharing.rst

@@ -0,0 +1,217 @@
+Model sharing and uploading
+===========================
+
+In this page, we will show you how to share a model you have trained or fine-tuned on new data with the community on
+the `model hub <https://huggingface.co/models>`__.
+
+.. note::
+
+    You will need to create an account on `huggingface.co <https://huggingface.co/join>`__ for this.
+
+    Optionally, you can join an existing organization or create a new one.
+
+Prepare your model for uploading
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We have seen in the :doc:`training tutorial <training>`: how to fine-tune a model on a given task. You have probably
+done something similar on your task, either using the model directly in your own training loop or using the
+:class:`~.transformers.Trainer`/:class:`~.transformers.TFTrainer` class. Let's see how you can share the result on
+the `model hub <https://huggingface.co/models>`__.
+
+Basic steps
+^^^^^^^^^^^
+
+.. 
+    When #5258 is merged, we can remove the need to create the directory.
+
+First, pick a directory with the name you want your model to have on the model hub (its full name will then be
+`username/awesome-name-you-picked` or `organization/awesome-name-you-picked`) and create it with either
+
+::
+
+    mkdir path/to/awesome-name-you-picked
+
+or in python
+
+::
+
+    import os
+    os.makedirs("path/to/awesome-name-you-picked")
+
+then you can save your model and tokenizer with:
+
+::
+
+    model.save_pretrained("path/to/awesome-name-you-picked")
+    tokenizer.save_pretrained("path/to/awesome-name-you-picked")
+
+Or, if you're using the Trainer API
+
+::
+
+    trainer.save_model("path/to/awesome-name-you-picked")
+    tokenizer.save_pretrained("path/to/awesome-name-you-picked")
+
+Make your model work on all frameworks
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. 
+    TODO Sylvain: make this automatic during the upload
+
+You probably have your favorite framework, but so will other users! That's why it's best to upload your model with both
+PyTorch `and` TensorFlow checkpoints to make it easier to use (if you skip this step, users will still be able to load
+your model in another framework, but it will be slower, as it will have to be converted on the fly). Don't worry, it's super easy to do (and in a future version,
+it will all be automatic). You will need to install both PyTorch and TensorFlow for this step, but you don't need to
+worry about the GPU, so it should be very easy. Check the
+`TensorFlow installation page <https://www.tensorflow.org/install/pip#tensorflow-2.0-rc-is-available>`__ 
+and/or the `PyTorch installation page <https://pytorch.org/get-started/locally/#start-locally>`__ to see how.
+
+First check that your model class exists in the other framework, that is try to import the same model by either adding
+or removing TF. For instance, if you trained a :class:`~transformers.DistilBertForSequenceClassification`, try to
+type
+
+::
+
+    from transformers import TFDistilBertForSequenceClassification
+
+and if you trained a :class:`~transformers.TFDistilBertForSequenceClassification`, try to
+type
+
+::
+
+    from transformers import DistilBertForSequenceClassification
+
+This will give back an error if your model does not exist in the other framework (something that should be pretty rare
+since we're aiming for full parity between the two frameworks). In this case, skip this and go to the next step.
+
+Now, if you trained your model in PyTorch and have to create a TensorFlow version, adapt the following code to your
+model class:
+
+::
+
+    tf_model = TFDistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_pt=True)
+    tf_model.save_pretrained("path/to/awesome-name-you-picked")
+
+and if you trained your model in TensorFlow and have to create a PyTorch version, adapt the following code to your
+model class:
+
+::
+
+    pt_model = DistilBertForSequenceClassification.from_pretrained("path/to/awesome-name-you-picked", from_tf=True)
+    pt_model.save_pretrained("path/to/awesome-name-you-picked")
+
+That's all there is to it!
+
+Check the directory before uploading
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Make sure there are no garbage files in the directory you'll upload. It should only have:
+
+- a `config.json` file, which saves the :doc:`configuration <main_classes/configuration>` of your model ;
+- a `pytorch_model.bin` file, which is the PyTorch checkpoint (unless you can't have it for some reason) ;
+- a `tf_model.h5` file, which is the TensorFlow checkpoint (unless you can't have it for some reason) ;
+- a `special_tokens_map.json`, which is part of your :doc:`tokenizer <main_classes/tokenizer>` save;
+- a `tokenizer_config.json`, which is part of your :doc:`tokenizer <main_classes/tokenizer>` save;
+- a `vocab.txt`, which is the vocabulary of your tokenizer, part of your :doc:`tokenizer <main_classes/tokenizer>`
+  save;
+- maybe a `added_tokens.json`, which is part of your :doc:`tokenizer <main_classes/tokenizer>` save.
+
+Other files can safely be deleted.
+
+Upload your model with the CLI
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Now go in a terminal and run the following command. It should be in the virtual enviromnent where you installed 🤗
+Transformers, since that command :obj:`transformers-cli` comes from the library.
+
+::
+
+    transformers-cli login
+
+Then log in using the same credentials as on huggingface.co. To upload your model, just type
+
+::
+
+    transformers-cli upload path/to/awesome-name-you-picked/
+
+This will upload the folder containing the weights, tokenizer and configuration we prepared in the previous section.
+
+If you want to upload a single file (a new version of your model, or the other framework checkpoint you want to add),
+just type:
+
+::
+
+    transformers-cli upload path/to/awesome-name-you-picked/that-file 
+
+or
+
+::
+
+   transformers-cli upload path/to/awesome-name-you-picked/that-file --filename awesome-name-you-picked/new_name
+
+if you want to change its filename.
+
+This uploads the model to your personal account. If you want your model to be namespaced by your organization name
+rather than your username, add the following flag to any command:
+
+::
+
+    --organization organization_name
+
+so for instance:
+
+::
+
+    transformers-cli upload path/to/awesome-name-you-picked/ --organization organization_name
+
+Your model will then be accessible through its identifier, which is, as we saw above,
+`username/awesome-name-you-picked` or `organization/awesome-name-you-picked`.
+
+Add a model card
+^^^^^^^^^^^^^^^^
+
+To make sure everyone knows what your model can do, what its limitations and potential bias or ethetical
+considerations, please add a README.md model card to the 🤗 Transformers repo under `model_cards/`. It should then be
+placed in a subfolder with your username or organization, then another subfolder named like your model
+(`awesome-name-you-picked`). Or just click on the "Create a model card on GitHub" button on the model page, it will
+get you directly to the right location. If you need one, `here <https://github.com/huggingface/model_card>`__ is a
+model card template (meta-suggestions are welcome).
+
+If your model is fine-tuned from another model coming from the model hub (all 🤗 Transformers pretrained models do),
+don't forget to link to its model card so that people can fully trace how your model was built.
+
+If you have never made a pull request to the 🤗 Transformers repo, look at the
+:doc:`contributing guide <contributing>` to see the steps to follow.
+
+.. Note::
+
+    You can also send your model card in the folder you uploaded with the CLI by placing it in a `README.md` file
+    inside `path/to/awesome-name-you-picked/`.
+
+Using your model
+^^^^^^^^^^^^^^^^
+
+Your model now has a page on huggingface.co/models 🔥
+
+Anyone can load it from code:
+
+::
+
+    tokenizer = AutoTokenizer.from_pretrained("namespace/awesome-name-you-picked")
+    model = AutoModel.from_pretrained("namespace/awesome-name-you-picked")
+
+Additional commands
+^^^^^^^^^^^^^^^^^^^
+
+You can list all the files you uploaded on the hub like this:
+
+::
+
+    transformers-cli s3 ls
+
+You can also delete unneeded files with
+
+::
+
+    transformers-cli s3 rm awesome-name-you-picked/filename
+

docs/source/quicktour.rst

@@ -146,8 +146,9 @@ Using the tokenizer
 
 We mentioned the tokenizer is responsible for the preprocessing of your texts. First, it will split a given text in
 words (or part of words, punctuation symbols, etc.) usually called `tokens`. There are multiple rules that can govern
-that process, which is why we need to instantiate the tokenizer using the name of the model, to make sure we use the
-same rules as when the model was pretrained.
+that process (you can learn more about them in the :doc:`tokenizer_summary <tokenizer_summary>`, which is why we need
+to instantiate the tokenizer using the name of the model, to make sure we use the same rules as when the model was
+pretrained.
 
 The second step is to convert those `tokens` into numbers, to be able to build a tensor out of them and feed them to
 the model. To do this, the tokenizer has a `vocab`, which is the part we download when we instantiate it with the
@@ -282,7 +283,7 @@ Models are standard `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#to
 `tf.keras.Model <https://www.tensorflow.org/api_docs/python/tf/keras/Model>`__ so you can use them in your usual
 training loop. 🤗 Transformers also provides a :class:`~transformers.Trainer` (or :class:`~transformers.TFTrainer` if
 you are using TensorFlow) class to help with your training (taking care of things such as distributed training, mixed
-precision, etc.). See the training tutorial (coming soon) for more details.
+precision, etc.). See the :doc:`training tutorial <training>` for more details.
 
 Once your model is fine-tuned, you can save it with its tokenizer the following way:
 

docs/source/serialization.rst

@@ -1,89 +0,0 @@
-Serialization best-practices
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-This section explain how you can save and re-load a fine-tuned model (BERT, GPT, GPT-2 and Transformer-XL).
-There are three types of files you need to save to be able to reload a fine-tuned model:
-
-
-* the model itself which should be saved following PyTorch serialization `best practices <https://pytorch.org/docs/stable/notes/serialization.html#best-practices>`__\ ,
-* the configuration file of the model which is saved as a JSON file, and
-* the vocabulary (and the merges for the BPE-based models GPT and GPT-2).
-
-The *default filenames* of these files are as follow:
-
-
-* the model weights file: ``pytorch_model.bin``\ ,
-* the configuration file: ``config.json``\ ,
-* the vocabulary file: ``vocab.txt`` for BERT and Transformer-XL, ``vocab.json`` for GPT/GPT-2 (BPE vocabulary),
-* for GPT/GPT-2 (BPE vocabulary) the additional merges file: ``merges.txt``.
-
-**If you save a model using these *default filenames*\ , you can then re-load the model and tokenizer using the ``from_pretrained()`` method.**
-
-Here is the recommended way of saving the model, configuration and vocabulary to an ``output_dir`` directory and reloading the model and tokenizer afterwards:
-
-.. code-block:: python
-
-   from transformers import WEIGHTS_NAME, CONFIG_NAME
-
-   output_dir = "./models/"
-
-   # Step 1: Save a model, configuration and vocabulary that you have fine-tuned
-
-   # If we have a distributed model, save only the encapsulated model
-   # (it was wrapped in PyTorch DistributedDataParallel or DataParallel)
-   model_to_save = model.module if hasattr(model, 'module') else model
-
-   # If we save using the predefined names, we can load using `from_pretrained`
-   output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
-   output_config_file = os.path.join(output_dir, CONFIG_NAME)
-
-   torch.save(model_to_save.state_dict(), output_model_file)
-   model_to_save.config.to_json_file(output_config_file)
-   tokenizer.save_pretrained(output_dir)
-
-   # Step 2: Re-load the saved model and vocabulary
-
-   # Example for a Bert model
-   model = BertForQuestionAnswering.from_pretrained(output_dir)
-   tokenizer = BertTokenizer.from_pretrained(output_dir)  # Add specific options if needed
-   # Example for a GPT model
-   model = OpenAIGPTDoubleHeadsModel.from_pretrained(output_dir)
-   tokenizer = OpenAIGPTTokenizer.from_pretrained(output_dir)
-
-Here is another way you can save and reload the model if you want to use specific paths for each type of files:
-
-.. code-block:: python
-
-   output_model_file = "./models/my_own_model_file.bin"
-   output_config_file = "./models/my_own_config_file.bin"
-   output_vocab_file = "./models/my_own_vocab_file.bin"
-
-   # Step 1: Save a model, configuration and vocabulary that you have fine-tuned
-
-   # If we have a distributed model, save only the encapsulated model
-   # (it was wrapped in PyTorch DistributedDataParallel or DataParallel)
-   model_to_save = model.module if hasattr(model, 'module') else model
-
-   torch.save(model_to_save.state_dict(), output_model_file)
-   model_to_save.config.to_json_file(output_config_file)
-   tokenizer.save_vocabulary(output_vocab_file)
-
-   # Step 2: Re-load the saved model and vocabulary
-
-   # We didn't save using the predefined WEIGHTS_NAME, CONFIG_NAME names, we cannot load using `from_pretrained`.
-   # Here is how to do it in this situation:
-
-   # Example for a Bert model
-   config = BertConfig.from_json_file(output_config_file)
-   model = BertForQuestionAnswering(config)
-   state_dict = torch.load(output_model_file)
-   model.load_state_dict(state_dict)
-   tokenizer = BertTokenizer(output_vocab_file, do_lower_case=args.do_lower_case)
-
-   # Example for a GPT model
-   config = OpenAIGPTConfig.from_json_file(output_config_file)
-   model = OpenAIGPTDoubleHeadsModel(config)
-   state_dict = torch.load(output_model_file)
-   model.load_state_dict(state_dict)
-   tokenizer = OpenAIGPTTokenizer(output_vocab_file)
-

docs/source/task_summary.rst

@@ -74,7 +74,7 @@ of each other. The process is the following:
   with the weights stored in the checkpoint.
 - Build a sequence from the two sentences, with the correct model-specific separators token type ids
   and attention masks (:func:`~transformers.PreTrainedTokenizer.encode` and
-  :func:`~transformers.PreTrainedTokenizer.encode_plus` take care of this)
+  :func:`~transformers.PreTrainedTokenizer.__call__` take care of this)
 - Pass this sequence through the model so that it is classified in one of the two available classes: 0
   (not a paraphrase) and 1 (is a paraphrase)
 - Compute the softmax of the result to get probabilities over the classes
@@ -95,8 +95,8 @@ of each other. The process is the following:
     >>> sequence_1 = "Apples are especially bad for your health"
     >>> sequence_2 = "HuggingFace's headquarters are situated in Manhattan"
 
-    >>> paraphrase = tokenizer.encode_plus(sequence_0, sequence_2, return_tensors="pt")
-    >>> not_paraphrase = tokenizer.encode_plus(sequence_0, sequence_1, return_tensors="pt")
+    >>> paraphrase = tokenizer(sequence_0, sequence_2, return_tensors="pt")
+    >>> not_paraphrase = tokenizer(sequence_0, sequence_1, return_tensors="pt")
 
     >>> paraphrase_classification_logits = model(**paraphrase)[0]
     >>> not_paraphrase_classification_logits = model(**not_paraphrase)[0]
@@ -128,8 +128,8 @@ of each other. The process is the following:
     >>> sequence_1 = "Apples are especially bad for your health"
     >>> sequence_2 = "HuggingFace's headquarters are situated in Manhattan"
 
-    >>> paraphrase = tokenizer.encode_plus(sequence_0, sequence_2, return_tensors="tf")
-    >>> not_paraphrase = tokenizer.encode_plus(sequence_0, sequence_1, return_tensors="tf")
+    >>> paraphrase = tokenizer(sequence_0, sequence_2, return_tensors="tf")
+    >>> not_paraphrase = tokenizer(sequence_0, sequence_1, return_tensors="tf")
 
     >>> paraphrase_classification_logits = model(paraphrase)[0]
     >>> not_paraphrase_classification_logits = model(not_paraphrase)[0]
@@ -221,7 +221,7 @@ Here is an example of question answering using a model and a tokenizer. The proc
     ... ]
 
     >>> for question in questions:
-    ...     inputs = tokenizer.encode_plus(question, text, add_special_tokens=True, return_tensors="pt")
+    ...     inputs = tokenizer(question, text, add_special_tokens=True, return_tensors="pt")
     ...     input_ids = inputs["input_ids"].tolist()[0]
     ...
     ...     text_tokens = tokenizer.convert_ids_to_tokens(input_ids)
@@ -263,7 +263,7 @@ Here is an example of question answering using a model and a tokenizer. The proc
     ... ]
 
     >>> for question in questions:
-    ...     inputs = tokenizer.encode_plus(question, text, add_special_tokens=True, return_tensors="tf")
+    ...     inputs = tokenizer(question, text, add_special_tokens=True, return_tensors="tf")
     ...     input_ids = inputs["input_ids"].numpy()[0]
     ...
     ...     text_tokens = tokenizer.convert_ids_to_tokens(input_ids)

docs/source/tokenizer_summary.rst

@@ -0,0 +1,243 @@
+Tokenizer summary
+-----------------
+
+In this page, we will have a closer look at tokenization. As we saw in
+:doc:`the preprocessing tutorial <preprocessing>`, tokenizing a text is splitting it into words or subwords, which then
+are converted to ids. The second part is pretty straightforward, here we will focus on the first part. More
+specifically, we will look at the three main different kinds of tokenizers used in 🤗 Transformers:
+:ref:`Byte-Pair Encoding (BPE) <byte-pair-encoding>`, :ref:`WordPiece <wordpiece>` and
+:ref:`SentencePiece <sentencepiece>`, and provide examples of models using each of those.
+
+Note that on each model page, you can look at the documentation of the associated tokenizer to know which of those
+algorithms the pretrained model used. For instance, if we look at :class:`~transformers.BertTokenizer`, we can see it's
+using :ref:`WordPiece <wordpiece>`.
+
+Introduction to tokenization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Splitting a text in smaller chunks is a task that's harder than it looks, and there are multiple ways of doing it. For
+instance, let's look at the sentence "Don't you love 🤗 Transformers? We sure do." A first simple way of tokenizing
+this text is just to split it by spaces, which would give:
+
+::
+
+    ["Don't", "you", "love", "🤗", "Transformers?", "We", "sure", "do."]
+
+This is a nice first step, but if we look at the tokens "Transformers?" or "do.", we can see we can do better. Those
+will be different than the tokens "Transformers" and "do" for our model, so we should probably take the punctuation
+into account. This would give:
+
+::
+
+    ["Don", "'", "t", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]
+
+which is better already. One thing that is annoying though is how it dealt with "Don't". "Don't" stands for do not, so
+it should probably be better tokenized as ``["Do", "n't"]``. This is where things start getting more complicated, and
+part of the reason each kind of model has its own tokenizer class. Depending on the rules we apply to split our texts
+into tokens, we'll get different tokenized versions of the same text. And of course, a given pretrained model won't
+perform properly if you don't use the exact same rules as the persons who pretrained it.
+
+`spaCy <https://spacy.io/>`__ and `Moses <http://www.statmt.org/moses/?n=Development.GetStarted>`__ are two popular
+rule-based tokenizers. On the text above, they'd output something like:
+
+::
+
+    ["Do", "n't", "you", "love", "🤗", "Transformers", "?", "We", "sure", "do", "."]
+
+Space/punctuation-tokenization and rule-based tokenization are both examples of word tokenization, which is splitting a
+sentence into words. While it's the most intuitive way to separate texts in smaller chunks, it can have a problem when
+you have a huge corpus: it usually yields a very big vocabulary (the set of all unique tokens used).
+:doc:`Transformer XL <model_doc/transformerxl>` for instance uses space/punctuation-tokenization, and has a vocabulary
+size of 267,735!
+
+A huge vocabulary size means a huge embedding matrix at the start of the model, which will cause memory problems.
+TransformerXL deals with it by using a special kind of embeddings called adaptive embeddings, but in general,
+transformers model rarely have a vocabulary size greater than 50,000, especially if they are trained on a single
+language.
+
+So if tokenizing on words is unsatisfactory, we could go on the opposite direction and simply tokenize on characters.
+While it's very simple and would save a lot of memory, this doesn't allow the model to learn representations of texts
+as meaningful as when using a word tokenization, leading to a loss of performance. So to get the best of both worlds,
+all transformers models use a hybrid between word-level and character-level tokenization called subword tokenization.
+
+Subword tokenization
+^^^^^^^^^^^^^^^^^^^^
+
+Subword tokenization algorithms rely on the principle that most common words should be left as is, but rare words
+should be decomposed in meaningful subword units. For instance "annoyingly" might be considered a rare word and
+decomposed as "annoying" and "ly". This is especially useful in agglutinative languages such as Turkish, where you can
+form (almost) arbitrarily long complex words by stringing together some subwords.
+
+This allows the model to keep a reasonable vocabulary while still learning useful representations for common words or
+subwords. This also gives the ability to the model to process words it has never seen before, by decomposing them into
+subwords it knows. For instance, the base :class:`~transformers.BertTokenizer` will tokenize "I have a new GPU!" like
+this:
+
+::
+
+    >>> from transformers import BertTokenizer
+    >>> tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+    >>> tokenizer.tokenize("I have a new GPU!")
+    ['i', 'have', 'a', 'new', 'gp', '##u', '!']
+
+Since we are considering the uncased model, the sentence was lowercased first. Then all the words were present in the
+vocabulary of the tokenizer, except for "gpu", so the tokenizer split it in subwords it knows: "gp" and "##u". The "##"
+means that the rest of the token should be attached to the previous one, without space (for when we need to decode
+predictions and reverse the tokenization).
+
+Another example is when we use the base :class:`~transformers.XLNetTokenizer` to tokenize our previous text:
+
+::
+
+    >>> from transformers import XLNetTokenizer
+    >>> tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
+    >>> tokenizer.tokenize("Don't you love 🤗 Transformers? We sure do.")
+    ['▁Don', "'", 't', '▁you', '▁love', '▁', '🤗', '▁', 'Transform', 'ers', '?', '▁We', '▁sure', '▁do', '.']
+
+We'll get back to the meaning of those '▁' when we look at :ref:`SentencePiece <sentencepiece>` but you can see
+Transformers has been split into "Transform" and "ers".
+
+Let's now look at how the different subword tokenization algorithms work. Note that they all rely on some form of
+training which is usually done on the corpus the corresponding model will be trained on.
+
+.. _byte-pair-encoding:
+
+Byte-Pair Encoding
+~~~~~~~~~~~~~~~~~~
+
+Byte-Pair Encoding was introduced in `this paper <https://arxiv.org/abs/1508.07909>`__. It relies on a pretokenizer
+splitting the training data into words, which can be a simple space tokenization
+(:doc:`GPT-2 <model_doc/gpt2>` and :doc:`Roberta <model_doc/roberta>` uses this for instance) or a rule-based tokenizer
+(:doc:`XLM <model_doc/xlm>` use Moses for most languages, as does :doc:`FlauBERT <model_doc/flaubert>`),
+
+:doc:`GPT <model_doc/gpt>` uses Spacy and ftfy) and, counts the frequency of each word in the training corpus.
+
+It then begins from the list of all characters, and will learn merge rules to form a new token from two symbols in the
+vocabulary until it has learned a vocabulary of the desired size (this is a hyperparameter to pick).
+
+Let's say that after the pre-tokenization we have the following words (the number indicating the frequency of each
+word):
+
+::
+
+    ('hug', 10), ('pug', 5), ('pun', 12), ('bun', 4), ('hugs', 5)
+
+Then the base vocabulary is ['b', 'g', 'h', 'n', 'p', 's', 'u'] and all our words are first split by character:
+
+::
+
+    ('h' 'u' 'g', 10), ('p' 'u' 'g', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'u' 'g' 's', 5)
+
+We then take each pair of symbols and look at the most frequent. For instance 'hu' is present `10 + 5 = 15` times (10
+times in the 10 occurrences of 'hug', 5 times in the 5 occurrences of 'hugs'). The most frequent here is 'ug', present
+`10 + 5 + 2 + 5 = 22` times in total. So the first merge rule the tokenizer learns is to group all 'u' and 'g' together
+then it adds 'ug' to the vocabulary. Our corpus then becomes
+
+::
+
+    ('h' 'ug', 10), ('p' 'ug', 5), ('p' 'u' 'n', 12), ('b' 'u' 'n', 4), ('h' 'ug' 's', 5)
+
+and we continue by looking at the next most common pair of symbols. It's 'un', present 16 times, so we merge those two
+and add 'un' to the vocabulary. Then it's 'hug' (as 'h' + 'ug'), present 15 times, so we merge those two and add 'hug'
+to the vocabulary.
+
+At this stage, the vocabulary is ``['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']`` and our corpus is
+represented as
+
+::
+
+    ('hug', 10), ('p' 'ug', 5), ('p' 'un', 12), ('b' 'un', 4), ('hug' 's', 5)
+
+If we stop there, the tokenizer can apply the rules it learned to new words (as long as they don't contain characters that
+were not in the base vocabulary). For instance 'bug' would be tokenized as ``['b', 'ug']`` but mug would be tokenized as
+``['<unk>', 'ug']`` since the 'm' is not in the base vocabulary. This doesn't happen to letters in general (since the
+base corpus uses all of them), but to special characters like emojis.
+
+As we said before, the vocabulary size (which is the base vocabulary size + the number of merges) is a hyperparameter
+to choose. For instance :doc:`GPT <model_doc/gpt>` has a vocabulary size of 40,478 since they have 478 base characters
+and chose to stop the training of the tokenizer at 40,000 merges.
+
+Byte-level BPE
+^^^^^^^^^^^^^^
+
+To deal with the fact the base vocabulary needs to get all base characters, which can be quite big if one allows for
+all unicode characters, the
+`GPT-2 paper <https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf>`__
+introduces a clever trick, which is to use bytes as the base vocabulary (which gives a size of 256). With some
+additional rules to deal with punctuation, this manages to be able to tokenize every text without needing an unknown
+token. For instance, the :doc:`GPT-2 model <model_doc/gpt>` has a vocabulary size of 50,257, which corresponds to the
+256 bytes base tokens, a special end-of-text token and the symbols learned with 50,000 merges.
+
+.. _wordpiece:
+
+WordPiece
+=========
+
+WordPiece is the subword tokenization algorithm used for :doc:`BERT <model_doc/bert>` (as well as
+:doc:`DistilBERT <model_doc/distilbert>` and :doc:`Electra <model_doc/electra>`) and was outlined in
+`this paper <https://static.googleusercontent.com/media/research.google.com/ja//pubs/archive/37842.pdf>`__. It relies
+on the same base as BPE, which is to initialize the vocabulary to every character present in the corpus and
+progressively learn a given number of merge rules, the difference is that it doesn't choose the pair that is the most
+frequent but the one that will maximize the likelihood on the corpus once merged. 
+
+What does this mean? Well, in the previous example, it means we would only merge 'u' and 'g' if the probability of
+having 'ug' divided by the probability of having 'u' then 'g' is greater than for any other pair of symbols. It's
+subtly different from what BPE does in the sense that it evaluates what it "loses" by merging two symbols and makes
+sure it's `worth it`.
+
+.. _unigram:
+
+Unigram
+=======
+
+Unigram is a subword tokenization algorithm introduced in `this paper <https://arxiv.org/pdf/1804.10959.pdf>`__.
+Instead of starting with a group of base symbols and learning merges with some rule, like BPE or WordPiece, it starts
+from a large vocabulary (for instance, all pretokenized words and the most common substrings) that it will trim down
+progressively. It's not used directly for any of the pretrained models in the library, but it's used in conjunction
+with :ref:`SentencePiece <sentencepiece>`.
+
+More specifically, at a given step, unigram computes a loss from the corpus we have and the current vocabulary, then,
+for each subword, evaluate how much the loss would augment if the subword was removed from the vocabulary. It then
+sorts the subwords by this quantity (that represents how worse the loss becomes if the token is removed) and removes
+all the worst p tokens (for instance p could be 10% or 20%). It then repeats the process until the vocabulary has
+reached the desired size, always keeping the base characters (to be able to tokenize any word written with them, like
+BPE or WordPiece).
+
+Contrary to BPE and WordPiece that work out rules in a certain order that you can then apply in the same order when
+tokenizing new text, Unigram will have several ways of tokenizing a new text. For instance, if it ends up with the
+vocabulary
+
+::
+
+    ['b', 'g', 'h', 'n', 'p', 's', 'u', 'ug', 'un', 'hug']
+
+we had before, it could tokenize "hugs" as ``['hug', 's']``, ``['h', 'ug', 's']`` or ``['h', 'u', 'g', 's']``. So which
+one choose? On top of saving the vocabulary, the trained tokenizer will save the probability of each token in the
+training corpus. You can then give a probability to each tokenization (which is the product of the probabilities of the
+tokens forming it) and pick the most likely one (or if you want to apply some data augmentation, you could sample one
+of the tokenization according to their probabilities).
+
+Those probabilities are what are used to define the loss that trains the tokenizer: if our corpus consists of the
+words :math:`x_{1}, \dots, x_{N}` and if for the word :math:`x_{i}` we note :math:`S(x_{i})` the set of all possible
+tokenizations of :math:`x_{i}` (with the current vocabulary), then the loss is defined as
+
+.. math::
+    \mathcal{L} = -\sum_{i=1}^{N} \log \left ( \sum_{x \in S(x_{i})} p(x) \right )
+
+.. _sentencepiece:
+
+SentencePiece
+=============
+
+All the methods we have been looking at so far required some from of pretrokenization, which has a central problem: not
+all languages use spaces to separate words. This is a problem :doc:`XLM <model_doc/xlm>` solves by using specific
+pretokenizers for each of those languages (in this case, Chinese, Japanese and Thai). To solve this problem,
+SentencePiece (introduced in `this paper <https://arxiv.org/pdf/1808.06226.pdf>`__) treats the input as a raw stream,
+includes the space in the set of characters to use, then uses BPE or unigram to construct the appropriate vocabulary.
+
+That's why in the example we saw before using :class:`~transformers.XLNetTokenizer` (which uses SentencePiece), we had
+some '▁' characters, that represent spaces. Decoding a tokenized text is then super easy: we just have to concatenate
+all of them together and replace those '▁' by spaces.
+
+All transformers models in the library that use SentencePiece use it with unigram. Examples of models using it are
+:doc:`ALBERT <model_doc/albert>`, :doc:`XLNet <model_doc/xlnet>` or the :doc:`Marian framework <model_doc/marian>`.

docs/source/training.rst

@@ -39,7 +39,7 @@ of the specified model are used to initialize the model. The
 library also includes a number of task-specific final layers or 'heads' whose
 weights are instantiated randomly when not present in the specified
 pre-trained model. For example, instantiating a model with
-``BertForSequenceClassification.from_pretrained('bert-base-uncased', num_classes=2)``
+``BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)``
 will create a BERT model instance with encoder weights copied from the
 ``bert-base-uncased`` model and a randomly initialized sequence
 classification head on top of the encoder with an output size of 2. Models
@@ -77,7 +77,7 @@ other than bias and layer normalization terms:
     optimizer = AdamW(optimizer_grouped_parameters, lr=1e-5)
     
 Now we can set up a simple dummy training batch using
-:func:`~transformers.PreTrainedTokenizer.batch_encode_plus`. This returns a
+:func:`~transformers.PreTrainedTokenizer.__call__`. This returns a
 :func:`~transformers.BatchEncoding` instance which
 prepares everything we might need to pass to the model.
 
@@ -272,7 +272,7 @@ optimize.
 :func:`~transformers.Trainer` uses a built-in default function to collate
 batches and prepare them to be fed into the model. If needed, you can also
 use the ``data_collator`` argument to pass your own collator function which
-takes in the data in the format provides by your dataset and returns a
+takes in the data in the format provided by your dataset and returns a
 batch ready to be fed into the model. Note that
 :func:`~transformers.TFTrainer` expects the passed datasets to be dataset
 objects from ``tensorflow_datasets``.

examples/README.md

@@ -1,4 +1,4 @@
-## Examples
+# Examples
 
 Version 2.9 of 🤗 Transformers introduces a new [`Trainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer.py) class for PyTorch, and its equivalent [`TFTrainer`](https://github.com/huggingface/transformers/blob/master/src/transformers/trainer_tf.py) for TF 2.
 Running the examples requires PyTorch 1.3.1+ or TensorFlow 2.1+.
@@ -13,7 +13,7 @@ Here is the list of all our examples:
 This is still a work-in-progress – in particular documentation is still sparse – so please **contribute improvements/pull requests.**
 
 
-# The Big Table of Tasks
+## The Big Table of Tasks
 
 | Task | Example datasets | Trainer support | TFTrainer support | pytorch-lightning | Colab
 |---|---|:---:|:---:|:---:|:---:|
@@ -24,8 +24,8 @@ This is still a work-in-progress – in particular documentation is still sparse
 | [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering)     | SQuAD           | -  | ✅ | -  | -
 | [**`text-generation`**](https://github.com/huggingface/transformers/tree/master/examples/text-generation)           | -               | n/a | n/a | n/a | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)
 | [**`distillation`**](https://github.com/huggingface/transformers/tree/master/examples/distillation)       | All               | -  | -  | -  | -
-| [**`summarization`**](https://github.com/huggingface/transformers/tree/master/examples/summarization)     | CNN/Daily Mail    | -  | -  | -  | -
-| [**`translation`**](https://github.com/huggingface/transformers/tree/master/examples/translation)         | WMT               | -  | -  | -  | -
+| [**`summarization`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq)     | CNN/Daily Mail    | -  | -  | ✅  | -
+| [**`translation`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq)         | WMT               | -  | -  | ✅  | -
 | [**`bertology`**](https://github.com/huggingface/transformers/tree/master/examples/bertology)             | -                 | -  | -  | -  | -
 | [**`adversarial`**](https://github.com/huggingface/transformers/tree/master/examples/adversarial)         | HANS              | ✅ | -  | -  | -
 

examples/adversarial/utils_hans.py

@@ -298,12 +298,13 @@ def hans_convert_examples_to_features(
         if ex_index % 10000 == 0:
             logger.info("Writing example %d" % (ex_index))
 
-        inputs = tokenizer.encode_plus(
+        inputs = tokenizer(
             example.text_a,
             example.text_b,
             add_special_tokens=True,
             max_length=max_length,
-            pad_to_max_length=True,
+            padding="max_length",
+            truncation=True,
             return_overflowing_tokens=True,
         )
 

examples/benchmarking/run_benchmark_tf.py

@@ -13,15 +13,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" Benchmarking the library on inference and training in Tensorflow"""
+""" Benchmarking the library on inference and training in TensorFlow"""
 
-from transformers import HfArgumentParser, TensorflowBenchmark, TensorflowBenchmarkArguments
+from transformers import HfArgumentParser, TensorFlowBenchmark, TensorFlowBenchmarkArguments
 
 
 def main():
-    parser = HfArgumentParser(TensorflowBenchmarkArguments)
+    parser = HfArgumentParser(TensorFlowBenchmarkArguments)
     benchmark_args = parser.parse_args_into_dataclasses()[0]
-    benchmark = TensorflowBenchmark(args=benchmark_args)
+    benchmark = TensorFlowBenchmark(args=benchmark_args)
     benchmark.run()
 
 

examples/benchmarking/time_xla_1.csv

@@ -1,4 +0,0 @@
-model,batch_size,sequence_length,result
-aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2,8,512,0.2032
-aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2,64,512,1.5279
-aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2,256,512,6.1837

examples/bert-loses-patience/run_glue_with_pabee.py

@@ -226,8 +226,6 @@ def train(args, train_dataset, model, tokenizer):
                 if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
                     # Save model checkpoint
                     output_dir = os.path.join(args.output_dir, "checkpoint-{}".format(global_step))
-                    if not os.path.exists(output_dir):
-                        os.makedirs(output_dir)
                     model_to_save = (
                         model.module if hasattr(model, "module") else model
                     )  # Take care of distributed/parallel training
@@ -649,10 +647,6 @@ def main():
 
     # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
     if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
-        # Create output directory if needed
-        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
-            os.makedirs(args.output_dir)
-
         logger.info("Saving model checkpoint to %s", args.output_dir)
         # Save a trained model, configuration and tokenizer using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`

examples/contrib/mm-imdb/run_mmimdb.py

@@ -521,10 +521,6 @@ def main():
 
     # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
     if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
-        # Create output directory if needed
-        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
-            os.makedirs(args.output_dir)
-
         logger.info("Saving model checkpoint to %s", args.output_dir)
         # Save a trained model, configuration and tokenizer using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`

examples/contrib/run_swag.py

@@ -383,8 +383,6 @@ def train(args, train_dataset, model, tokenizer):
                 if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
                     # Save model checkpoint
                     output_dir = os.path.join(args.output_dir, "checkpoint-{}".format(global_step))
-                    if not os.path.exists(output_dir):
-                        os.makedirs(output_dir)
                     model_to_save = (
                         model.module if hasattr(model, "module") else model
                     )  # Take care of distributed/parallel training
@@ -651,10 +649,6 @@ def main():
 
     # Save the trained model and the tokenizer
     if args.local_rank == -1 or torch.distributed.get_rank() == 0:
-        # Create output directory if needed
-        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
-            os.makedirs(args.output_dir)
-
         logger.info("Saving model checkpoint to %s", args.output_dir)
         # Save a trained model, configuration and tokenizer using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`

examples/distillation/run_squad_w_distillation.py

@@ -809,10 +809,6 @@ def main():
 
     # Save the trained model and the tokenizer
     if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
-        # Create output directory if needed
-        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
-            os.makedirs(args.output_dir)
-
         logger.info("Saving model checkpoint to %s", args.output_dir)
         # Save a trained model, configuration and tokenizer using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`

examples/lightning_base.py

@@ -122,12 +122,9 @@ class BaseTransformer(pl.LightningModule):
         else:
             optimizer.step()
         optimizer.zero_grad()
-        self.lr_scheduler.step()
-
-    def get_tqdm_dict(self):
-        avg_loss = getattr(self.trainer, "avg_loss", 0.0)
-        tqdm_dict = {"loss": "{:.3f}".format(avg_loss), "lr": self.lr_scheduler.get_last_lr()[-1]}
-        return tqdm_dict
+        self.lr_scheduler.step()  # By default, PL will only step every epoch.
+        lrs = {f"lr_group_{i}": lr for i, lr in enumerate(self.lr_scheduler.get_lr())}
+        self.logger.log_metrics(lrs)
 
     def test_step(self, batch, batch_nb):
         return self.validation_step(batch, batch_nb)
@@ -202,7 +199,7 @@ class BaseTransformer(pl.LightningModule):
         parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
         parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
         parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
-        parser.add_argument("--warmup_steps", default=500, type=int, help="Linear warmup over warmup_steps.")
+        parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")
         parser.add_argument("--num_workers", default=4, type=int, help="kwarg passed to DataLoader")
         parser.add_argument(
             "--num_train_epochs", default=3, type=int, help="Total number of training epochs to perform."

examples/longform-qa/eli5_utils.py

@@ -193,12 +193,12 @@ def make_qa_retriever_model(model_name="google/bert_uncased_L-8_H-512_A-8", from
 def make_qa_retriever_batch(qa_list, tokenizer, max_len=64, device="cuda:0"):
     q_ls = [q for q, a in qa_list]
     a_ls = [a for q, a in qa_list]
-    q_toks = tokenizer.batch_encode_plus(q_ls, max_length=max_len, pad_to_max_length=True)
+    q_toks = tokenizer(q_ls, max_length=max_len, padding="max_length", truncation=True)
     q_ids, q_mask = (
         torch.LongTensor(q_toks["input_ids"]).to(device),
         torch.LongTensor(q_toks["attention_mask"]).to(device),
     )
-    a_toks = tokenizer.batch_encode_plus(a_ls, max_length=max_len, pad_to_max_length=True)
+    a_toks = tokenizer(a_ls, max_length=max_len, padding="max_length", truncation=True)
     a_ids, a_mask = (
         torch.LongTensor(a_toks["input_ids"]).to(device),
         torch.LongTensor(a_toks["attention_mask"]).to(device),
@@ -375,12 +375,12 @@ def make_qa_s2s_model(model_name="facebook/bart-large", from_file=None, device="
 def make_qa_s2s_batch(qa_list, tokenizer, max_len=64, max_a_len=360, device="cuda:0"):
     q_ls = [q for q, a in qa_list]
     a_ls = [a for q, a in qa_list]
-    q_toks = tokenizer.batch_encode_plus(q_ls, max_length=max_len, pad_to_max_length=True)
+    q_toks = tokenizer(q_ls, max_length=max_len, padding="max_length", truncation=True)
     q_ids, q_mask = (
         torch.LongTensor(q_toks["input_ids"]).to(device),
         torch.LongTensor(q_toks["attention_mask"]).to(device),
     )
-    a_toks = tokenizer.batch_encode_plus(a_ls, max_length=min(max_len, max_a_len), pad_to_max_length=True)
+    a_toks = tokenizer(a_ls, max_length=min(max_len, max_a_len), padding="max_length", truncation=True)
     a_ids, a_mask = (
         torch.LongTensor(a_toks["input_ids"]).to(device),
         torch.LongTensor(a_toks["attention_mask"]).to(device),
@@ -531,7 +531,7 @@ def qa_s2s_generate(
 # ELI5-trained retrieval model usage
 ###############
 def embed_passages_for_retrieval(passages, tokenizer, qa_embedder, max_length=128, device="cuda:0"):
-    a_toks = tokenizer.batch_encode_plus(passages, max_length=max_length, pad_to_max_length=True)
+    a_toks = tokenizer(passages, max_length=max_length, padding="max_length", truncation=True)
     a_ids, a_mask = (
         torch.LongTensor(a_toks["input_ids"]).to(device),
         torch.LongTensor(a_toks["attention_mask"]).to(device),
@@ -542,7 +542,7 @@ def embed_passages_for_retrieval(passages, tokenizer, qa_embedder, max_length=12
 
 
 def embed_questions_for_retrieval(q_ls, tokenizer, qa_embedder, device="cuda:0"):
-    q_toks = tokenizer.batch_encode_plus(q_ls, max_length=128, pad_to_max_length=True)
+    q_toks = tokenizer(q_ls, max_length=128, padding="max_length", truncation=True)
     q_ids, q_mask = (
         torch.LongTensor(q_toks["input_ids"]).to(device),
         torch.LongTensor(q_toks["attention_mask"]).to(device),

examples/movement-pruning/emmental/modeling_bert_masked.py

@@ -424,7 +424,7 @@ MASKED_BERT_INPUTS_DOCSTRING = r"""
 
             Indices can be obtained using :class:`transformers.BertTokenizer`.
             See :func:`transformers.PreTrainedTokenizer.encode` and
-            :func:`transformers.PreTrainedTokenizer.encode_plus` for details.
+            :func:`transformers.PreTrainedTokenizer.__call__` for details.
 
             `What are input IDs? <../glossary.html#input-ids>`__
         attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):

examples/movement-pruning/masked_run_glue.py

@@ -875,10 +875,6 @@ def main():
 
     # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
     if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
-        # Create output directory if needed
-        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
-            os.makedirs(args.output_dir)
-
         logger.info("Saving model checkpoint to %s", args.output_dir)
         # Save a trained model, configuration and tokenizer using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`

examples/movement-pruning/masked_run_squad.py

@@ -1059,10 +1059,6 @@ def main():
 
     # Save the trained model and the tokenizer
     if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
-        # Create output directory if needed
-        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
-            os.makedirs(args.output_dir)
-
         logger.info("Saving model checkpoint to %s", args.output_dir)
         # Save a trained model, configuration and tokenizer using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`

examples/multiple-choice/run_tf_multiple_choice.py

@@ -108,7 +108,10 @@ def main():
         level=logging.INFO,
     )
     logger.warning(
-        "device: %s, n_gpu: %s, 16-bits training: %s", training_args.device, training_args.n_gpu, training_args.fp16,
+        "device: %s, n_replicas: %s, 16-bits training: %s",
+        training_args.device,
+        training_args.n_replicas,
+        training_args.fp16,
     )
     logger.info("Training/evaluation parameters %s", training_args)
 

examples/multiple-choice/utils_multiple_choice.py

@@ -510,12 +510,13 @@ def convert_examples_to_features(
             else:
                 text_b = example.question + " " + ending
 
-            inputs = tokenizer.encode_plus(
+            inputs = tokenizer(
                 text_a,
                 text_b,
                 add_special_tokens=True,
                 max_length=max_length,
-                pad_to_max_length=True,
+                padding="max_length",
+                truncation=True,
                 return_overflowing_tokens=True,
             )
             if "num_truncated_tokens" in inputs and inputs["num_truncated_tokens"] > 0:

examples/question-answering/run_squad.py

@@ -240,8 +240,6 @@ def train(args, train_dataset, model, tokenizer):
                 # Save model checkpoint
                 if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
                     output_dir = os.path.join(args.output_dir, "checkpoint-{}".format(global_step))
-                    if not os.path.exists(output_dir):
-                        os.makedirs(output_dir)
                     # Take care of distributed/parallel training
                     model_to_save = model.module if hasattr(model, "module") else model
                     model_to_save.save_pretrained(output_dir)
@@ -768,10 +766,6 @@ def main():
 
     # Save the trained model and the tokenizer
     if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
-        # Create output directory if needed
-        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
-            os.makedirs(args.output_dir)
-
         logger.info("Saving model checkpoint to %s", args.output_dir)
         # Save a trained model, configuration and tokenizer using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`

examples/question-answering/run_tf_squad.py

@@ -137,9 +137,9 @@ def main():
         level=logging.INFO,
     )
     logger.info(
-        "n_gpu: %s, distributed training: %s, 16-bits training: %s",
-        training_args.n_gpu,
-        bool(training_args.n_gpu > 1),
+        "n_replicas: %s, distributed training: %s, 16-bits training: %s",
+        training_args.n_replicas,
+        bool(training_args.n_replicas > 1),
         training_args.fp16,
     )
     logger.info("Training/evaluation parameters %s", training_args)

examples/seq2seq/README.md

@@ -1,8 +1,11 @@
+## Sequence to Sequence
+
 This directory contains examples for finetuning and evaluating transformers on summarization and translation tasks.
 Summarization support is more mature than translation support.
 Please tag @sshleifer with any issues/unexpected behaviors, or send a PR!
 For `bertabs` instructions, see `bertabs/README.md`. 
 
+
 ### Data
 
 CNN/DailyMail data
@@ -37,13 +40,6 @@ export ENRO_DIR=${PWD}/wmt_en_ro
 If you are using your own data, it must be formatted as one directory with 6 files: train.source, train.target, val.source, val.target, test.source, test.target.  
 The `.source` files are the input, the `.target` files are the desired output.
 
-### Evaluation
-
-To create summaries for each article in dataset, run:
-```bash
-python run_eval.py <path_to_test.source> test_generations.txt <model-name>  --score_path rouge_scores.txt
-```
-The default batch size, 4, fits in 16GB GPU memory, but may need to be adjusted to fit your system.
 
 
 ### Summarization Finetuning
@@ -64,6 +60,7 @@ The following command should work on a 16GB GPU:
 
 Tips:
 - 1 epoch at batch size 1 for bart-large takes 24 hours and requires 13GB GPU RAM with fp16 on an NVIDIA-V100. 
+- since you need to run from `examples/seq2seq`, and likely need to modify code, it is easiest to fork, then clone transformers and run `pip install -e .` before you get started.   
 - try `bart-base`, `--freeze_encoder` or `--freeze_embeds` for faster training/larger batch size.  (3hr/epoch with bs=8, see the "xsum_shared_task" command below)
 - `fp16_opt_level=O1` (the default works best).
 - If you are finetuning on your own dataset, start from `distilbart-cnn-12-6` if you want long summaries and `distilbart-xsum-12-6` if you want short summaries.
@@ -74,7 +71,7 @@ Load it with `BartForConditionalGeneration.from_pretrained(f'{output_dir}/best_t
 - If you want to run experiments on improving the summarization finetuning process, try the XSUM Shared Task (below). It's faster to train than CNNDM because the summaries are shorter.
 - For CNN/DailyMail, the default `val_max_target_length` and `test_max_target_length` will truncate the ground truth labels, resulting in slightly higher rouge scores. To get accurate rouge scores, you should rerun calculate_rouge on the `{output_dir}/test_generations.txt` file saved by `trainer.test()`
 - `--max_target_length=60 --val_max_target_length=60 --test_max_target_length=100 ` is a reasonable setting for XSUM.
-- `wandb` can be used by specifying `--logger wandb_shared` or `--logger wandb`. It is useful for reproducibility. 
+- `wandb` can be used by specifying `--logger wandb`. It is useful for reproducibility. Specify the environment variable `WANDB_PROJECT='hf_xsum'` to do the XSUM shared task. 
 - This warning can be safely ignored: 
     > "Some weights of BartForConditionalGeneration were not initialized from the model checkpoint at facebook/bart-large-xsum and are newly initialized: ['final_logits_bias']"
 - Both finetuning and eval are 30% faster with `--fp16`. For that you need to [install apex](https://github.com/NVIDIA/apex#quick-start).
@@ -109,25 +106,71 @@ from transformers import AutoModelForSeq2SeqLM
 model = AutoModelForSeq2SeqLM.from_pretrained(f'{output_dir}/best_tfmr')
 ```
 
-
-### XSUM Shared Task
+#### XSUM Shared Task
 Compare XSUM results with others by using `--logger wandb_shared`. This requires `wandb` registration.
 
 Here is an example command, but you can do whatever you want. Hopefully this will make debugging and collaboration easier!
 ```bash
-./finetune.sh \
+WANDB_PROJECT='hf_xsum' ./finetune.sh \
     --data_dir $XSUM_DIR \
     --output_dir xsum_frozen_embs \
     --model_name_or_path facebook/bart-large \
-    --logger wandb_shared \
     --train_batch_size 16 --eval_batch_size 16 --freeze_embeds --freeze_encoder \
     --num_train_epochs 6 \
-    --max_target_length=60 --val_max_target_length=60 --test_max_target_length=100
+    --max_target_length=60 --val_max_target_length=60 --test_max_target_length=100 \
+    --logger wandb
 ```
 
 You can see your wandb logs [here](https://app.wandb.ai/sshleifer/hf_xsum?workspace=user-)
 
+### Evaluation Commands
+
+To create summaries for each article in dataset, we use `run_eval.py`, here are a few commands that run eval for different tasks and models.
+If 'translation' is in your task name, the computed metric will be BLEU. Otherwise, ROUGE will be used.
+
+For t5, you need to specify --task translation_{src}_to_{tgt} as follows:
+```bash
+export DATA_DIR=wmt_en_ro
+python run_eval.py t5_base \
+    $DATA_DIR/val.source t5_val_generations.txt \
+    --reference_path $DATA_DIR/val.target \
+    --score_path enro_bleu.json \
+    --task translation_en_to_ro \
+    --n_obs 100 \
+    --device cuda \
+    --fp16 \
+    --bs 32
+```
+
+This command works for MBART, although the BLEU score is suspiciously low.
+```bash
+export DATA_DIR=wmt_en_ro
+python run_eval.py facebook/mbart-large-en-ro $DATA_DIR/val.source mbart_val_generations.txt \
+    --reference_path $DATA_DIR/val.target \
+    --score_path enro_bleu.json \
+    --task translation \
+    --n_obs 100 \
+    --device cuda \
+    --fp16 \
+    --bs 32
+```
+
+Summarization (xsum will be very similar):
+```bash
+export DATA_DIR=cnn_dm
+python run_eval.py sshleifer/distilbart-cnn-12-6 $DATA_DIR/val.source dbart_val_generations.txt \
+    --reference_path $DATA_DIR/val.target \
+    --score_path cnn_rouge.json \
+    --task summarization \
+    --n_obs 100 \
+    --device cuda \
+    --fp16 \
+    --bs 32
+```
+
+
 ### DistilBART
+![DBART](https://huggingface.co/front/thumbnails/distilbart_large.png)
 
 For the CNN/DailyMail dataset, (relatively longer, more extractive summaries), we found a simple technique that works:
 you just copy alternating layers from `bart-large-cnn` and finetune more on the same data. 

examples/seq2seq/bertabs/run_summarization.py

@@ -30,7 +30,7 @@ Batch = namedtuple("Batch", ["document_names", "batch_size", "src", "segs", "mas
 
 def evaluate(args):
     tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
-    model = BertAbs.from_pretrained("bertabs-finetuned-cnndm")
+    model = BertAbs.from_pretrained("remi/bertabs-finetuned-extractive-abstractive-summarization")
     model.to(args.device)
     model.eval()
 

examples/seq2seq/distillation.py

@@ -92,8 +92,6 @@ class BartSummarizationDistiller(SummarizationModule):
         student = BartForConditionalGeneration(student_cfg)
         student, _ = init_student(student, teacher)
         save_dir = self.output_dir.joinpath("student")
-        save_dir.mkdir(exist_ok=True)
-
         self.copy_to_student(d_layers_to_copy, e_layers_to_copy, hparams, student, teacher)
         student.save_pretrained(save_dir)
         hparams.model_name_or_path = str(save_dir)

examples/seq2seq/finetune.py

@@ -3,6 +3,7 @@ import glob
 import logging
 import os
 import time
+import warnings
 from collections import defaultdict
 from pathlib import Path
 from typing import Dict, List, Tuple
@@ -216,6 +217,8 @@ class SummarizationModule(BaseTransformer):
         scheduler = get_linear_schedule_with_warmup(
             self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=t_total
         )
+        if max(scheduler.get_last_lr()) > 0:
+            warnings.warn("All learning rates are 0")
         self.lr_scheduler = scheduler
         return dataloader
 
@@ -295,8 +298,6 @@ def main(args, model=None) -> SummarizationModule:
             model: SummarizationModule = SummarizationModule(args)
         else:
             model: SummarizationModule = TranslationModule(args)
-
-    dataset = Path(args.data_dir).name
     if (
         args.logger == "default"
         or args.fast_dev_run
@@ -307,12 +308,12 @@ def main(args, model=None) -> SummarizationModule:
     elif args.logger == "wandb":
         from pytorch_lightning.loggers import WandbLogger
 
-        logger = WandbLogger(name=model.output_dir.name, project=dataset)
+        logger = WandbLogger(name=model.output_dir.name)
 
     elif args.logger == "wandb_shared":
         from pytorch_lightning.loggers import WandbLogger
 
-        logger = WandbLogger(name=model.output_dir.name, project=f"hf_{dataset}")
+        logger = WandbLogger(name=model.output_dir.name)
     trainer: pl.Trainer = generic_train(
         model,
         args,

examples/seq2seq/finetune_bart_tiny.sh

@@ -12,7 +12,7 @@ export OUTPUT_DIR=${CURRENT_DIR}/${OUTPUT_DIR_NAME}
 # Make output directory if it doesn't exist
 mkdir -p $OUTPUT_DIR
 
-# Add parent directory to python path to access lightning_base.py and utils.py
+# Add parent directory to python path to access lightning_base.py and testing_utils.py
 export PYTHONPATH="../":"${PYTHONPATH}"
 python finetune.py \
 --data_dir=cnn_tiny/ \

examples/seq2seq/run_eval.py

@@ -9,9 +9,9 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 
 
 try:
-    from .utils import calculate_rouge, use_task_specific_params, calculate_bleu_score
+    from .utils import calculate_rouge, use_task_specific_params, calculate_bleu_score, trim_batch
 except ImportError:
-    from utils import calculate_rouge, use_task_specific_params, calculate_bleu_score
+    from utils import calculate_rouge, use_task_specific_params, calculate_bleu_score, trim_batch
 
 DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
@@ -29,6 +29,7 @@ def generate_summaries_or_translations(
     batch_size: int = 8,
     device: str = DEFAULT_DEVICE,
     fp16=False,
+    task="summarization",
     **gen_kwargs,
 ) -> None:
     fout = Path(out_file).open("w", encoding="utf-8")
@@ -40,15 +41,16 @@ def generate_summaries_or_translations(
     tokenizer = AutoTokenizer.from_pretrained(model_name)
 
     # update config with summarization specific params
-    use_task_specific_params(model, "summarization")
+    use_task_specific_params(model, task)
 
     for batch in tqdm(list(chunks(examples, batch_size))):
         if "t5" in model_name:
             batch = [model.config.prefix + text for text in batch]
-        batch = tokenizer.batch_encode_plus(
-            batch, max_length=1024, return_tensors="pt", truncation=True, pad_to_max_length=True
-        ).to(device)
-        summaries = model.generate(**batch, **gen_kwargs)
+        batch = tokenizer(batch, max_length=1024, return_tensors="pt", truncation=True, padding="max_length").to(
+            device
+        )
+        input_ids, attention_mask = trim_batch(**batch, pad_token_id=tokenizer.pad_token_id)
+        summaries = model.generate(input_ids=input_ids, attention_mask=attention_mask, **gen_kwargs)
         dec = tokenizer.batch_decode(summaries, skip_special_tokens=True, clean_up_tokenization_spaces=False)
         for hypothesis in dec:
             fout.write(hypothesis + "\n")
@@ -57,30 +59,42 @@ def generate_summaries_or_translations(
 
 def run_generate():
     parser = argparse.ArgumentParser()
-    parser.add_argument("input_path", type=str, help="like cnn_dm/test.source")
-    parser.add_argument("output_path", type=str, help="where to save summaries")
     parser.add_argument("model_name", type=str, help="like facebook/bart-large-cnn,t5-base, etc.")
+    parser.add_argument("input_path", type=str, help="like cnn_dm/test.source")
+    parser.add_argument("save_path", type=str, help="where to save summaries")
+
     parser.add_argument("--reference_path", type=str, required=False, help="like cnn_dm/test_reference_summaries.txt")
     parser.add_argument("--score_path", type=str, required=False, help="where to save the rouge score in json format")
-    parser.add_argument("--metric", type=str, choices=["bleu", "rouge"], default="rouge")
     parser.add_argument("--device", type=str, required=False, default=DEFAULT_DEVICE, help="cuda, cuda:1, cpu etc.")
+    parser.add_argument("--task", type=str, default="summarization", help="typically translation or summarization")
     parser.add_argument("--bs", type=int, default=8, required=False, help="batch size")
+    parser.add_argument(
+        "--n_obs", type=int, default=-1, required=False, help="How many observations. Defaults to all."
+    )
     parser.add_argument("--fp16", action="store_true")
     args = parser.parse_args()
     examples = [" " + x.rstrip() if "t5" in args.model_name else x.rstrip() for x in open(args.input_path).readlines()]
+    if args.n_obs > 0:
+        examples = examples[: args.n_obs]
 
     generate_summaries_or_translations(
-        examples, args.output_path, args.model_name, batch_size=args.bs, device=args.device, fp16=args.fp16
+        examples,
+        args.save_path,
+        args.model_name,
+        batch_size=args.bs,
+        device=args.device,
+        fp16=args.fp16,
+        task=args.task,
     )
-
-    output_lns = [x.rstrip() for x in open(args.output_path).readlines()]
-    scores = {}
-    if args.reference_path is not None:
-        score_fn = {"bleu": calculate_bleu_score, "rouge": calculate_rouge}[args.metric]
-        reference_lns = [x.rstrip() for x in open(args.reference_path).readlines()]
-        scores: dict = score_fn(output_lns, reference_lns)
-        if args.score_path is not None:
-            json.dump(scores, open("score_path", "w+"))
+    if args.reference_path is None:
+        return
+    # Compute scores
+    score_fn = calculate_bleu_score if "translation" in args.task else calculate_rouge
+    output_lns = [x.rstrip() for x in open(args.save_path).readlines()]
+    reference_lns = [x.rstrip() for x in open(args.reference_path).readlines()][: len(output_lns)]
+    scores: dict = score_fn(output_lns, reference_lns)
+    if args.score_path is not None:
+        json.dump(scores, open(args.score_path, "w+"))
     return scores
 
 

examples/seq2seq/test_seq2seq_examples.py

@@ -12,6 +12,7 @@ import torch
 from torch.utils.data import DataLoader
 
 from transformers import AutoTokenizer
+from transformers.testing_utils import require_multigpu
 
 from .distillation import distill_main, evaluate_checkpoint
 from .finetune import main
@@ -107,7 +108,7 @@ class TestSummarizationDistiller(unittest.TestCase):
         logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks
         return cls
 
-    @unittest.skipUnless(torch.cuda.device_count() > 1, "skipping multiGPU test")
+    @require_multigpu
     def test_multigpu(self):
         updates = dict(no_teacher=True, freeze_encoder=True, gpus=2, sortish_sampler=False,)
         self._test_distiller_cli(updates)
@@ -193,13 +194,12 @@ class TestSummarizationDistiller(unittest.TestCase):
 
 @pytest.mark.parametrize(["model"], [pytest.param(T5_TINY), pytest.param(BART_TINY), pytest.param(MBART_TINY)])
 def test_run_eval_bart(model):
-    tmp = Path(tempfile.gettempdir()) / "utest_generations_bart_sum.hypo"
-
-    output_file_name = Path(tempfile.gettempdir()) / "utest_output_bart_sum.hypo"
+    input_file_name = Path(tempfile.mkdtemp()) / "utest_input.source"
+    output_file_name = input_file_name.parent / "utest_output.txt"
     assert not output_file_name.exists()
     articles = [" New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County."]
-    _dump_articles(tmp, articles)
-    testargs = ["run_eval.py", str(tmp), str(output_file_name), model]  # TODO: test score_path
+    _dump_articles(input_file_name, articles)
+    testargs = ["run_eval.py", model, str(input_file_name), str(output_file_name)]  # TODO: test score_path
     with patch.object(sys, "argv", testargs):
         run_generate()
         assert Path(output_file_name).exists()

examples/seq2seq/train_distilbart_cnn.sh

@@ -16,9 +16,9 @@ python finetune.py \
     --freeze_encoder --freeze_embeds --data_dir $CNN_DIR \
     --max_target_length 142 --val_max_target_length=142 \
     --train_batch_size=$BS --eval_batch_size=$BS --gradient_accumulation_steps=$GAS \
-    --data_dir $CNN_DIR \
     --model_name_or_path sshleifer/student_cnn_12_6 \
     --tokenizer_name facebook/bart-large \
+    --warmup_steps 500 \
     --output_dir distilbart-cnn-12-6 \
     $@
 

examples/seq2seq/train_distilbart_xsum.sh

@@ -16,5 +16,6 @@ python distillation.py \
   --alpha_hid=3. --length_penalty=0.5 \
   --train_batch_size=$BS --eval_batch_size=$BS --gradient_accumulation_steps=$GAS --num_train_epochs=6 \
   --tokenizer_name facebook/bart-large \
+  --warmup_steps 500 \
   --output_dir distilbart_xsum_12_6 \
   $@

examples/seq2seq/utils.py

@@ -41,12 +41,12 @@ def encode_file(
     assert lns, f"found empty file at {data_path}"
     examples = []
     for text in tqdm(lns, desc=f"Tokenizing {data_path.name}"):
-        tokenized = tokenizer.batch_encode_plus(
+        tokenized = tokenizer(
             [text],
             max_length=max_length,
-            pad_to_max_length=pad_to_max_length,
-            add_prefix_space=True,
+            padding="max_length" if pad_to_max_length else None,
             truncation=True,
+            add_prefix_space=True,
             return_tensors=return_tensors,
         )
         assert tokenized.input_ids.shape[1] == max_length
@@ -60,8 +60,9 @@ def lmap(f: Callable, x: Iterable) -> List:
     return list(map(f, x))
 
 
-def calculate_bleu_score(output_lns, refs_lns) -> dict:
-    return {"bleu": corpus_bleu(output_lns, [refs_lns]).score}
+def calculate_bleu_score(output_lns, refs_lns, **kwargs) -> dict:
+    """Uses sacrebleu's corpus_bleu implementation."""
+    return {"bleu": corpus_bleu(output_lns, [refs_lns], **kwargs).score}
 
 
 def trim_batch(

examples/text-classification/run_tf_glue.py

@@ -131,9 +131,9 @@ def main():
         level=logging.INFO,
     )
     logger.info(
-        "n_gpu: %s, distributed training: %s, 16-bits training: %s",
-        training_args.n_gpu,
-        bool(training_args.n_gpu > 1),
+        "n_replicas: %s, distributed training: %s, 16-bits training: %s",
+        training_args.n_replicas,
+        bool(training_args.n_replicas > 1),
         training_args.fp16,
     )
     logger.info("Training/evaluation parameters %s", training_args)

examples/text-classification/run_xnli.py

@@ -573,10 +573,6 @@ def main():
 
     # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
     if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
-        # Create output directory if needed
-        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
-            os.makedirs(args.output_dir)
-
         logger.info("Saving model checkpoint to %s", args.output_dir)
         # Save a trained model, configuration and tokenizer using `save_pretrained()`.
         # They can then be reloaded using `from_pretrained()`

examples/text-generation/run_generation.py

@@ -214,8 +214,14 @@ def main():
     if requires_preprocessing:
         prepare_input = PREPROCESSING_FUNCTIONS.get(args.model_type)
         preprocessed_prompt_text = prepare_input(args, model, tokenizer, prompt_text)
+
+        if model.__class__.__name__ in ["TransfoXLLMHeadModel"]:
+            tokenizer_kwargs = {"add_space_before_punct_symbol": True}
+        else:
+            tokenizer_kwargs = {}
+
         encoded_prompt = tokenizer.encode(
-            preprocessed_prompt_text, add_special_tokens=False, return_tensors="pt", add_space_before_punct_symbol=True
+            preprocessed_prompt_text, add_special_tokens=False, return_tensors="pt", **tokenizer_kwargs
         )
     else:
         encoded_prompt = tokenizer.encode(prompt_text, add_special_tokens=False, return_tensors="pt")

examples/token-classification/run_ner.py

@@ -75,7 +75,8 @@ class DataTrainingArguments:
         metadata={"help": "The input data dir. Should contain the .txt files for a CoNLL-2003-formatted task."}
     )
     labels: Optional[str] = field(
-        metadata={"help": "Path to a file containing all labels. If not specified, CoNLL-2003 labels are used."}
+        default=None,
+        metadata={"help": "Path to a file containing all labels. If not specified, CoNLL-2003 labels are used."},
     )
     max_seq_length: int = field(
         default=128,

examples/token-classification/run_tf_ner.py

@@ -109,9 +109,9 @@ def main():
         level=logging.INFO,
     )
     logger.info(
-        "n_gpu: %s, distributed training: %s, 16-bits training: %s",
-        training_args.n_gpu,
-        bool(training_args.n_gpu > 1),
+        "n_replicas: %s, distributed training: %s, 16-bits training: %s",
+        training_args.n_replicas,
+        bool(training_args.n_replicas > 1),
         training_args.fp16,
     )
     logger.info("Training/evaluation parameters %s", training_args)

model_cards/MoseliMotsoehli/TswanaBert/README.md

@@ -0,0 +1,74 @@
+---
+language: setswana
+---
+
+# TswanaBert
+Pretrained model on the Tswana language using a masked language modeling (MLM) objective.
+
+## Model Description.
+TswanaBERT is a transformer model pre-trained on a corpus of Setswana in a self-supervised fashion by masking part of the input words and training to predict the masks by using byte-level tokens.
+
+## Intended uses & limitations
+The model can  be used for either masked language modeling or next word prediction. It can also be fine-tuned on a specific down-stream NLP application. 
+
+#### How to use
+
+```python
+>>> from transformers import pipeline
+>>> from transformers import AutoTokenizer, AutoModelWithLMHead
+
+>>> tokenizer = AutoTokenizer.from_pretrained("MoseliMotsoehli/TswanaBert")
+>>> model = AutoModelWithLMHead.from_pretrained("MoseliMotsoehli/TswanaBert")
+>>> unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer)
+>>> unmasker("Ntshopotse <mask> e godile.")
+
+[{'score': 0.32749542593955994,
+  'sequence': '<s>Ntshopotse setse e godile.</s>',
+  'token': 538,
+  'token_str': 'Ġsetse'},
+ {'score': 0.060260992497205734,
+  'sequence': '<s>Ntshopotse le e godile.</s>',
+  'token': 270,
+  'token_str': 'Ġle'},
+ {'score': 0.058460816740989685,
+  'sequence': '<s>Ntshopotse bone e godile.</s>',
+  'token': 364,
+  'token_str': 'Ġbone'},
+ {'score': 0.05694682151079178,
+  'sequence': '<s>Ntshopotse ga e godile.</s>',
+  'token': 298,
+  'token_str': 'Ġga'},
+ {'score': 0.0565204992890358,
+  'sequence': '<s>Ntshopotse, e godile.</s>',
+  'token': 16,
+  'token_str': ','}]
+```
+
+#### Limitations and bias
+The model is trained on a relatively small collection of setwana, mostly from news articles and creative writtings, and so is not representative enough of the language as yet.
+
+## Training data
+
+1. The largest portion of this dataset (10k)  sentences of text, comes from the [Leipzig Corpora Collection](https://wortschatz.uni-leipzig.de/en/download)
+
+2. I Then added SABC news headlines collected by Marivate Vukosi, & Sefara Tshephisho, (2020)  that is generously made available on [zenoodo](http://doi.org/10.5281/zenodo.3668495 ). This added 185 tswana sentences to my corpus. 
+
+3. I went on to add 300 more sentences by scrapping following news sites and blogs that mosty originate in Botswana. I actively continue to expand the dataset.
+
+* http://setswana.blogspot.com/
+* https://omniglot.com/writing/tswana.php
+* http://www.dailynews.gov.bw/
+* http://www.mmegi.bw/index.php
+* https://tsena.co.bw
+* http://www.botswana.co.za/Cultural_Issues-travel/botswana-country-guide-en-route.html
+* https://www.poemhunter.com/poem/2013-setswana/
+https://www.poemhunter.com/poem/ngwana-wa-mosetsana/
+ 
+
+### BibTeX entry and citation info
+
+```bibtex
+@inproceedings{author = {Moseli Motsoehli},
+  year={2020}
+}
+```

model_cards/SparkBeyond/roberta-large-sts-b/README.md

@@ -40,7 +40,7 @@ def roberta_similarity_batches(to_predict):
   return similarity_scores
 
 def similarity_roberta(model, tokenizer, sent_pairs):
-  batch_token = tokenizer.batch_encode_plus(sent_pairs, pad_to_max_length=True, max_length=500)
+  batch_token = tokenizer(sent_pairs, padding='max_length', truncation=True, max_length=500)
   res = model(torch.tensor(batch_token['input_ids']).cuda(), attention_mask=torch.tensor(batch_token["attention_mask"]).cuda())  
   return res
 

model_cards/a-ware/bart-squadv2/README.md

@@ -60,7 +60,7 @@ tokenizer = BartTokenizer.from_pretrained('a-ware/bart-squadv2')
 model = BartForQuestionAnswering.from_pretrained('a-ware/bart-squadv2')
 
 question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
-encoding = tokenizer.encode_plus(question, text, return_tensors='pt')
+encoding = tokenizer(question, text, return_tensors='pt')
 input_ids = encoding['input_ids']
 attention_mask = encoding['attention_mask']
 

model_cards/a-ware/xlmroberta-squadv2/README.md

@@ -43,7 +43,7 @@ tokenizer = XLMRobertaTokenizer.from_pretrained('a-ware/xlmroberta-squadv2')
 model = XLMRobertaForQuestionAnswering.from_pretrained('a-ware/xlmroberta-squadv2')
 
 question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
-encoding = tokenizer.encode_plus(question, text, return_tensors='pt')
+encoding = tokenizer(question, text, return_tensors='pt')
 input_ids = encoding['input_ids']
 attention_mask = encoding['attention_mask']
 

model_cards/asafaya/bert-large-arabic/README.md

@@ -0,0 +1,59 @@
+---
+language: arabic
+---
+
+# Arabic BERT Large Model
+
+Pretrained BERT Large language model for Arabic
+
+_If you use this model in your work, please cite this paper (to appear in 2020):_
+
+```
+@inproceedings{
+  title={KUISAIL at SemEval-2020 Task 12: BERT-CNN for Offensive Speech Identification in Social Media},
+  author={Safaya, Ali and Abdullatif, Moutasem and Yuret, Deniz},
+  booktitle={Proceedings of the International Workshop on Semantic Evaluation (SemEval)},
+  year={2020}
+}
+```
+
+## Pretraining Corpus
+
+`arabic-bert-large` model was pretrained on ~8.2 Billion words:
+
+- Arabic version of [OSCAR](https://traces1.inria.fr/oscar/) - filtered from [Common Crawl](http://commoncrawl.org/)
+- Recent dump of Arabic [Wikipedia](https://dumps.wikimedia.org/backup-index.html)
+
+and other Arabic resources which sum up to ~95GB of text.
+
+__Notes on training data:__
+
+- Our final version of corpus contains some non-Arabic words inlines, which we did not remove from sentences since that would affect some tasks like NER.
+- Although non-Arabic characters were lowered as a preprocessing step, since Arabic characters does not have upper or lower case, there is no cased and uncased version of the model.
+- The corpus and vocabulary set are not restricted to Modern Standard Arabic, they contain some dialectical Arabic too.
+
+## Pretraining details
+
+- This model was trained using Google BERT's github [repository](https://github.com/google-research/bert) on a single TPU v3-8 provided for free from [TFRC](https://www.tensorflow.org/tfrc).
+- Our pretraining procedure follows training settings of bert with some changes: trained for 3M training steps with batchsize of 128, instead of 1M with batchsize of 256.
+
+## Load Pretrained Model
+
+You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this:  
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-large-arabic")
+model = AutoModel.from_pretrained("asafaya/bert-large-arabic")
+```
+
+## Results
+
+For further details on the models performance or any other queries, please refer to [Arabic-BERT](https://github.com/alisafaya/Arabic-BERT)
+
+## Acknowledgement
+
+Thanks to Google for providing free TPU for the training process and for Huggingface for hosting this model on their servers 😊
+
+

model_cards/asafaya/bert-medium-arabic/README.md

@@ -0,0 +1,59 @@
+---
+language: arabic
+---
+
+# Arabic BERT Medium Model
+
+Pretrained BERT Medium language model for Arabic
+
+_If you use this model in your work, please cite this paper (to appear in 2020):_
+
+```
+@inproceedings{
+  title={KUISAIL at SemEval-2020 Task 12: BERT-CNN for Offensive Speech Identification in Social Media},
+  author={Safaya, Ali and Abdullatif, Moutasem and Yuret, Deniz},
+  booktitle={Proceedings of the International Workshop on Semantic Evaluation (SemEval)},
+  year={2020}
+}
+```
+
+## Pretraining Corpus
+
+`arabic-bert-medium` model was pretrained on ~8.2 Billion words:
+
+- Arabic version of [OSCAR](https://traces1.inria.fr/oscar/) - filtered from [Common Crawl](http://commoncrawl.org/)
+- Recent dump of Arabic [Wikipedia](https://dumps.wikimedia.org/backup-index.html)
+
+and other Arabic resources which sum up to ~95GB of text.
+
+__Notes on training data:__
+
+- Our final version of corpus contains some non-Arabic words inlines, which we did not remove from sentences since that would affect some tasks like NER.
+- Although non-Arabic characters were lowered as a preprocessing step, since Arabic characters does not have upper or lower case, there is no cased and uncased version of the model.
+- The corpus and vocabulary set are not restricted to Modern Standard Arabic, they contain some dialectical Arabic too.
+
+## Pretraining details
+
+- This model was trained using Google BERT's github [repository](https://github.com/google-research/bert) on a single TPU v3-8 provided for free from [TFRC](https://www.tensorflow.org/tfrc).
+- Our pretraining procedure follows training settings of bert with some changes: trained for 3M training steps with batchsize of 128, instead of 1M with batchsize of 256.
+
+## Load Pretrained Model
+
+You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this:  
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-medium-arabic")
+model = AutoModel.from_pretrained("asafaya/bert-medium-arabic")
+```
+
+## Results
+
+For further details on the models performance or any other queries, please refer to [Arabic-BERT](https://github.com/alisafaya/Arabic-BERT)
+
+## Acknowledgement
+
+Thanks to Google for providing free TPU for the training process and for Huggingface for hosting this model on their servers 😊
+
+

model_cards/asafaya/bert-mini-arabic/README.md

@@ -0,0 +1,61 @@
+---
+language: arabic
+datasets:
+- oscar
+- wikipedia
+---
+
+# Arabic BERT Mini Model
+
+Pretrained BERT Mini language model for Arabic
+
+_If you use this model in your work, please cite this paper (to appear in 2020):_
+
+```
+@inproceedings{
+  title={KUISAIL at SemEval-2020 Task 12: BERT-CNN for Offensive Speech Identification in Social Media},
+  author={Safaya, Ali and Abdullatif, Moutasem and Yuret, Deniz},
+  booktitle={Proceedings of the International Workshop on Semantic Evaluation (SemEval)},
+  year={2020}
+}
+```
+
+## Pretraining Corpus
+
+`arabic-bert-mini` model was pretrained on ~8.2 Billion words:
+
+- Arabic version of [OSCAR](https://traces1.inria.fr/oscar/) - filtered from [Common Crawl](http://commoncrawl.org/)
+- Recent dump of Arabic [Wikipedia](https://dumps.wikimedia.org/backup-index.html)
+
+and other Arabic resources which sum up to ~95GB of text.
+
+__Notes on training data:__
+
+- Our final version of corpus contains some non-Arabic words inlines, which we did not remove from sentences since that would affect some tasks like NER.
+- Although non-Arabic characters were lowered as a preprocessing step, since Arabic characters does not have upper or lower case, there is no cased and uncased version of the model.
+- The corpus and vocabulary set are not restricted to Modern Standard Arabic, they contain some dialectical Arabic too.
+
+## Pretraining details
+
+- This model was trained using Google BERT's github [repository](https://github.com/google-research/bert) on a single TPU v3-8 provided for free from [TFRC](https://www.tensorflow.org/tfrc).
+- Our pretraining procedure follows training settings of bert with some changes: trained for 3M training steps with batchsize of 128, instead of 1M with batchsize of 256.
+
+## Load Pretrained Model
+
+You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this:  
+
+```python
+from transformers import AutoTokenizer, AutoModel
+
+tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-mini-arabic")
+model = AutoModel.from_pretrained("asafaya/bert-mini-arabic")
+```
+
+## Results
+
+For further details on the models performance or any other queries, please refer to [Arabic-BERT](https://github.com/alisafaya/Arabic-BERT)
+
+## Acknowledgement
+
+Thanks to Google for providing free TPU for the training process and for Huggingface for hosting this model on their servers 😊
+

model_cards/bert-base-cased-README.md

@@ -1,10 +1,230 @@
 ---
+language: english
 tags:
 - exbert
-
 license: apache-2.0
+datasets:
+- bookcorpus
+- wikipedia
 ---
 
+# BERT base model (cased)
+
+Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in
+[this paper](https://arxiv.org/abs/1810.04805) and first released in
+[this repository](https://github.com/google-research/bert). This model is case-sensitive: it makes a difference between
+english and English.
+
+Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by
+the Hugging Face team.
+
+## Model description
+
+BERT is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it
+was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of
+publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it
+was pretrained with two objectives:
+
+- Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run
+  the entire masked sentence through the model and has to predict the masked words. This is different from traditional
+  recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like
+  GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the
+  sentence.
+- Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes
+  they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to
+  predict if the two sentences were following each other or not.
+
+This way, the model learns an inner representation of the English language that can then be used to extract features
+useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard
+classifier using the features produced by the BERT model as inputs.
+
+## Intended uses & limitations
+
+You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to
+be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for
+fine-tuned versions on a task that interests you.
+
+Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked)
+to make decisions, such as sequence classification, token classification or question answering. For tasks such as text
+generation you should look at model like GPT2.
+
+### How to use
+
+You can use this model directly with a pipeline for masked language modeling:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='bert-base-cased')
+>>> unmasker("Hello I'm a [MASK] model.")
+
+[{'sequence': "[CLS] Hello I'm a fashion model. [SEP]",
+  'score': 0.09019174426794052,
+  'token': 4633,
+  'token_str': 'fashion'},
+ {'sequence': "[CLS] Hello I'm a new model. [SEP]",
+  'score': 0.06349995732307434,
+  'token': 1207,
+  'token_str': 'new'},
+ {'sequence': "[CLS] Hello I'm a male model. [SEP]",
+  'score': 0.06228214129805565,
+  'token': 2581,
+  'token_str': 'male'},
+ {'sequence': "[CLS] Hello I'm a professional model. [SEP]",
+  'score': 0.0441727414727211,
+  'token': 1848,
+  'token_str': 'professional'},
+ {'sequence': "[CLS] Hello I'm a super model. [SEP]",
+  'score': 0.03326151892542839,
+  'token': 7688,
+  'token_str': 'super'}]
+```
+
+Here is how to use this model to get the features of a given text in PyTorch:
+
+```python
+from transformers import BertTokenizer, TFBertModel
+tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
+model = TFBertModel.from_pretrained("bert-base-cased")
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='pt')
+output = model(**encoded_input)
+```
+
+and in TensorFlow:
+
+```python
+from transformers import BertTokenizer, BertModel
+tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
+model = BertModel.from_pretrained("bert-base-cased")
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='tf')
+output = model(encoded_input)
+```
+
+### Limitations and bias
+
+Even if the training data used for this model could be characterized as fairly neutral, this model can have biased
+predictions:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='bert-base-cased')
+>>> unmasker("The man worked as a [MASK].")
+
+[{'sequence': '[CLS] The man worked as a lawyer. [SEP]',
+  'score': 0.04804691672325134,
+  'token': 4545,
+  'token_str': 'lawyer'},
+ {'sequence': '[CLS] The man worked as a waiter. [SEP]',
+  'score': 0.037494491785764694,
+  'token': 17989,
+  'token_str': 'waiter'},
+ {'sequence': '[CLS] The man worked as a cop. [SEP]',
+  'score': 0.035512614995241165,
+  'token': 9947,
+  'token_str': 'cop'},
+ {'sequence': '[CLS] The man worked as a detective. [SEP]',
+  'score': 0.031271643936634064,
+  'token': 9140,
+  'token_str': 'detective'},
+ {'sequence': '[CLS] The man worked as a doctor. [SEP]',
+  'score': 0.027423162013292313,
+  'token': 3995,
+  'token_str': 'doctor'}]
+
+>>> unmasker("The woman worked as a [MASK].")
+
+[{'sequence': '[CLS] The woman worked as a nurse. [SEP]',
+  'score': 0.16927455365657806,
+  'token': 7439,
+  'token_str': 'nurse'},
+ {'sequence': '[CLS] The woman worked as a waitress. [SEP]',
+  'score': 0.1501094549894333,
+  'token': 15098,
+  'token_str': 'waitress'},
+ {'sequence': '[CLS] The woman worked as a maid. [SEP]',
+  'score': 0.05600163713097572,
+  'token': 13487,
+  'token_str': 'maid'},
+ {'sequence': '[CLS] The woman worked as a housekeeper. [SEP]',
+  'score': 0.04838843643665314,
+  'token': 26458,
+  'token_str': 'housekeeper'},
+ {'sequence': '[CLS] The woman worked as a cook. [SEP]',
+  'score': 0.029980547726154327,
+  'token': 9834,
+  'token_str': 'cook'}]
+```
+
+This bias will also affect all fine-tuned versions of this model.
+
+## Training data
+
+The BERT model was pretrained on [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038
+unpublished books and [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and
+headers).
+
+## Training procedure
+
+### Preprocessing
+
+The texts are tokenized using WordPiece and a vocabulary size of 30,000. The inputs of the model are then of the form:
+
+```
+[CLS] Sentence A [SEP] Sentence B [SEP]
+```
+
+With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in
+the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a
+consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two
+"sentences" has a combined length of less than 512 tokens.
+
+The details of the masking procedure for each sentence are the following:
+- 15% of the tokens are masked.
+- In 80% of the cases, the masked tokens are replaced by `[MASK]`.
+- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace.
+- In the 10% remaining cases, the masked tokens are left as is.
+
+### Pretraining
+
+The model was trained on 4 cloud TPUs in Pod configuration (16 TPU chips total) for one million steps with a batch size
+of 256. The sequence length was limited to 128 tokens for 90% of the steps and 512 for the remaining 10%. The optimizer
+used is Adam with a learning rate of 1e-4, \\(\beta_{1} = 0.9\\) and \\(\beta_{2} = 0.999\\), a weight decay of 0.01,
+learning rate warmup for 10,000 steps and linear decay of the learning rate after.
+
+## Evaluation results
+
+When fine-tuned on downstream tasks, this model achieves the following results:
+
+Glue test results:
+
+| Task | MNLI-(m/mm) | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  | Average |
+|:----:|:-----------:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|:-------:|
+|      | 84.6/83.4   | 71.2 | 90.5 | 93.5  | 52.1 | 85.8  | 88.9 | 66.4 | 79.6    |
+
+
+### BibTeX entry and citation info
+
+```bibtex
+@article{DBLP:journals/corr/abs-1810-04805,
+  author    = {Jacob Devlin and
+               Ming{-}Wei Chang and
+               Kenton Lee and
+               Kristina Toutanova},
+  title     = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
+               Understanding},
+  journal   = {CoRR},
+  volume    = {abs/1810.04805},
+  year      = {2018},
+  url       = {http://arxiv.org/abs/1810.04805},
+  archivePrefix = {arXiv},
+  eprint    = {1810.04805},
+  timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},
+  biburl    = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+```
+
 <a href="https://huggingface.co/exbert/?model=bert-base-cased">
 	<img width="300px" src="https://hf-dinosaur.huggingface.co/exbert/button.png">
 </a>

model_cards/bert-base-multilingual-cased-README.md

@@ -1,5 +1,152 @@
 ---
 language: multilingual
-
 license: apache-2.0
+datasets:
+- wikipedia
 ---
+
+# BERT multilingual base model (uncased)
+
+Pretrained model on the top 104 languages with the largest Wikipedia using a masked language modeling (MLM) objective.
+It was introduced in [this paper](https://arxiv.org/abs/1810.04805) and first released in
+[this repository](https://github.com/google-research/bert). This model is case sensitive: it makes a difference
+between english and English.
+
+Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by
+the Hugging Face team.
+
+## Model description
+
+BERT is a transformers model pretrained on a large corpus of multilingual data in a self-supervised fashion. This means
+it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of
+publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it
+was pretrained with two objectives:
+
+- Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run
+  the entire masked sentence through the model and has to predict the masked words. This is different from traditional
+  recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like
+  GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the
+  sentence.
+- Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes
+  they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to
+  predict if the two sentences were following each other or not.
+
+This way, the model learns an inner representation of the languages in the training set that can then be used to
+extract features useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a
+standard classifier using the features produced by the BERT model as inputs.
+
+## Intended uses & limitations
+
+You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to
+be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for
+fine-tuned versions on a task that interests you.
+
+Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked)
+to make decisions, such as sequence classification, token classification or question answering. For tasks such as text
+generation you should look at model like GPT2.
+
+### How to use
+
+You can use this model directly with a pipeline for masked language modeling:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='bert-base-multilingual-cased')
+>>> unmasker("Hello I'm a [MASK] model.")
+
+[{'sequence': "[CLS] Hello I'm a model model. [SEP]",
+  'score': 0.10182085633277893,
+  'token': 13192,
+  'token_str': 'model'},
+ {'sequence': "[CLS] Hello I'm a world model. [SEP]",
+  'score': 0.052126359194517136,
+  'token': 11356,
+  'token_str': 'world'},
+ {'sequence': "[CLS] Hello I'm a data model. [SEP]",
+  'score': 0.048930276185274124,
+  'token': 11165,
+  'token_str': 'data'},
+ {'sequence': "[CLS] Hello I'm a flight model. [SEP]",
+  'score': 0.02036019042134285,
+  'token': 23578,
+  'token_str': 'flight'},
+ {'sequence': "[CLS] Hello I'm a business model. [SEP]",
+  'score': 0.020079681649804115,
+  'token': 14155,
+  'token_str': 'business'}]
+```
+
+Here is how to use this model to get the features of a given text in PyTorch:
+
+```python
+from transformers import BertTokenizer, BertModel
+tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
+model = BertModel.from_pretrained("bert-base-multilingual-cased")
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='pt')
+output = model(**encoded_input)
+```
+
+and in TensorFlow:
+
+```python
+from transformers import BertTokenizer, TFBertModel
+tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
+model = TFBertModel.from_pretrained("bert-base-multilingual-cased")
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='tf')
+output = model(encoded_input)
+```
+
+## Training data
+
+The BERT model was pretrained on the 104 languages with the largest Wikipedias. You can find the complete list
+[here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages).
+
+## Training procedure
+
+### Preprocessing
+
+The texts are lowercased and tokenized using WordPiece and a shared vocabulary size of 110,000. The languages with a
+larger Wikipedia are under-sampled and the ones with lower resources are oversampled. For languages like Chinese,
+Japanese Kanji and Korean Hanja that don't have space, a CJK Unicode block is added around every character. 
+
+The inputs of the model are then of the form:
+
+```
+[CLS] Sentence A [SEP] Sentence B [SEP]
+```
+
+With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in
+the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a
+consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two
+"sentences" has a combined length of less than 512 tokens.
+
+The details of the masking procedure for each sentence are the following:
+- 15% of the tokens are masked.
+- In 80% of the cases, the masked tokens are replaced by `[MASK]`.
+- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace.
+- In the 10% remaining cases, the masked tokens are left as is.
+
+
+### BibTeX entry and citation info
+
+```bibtex
+@article{DBLP:journals/corr/abs-1810-04805,
+  author    = {Jacob Devlin and
+               Ming{-}Wei Chang and
+               Kenton Lee and
+               Kristina Toutanova},
+  title     = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
+               Understanding},
+  journal   = {CoRR},
+  volume    = {abs/1810.04805},
+  year      = {2018},
+  url       = {http://arxiv.org/abs/1810.04805},
+  archivePrefix = {arXiv},
+  eprint    = {1810.04805},
+  timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},
+  biburl    = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+```

model_cards/bert-base-multilingual-uncased-README.md

@@ -1,5 +1,209 @@
 ---
-language: multilingual
-
+language: english
 license: apache-2.0
+datasets:
+- wikipedia
 ---
+
+# BERT multilingual base model (uncased)
+
+Pretrained model on the top 102 languages with the largest Wikipedia using a masked language modeling (MLM) objective.
+It was introduced in [this paper](https://arxiv.org/abs/1810.04805) and first released in
+[this repository](https://github.com/google-research/bert). This model is uncased: it does not make a difference
+between english and English.
+
+Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by
+the Hugging Face team.
+
+## Model description
+
+BERT is a transformers model pretrained on a large corpus of multilingual data in a self-supervised fashion. This means
+it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of
+publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it
+was pretrained with two objectives:
+
+- Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run
+  the entire masked sentence through the model and has to predict the masked words. This is different from traditional
+  recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like
+  GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the
+  sentence.
+- Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes
+  they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to
+  predict if the two sentences were following each other or not.
+
+This way, the model learns an inner representation of the languages in the training set that can then be used to
+extract features useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a
+standard classifier using the features produced by the BERT model as inputs.
+
+## Intended uses & limitations
+
+You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to
+be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for
+fine-tuned versions on a task that interests you.
+
+Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked)
+to make decisions, such as sequence classification, token classification or question answering. For tasks such as text
+generation you should look at model like GPT2.
+
+### How to use
+
+You can use this model directly with a pipeline for masked language modeling:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='bert-base-multilingual-uncased')
+>>> unmasker("Hello I'm a [MASK] model.")
+
+[{'sequence': "[CLS] hello i'm a top model. [SEP]",
+  'score': 0.1507750153541565,
+  'token': 11397,
+  'token_str': 'top'},
+ {'sequence': "[CLS] hello i'm a fashion model. [SEP]",
+  'score': 0.13075384497642517,
+  'token': 23589,
+  'token_str': 'fashion'},
+ {'sequence': "[CLS] hello i'm a good model. [SEP]",
+  'score': 0.036272723227739334,
+  'token': 12050,
+  'token_str': 'good'},
+ {'sequence': "[CLS] hello i'm a new model. [SEP]",
+  'score': 0.035954564809799194,
+  'token': 10246,
+  'token_str': 'new'},
+ {'sequence': "[CLS] hello i'm a great model. [SEP]",
+  'score': 0.028643041849136353,
+  'token': 11838,
+  'token_str': 'great'}]
+```
+
+Here is how to use this model to get the features of a given text in PyTorch:
+
+```python
+from transformers import BertTokenizer, BertModel
+tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased')
+model = BertModel.from_pretrained("bert-base-multilingual-uncased")
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='pt')
+output = model(**encoded_input)
+```
+
+and in TensorFlow:
+
+```python
+from transformers import BertTokenizer, TFBertModel
+tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased')
+model = TFBertModel.from_pretrained("bert-base-multilingual-uncased")
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='tf')
+output = model(encoded_input)
+```
+
+### Limitations and bias
+
+Even if the training data used for this model could be characterized as fairly neutral, this model can have biased
+predictions:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='bert-base-multilingual-uncased')
+>>> unmasker("The man worked as a [MASK].")
+
+[{'sequence': '[CLS] the man worked as a teacher. [SEP]',
+  'score': 0.07943806052207947,
+  'token': 21733,
+  'token_str': 'teacher'},
+ {'sequence': '[CLS] the man worked as a lawyer. [SEP]',
+  'score': 0.0629938617348671,
+  'token': 34249,
+  'token_str': 'lawyer'},
+ {'sequence': '[CLS] the man worked as a farmer. [SEP]',
+  'score': 0.03367974981665611,
+  'token': 36799,
+  'token_str': 'farmer'},
+ {'sequence': '[CLS] the man worked as a journalist. [SEP]',
+  'score': 0.03172805905342102,
+  'token': 19477,
+  'token_str': 'journalist'},
+ {'sequence': '[CLS] the man worked as a carpenter. [SEP]',
+  'score': 0.031021825969219208,
+  'token': 33241,
+  'token_str': 'carpenter'}]
+
+>>> unmasker("The Black woman worked as a [MASK].")
+
+[{'sequence': '[CLS] the black woman worked as a nurse. [SEP]',
+  'score': 0.07045423984527588,
+  'token': 52428,
+  'token_str': 'nurse'},
+ {'sequence': '[CLS] the black woman worked as a teacher. [SEP]',
+  'score': 0.05178029090166092,
+  'token': 21733,
+  'token_str': 'teacher'},
+ {'sequence': '[CLS] the black woman worked as a lawyer. [SEP]',
+  'score': 0.032601192593574524,
+  'token': 34249,
+  'token_str': 'lawyer'},
+ {'sequence': '[CLS] the black woman worked as a slave. [SEP]',
+  'score': 0.030507225543260574,
+  'token': 31173,
+  'token_str': 'slave'},
+ {'sequence': '[CLS] the black woman worked as a woman. [SEP]',
+  'score': 0.027691684663295746,
+  'token': 14050,
+  'token_str': 'woman'}]
+```
+
+This bias will also affect all fine-tuned versions of this model.
+
+## Training data
+
+The BERT model was pretrained on the 102 languages with the largest Wikipedias. You can find the complete list
+[here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages).
+
+## Training procedure
+
+### Preprocessing
+
+The texts are lowercased and tokenized using WordPiece and a shared vocabulary size of 110,000. The languages with a
+larger Wikipedia are under-sampled and the ones with lower resources are oversampled. For languages like Chinese,
+Japanese Kanji and Korean Hanja that don't have space, a CJK Unicode block is added around every character. 
+
+The inputs of the model are then of the form:
+
+```
+[CLS] Sentence A [SEP] Sentence B [SEP]
+```
+
+With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in
+the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a
+consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two
+"sentences" has a combined length of less than 512 tokens.
+
+The details of the masking procedure for each sentence are the following:
+- 15% of the tokens are masked.
+- In 80% of the cases, the masked tokens are replaced by `[MASK]`.
+- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace.
+- In the 10% remaining cases, the masked tokens are left as is.
+
+
+### BibTeX entry and citation info
+
+```bibtex
+@article{DBLP:journals/corr/abs-1810-04805,
+  author    = {Jacob Devlin and
+               Ming{-}Wei Chang and
+               Kenton Lee and
+               Kristina Toutanova},
+  title     = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
+               Understanding},
+  journal   = {CoRR},
+  volume    = {abs/1810.04805},
+  year      = {2018},
+  url       = {http://arxiv.org/abs/1810.04805},
+  archivePrefix = {arXiv},
+  eprint    = {1810.04805},
+  timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},
+  biburl    = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+```

model_cards/bert-base-uncased-README.md

@@ -93,9 +93,9 @@ output = model(**encoded_input)
 and in TensorFlow:
 
 ```python
-from transformers import BertTokenizer, BertModel
+from transformers import BertTokenizer, TFBertModel
 tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
-model = BertModel.from_pretrained("bert-base-uncased")
+model = TFBertModel.from_pretrained("bert-base-uncased")
 text = "Replace me by any text you'd like."
 encoded_input = tokenizer(text, return_tensors='tf')
 output = model(encoded_input)

model_cards/chrisliu298/arxiv_ai_gpt2/README.md

@@ -0,0 +1,65 @@
+---
+language: "en"
+tags:
+- gpt2
+- arxiv
+- transformers
+datasets:
+- https://github.com/staeiou/arxiv_archive/tree/v1.0.1
+---
+
+# ArXiv AI GPT-2
+
+## Model description
+
+This GPT-2 (774M) model is capable of generating abstracts given paper titles. It was trained using all research paper titles and abstracts under artificial intelligence (AI), machine learning (LG), computation and language (CL), and computer vision and pattern recognition (CV) on arXiv.
+
+## Intended uses & limitations
+
+#### How to use
+
+To generate paper abstracts, use the provided `generate.py` [here](https://gist.github.com/chrisliu298/ccb8144888eace069da64ad3e6472d64). This is very similar to the HuggingFace's `run_generation.py` [here](https://github.com/huggingface/transformers/tree/master/examples/text-generation). You can simply replace the text with with your own model path (line 89) and change the input string to your paper title (line 127). If you want to use your own script, make sure to prepend `<|startoftext|> ` at the front and append ` <|sep|>` at the end of the paper title.
+
+## Training data
+I selected a subset of the [arXiv Archive](https://github.com/staeiou/arxiv_archive) dataset (Geiger, 2019) as the training and evaluation data to fine-tune GPT-2. The original arXiv Archive dataset contains a full archive of metadata about papers on arxiv.org, from the start of the site in 1993 to the end of 2019. Our subset includes all the paper titles (query) and abstracts (context) under the Artificial Intelligence (cs.AI), Machine Learning (cs.LG), Computation and Language (cs.CL), and Computer Vision and Pattern Recognition (cs.CV) categories. I provide the information  of the sub-dataset and the distribution of the training and evaluation dataset as follows.
+
+
+|   Splits   |   Count    | Percentage (%) | BPE Token Count |
+| :--------: | :--------: | :------------: | :-------------: |
+|   Train    |   90,000   |     90.11      |   20,834,012    |
+| Validation |    4,940   |      4.95      |    1,195,056    |
+|    Test    |    4,940   |      4.95      |    1,218,754    |
+| **Total**  | **99,880** |    **100**     | **23,247,822**  |
+
+The original dataset is in the format of a tab-separated value, so we wrote a simple preprocessing script to convert it into a text file format, which is the input file type (a document) of the GPT-2 model. An example of a paper’s title and its abstract is shown below.
+
+```text
+<|startoftext|> Some paper title <|sep|> Some paper abstract <|endoftext|>
+```
+
+Because there are a lot of cross-domain papers in the dataset, I deduplicate the dataset using the arXiv ID, which is unique for every paper. I sort the paper by submission date, by doing so, one can examine GPT-2’s ability to use learned terminologies when it is prompted with paper titles from the “future.”
+
+
+## Training procedure
+
+I used block size = 512, batch size = 1, gradidnet accumulation = 1, learning rate = 1e-5, epochs = 5, and everything else follows the default model configuration.
+
+## Eval results
+
+The resulting GPT-2 large model's perplexity score on the test set is **14.9413**.
+
+## Reference
+
+```bibtex
+@dataset{r_stuart_geiger_2019_2533436,
+    author= {R. Stuart Geiger},
+    title={{ArXiV Archive: A tidy and complete archive of metadata for papers on arxiv.org, 1993-2019}},
+    month=jan,
+    year= 2019,
+    publisher={Zenodo},
+    version= {v1.0.1},
+    doi={10.5281/zenodo.2533436},
+    url={https://doi.org/10.5281/zenodo.2533436}
+}
+```
+

model_cards/distilbert-base-uncased-README.md

@@ -1,10 +1,218 @@
 ---
+language: english
 tags:
 - exbert
-
 license: apache-2.0
+datasets:
+- bookcorpus
+- wikipedia
 ---
 
+# DistilBERT base model (uncased)
+
+This model is a distilled version of the [BERT base mode](https://huggingface.co/distilbert-base-uncased). It was
+introduced in [this paper](https://arxiv.org/abs/1910.01108). The code for the distillation process can be found
+[here](https://github.com/huggingface/transformers/tree/master/examples/distillation). This model is uncased: it does
+not make a difference between english and English.
+
+## Model description
+
+DistilBERT is a transformers model, smaller and faster than BERT, which was pretrained on the same corpus in a
+self-supervised fashion, using the BERT base model as a teacher. This means it was pretrained on the raw texts only,
+with no humans labelling them in any way (which is why it can use lots of publicly available data) with an automatic
+process to generate inputs and labels from those texts using the BERT base model. More precisely, it was pretrained
+with three objectives:
+
+- Distillation loss: the model was trained to return the same probabilities as the BERT base model.
+- Masked language modeling (MLM): this is part of the original training loss of the BERT base model. When taking a
+  sentence, the model randomly masks 15% of the words in the input then run the entire masked sentence through the
+  model and has to predict the masked words. This is different from traditional recurrent neural networks (RNNs) that
+  usually see the words one after the other, or from autoregressive models like GPT which internally mask the future
+  tokens. It allows the model to learn a bidirectional representation of the sentence.
+- Cosine embedding loss: the model was also trained to generate hidden states as close as possible as the BERT base
+  model.
+
+This way, the model learns the same inner representation of the English language than its teacher model, while being
+faster for inference or downstream tasks.
+
+## Intended uses & limitations
+
+You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to
+be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=distilbert) to look for
+fine-tuned versions on a task that interests you.
+
+Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked)
+to make decisions, such as sequence classification, token classification or question answering. For tasks such as text
+generation you should look at model like GPT2.
+
+### How to use
+
+You can use this model directly with a pipeline for masked language modeling:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='distilbert-base-uncased')
+>>> unmasker("Hello I'm a [MASK] model.")
+
+[{'sequence': "[CLS] hello i'm a role model. [SEP]",
+  'score': 0.05292855575680733,
+  'token': 2535,
+  'token_str': 'role'},
+ {'sequence': "[CLS] hello i'm a fashion model. [SEP]",
+  'score': 0.03968575969338417,
+  'token': 4827,
+  'token_str': 'fashion'},
+ {'sequence': "[CLS] hello i'm a business model. [SEP]",
+  'score': 0.034743521362543106,
+  'token': 2449,
+  'token_str': 'business'},
+ {'sequence': "[CLS] hello i'm a model model. [SEP]",
+  'score': 0.03462274372577667,
+  'token': 2944,
+  'token_str': 'model'},
+ {'sequence': "[CLS] hello i'm a modeling model. [SEP]",
+  'score': 0.018145186826586723,
+  'token': 11643,
+  'token_str': 'modeling'}]
+```
+
+Here is how to use this model to get the features of a given text in PyTorch:
+
+```python
+from transformers import DistilBertTokenizer, DistilBertModel
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+model = DistilBertModel.from_pretrained("distilbert-base-uncased")
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='pt')
+output = model(**encoded_input)
+```
+
+and in TensorFlow:
+
+```python
+from transformers import DistilBertTokenizer, TFDistilBertModel
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+model = TFDistilBertModel.from_pretrained("distilbert-base-uncased")
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='tf')
+output = model(encoded_input)
+```
+
+### Limitations and bias
+
+Even if the training data used for this model could be characterized as fairly neutral, this model can have biased
+predictions. It also inherits some of
+[the bias of its teacher model](https://huggingface.co/bert-base-uncased#limitations-and-bias). 
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='distilbert-base-uncased')
+>>> unmasker("The White man worked as a [MASK].")
+
+[{'sequence': '[CLS] the white man worked as a blacksmith. [SEP]',
+  'score': 0.1235365942120552,
+  'token': 20987,
+  'token_str': 'blacksmith'},
+ {'sequence': '[CLS] the white man worked as a carpenter. [SEP]',
+  'score': 0.10142576694488525,
+  'token': 10533,
+  'token_str': 'carpenter'},
+ {'sequence': '[CLS] the white man worked as a farmer. [SEP]',
+  'score': 0.04985016956925392,
+  'token': 7500,
+  'token_str': 'farmer'},
+ {'sequence': '[CLS] the white man worked as a miner. [SEP]',
+  'score': 0.03932540491223335,
+  'token': 18594,
+  'token_str': 'miner'},
+ {'sequence': '[CLS] the white man worked as a butcher. [SEP]',
+  'score': 0.03351764753460884,
+  'token': 14998,
+  'token_str': 'butcher'}]
+
+>>> unmasker("The Black woman worked as a [MASK].")
+
+[{'sequence': '[CLS] the black woman worked as a waitress. [SEP]',
+  'score': 0.13283951580524445,
+  'token': 13877,
+  'token_str': 'waitress'},
+ {'sequence': '[CLS] the black woman worked as a nurse. [SEP]',
+  'score': 0.12586183845996857,
+  'token': 6821,
+  'token_str': 'nurse'},
+ {'sequence': '[CLS] the black woman worked as a maid. [SEP]',
+  'score': 0.11708822101354599,
+  'token': 10850,
+  'token_str': 'maid'},
+ {'sequence': '[CLS] the black woman worked as a prostitute. [SEP]',
+  'score': 0.11499975621700287,
+  'token': 19215,
+  'token_str': 'prostitute'},
+ {'sequence': '[CLS] the black woman worked as a housekeeper. [SEP]',
+  'score': 0.04722772538661957,
+  'token': 22583,
+  'token_str': 'housekeeper'}]
+```
+
+This bias will also affect all fine-tuned versions of this model.
+
+## Training data
+
+DistilBERT pretrained on the same data as BERT, which is [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset
+consisting of 11,038 unpublished books and [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia)
+(excluding lists, tables and headers).
+
+## Training procedure
+
+### Preprocessing
+
+The texts are lowercased and tokenized using WordPiece and a vocabulary size of 30,000. The inputs of the model are
+then of the form:
+
+```
+[CLS] Sentence A [SEP] Sentence B [SEP]
+```
+
+With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in
+the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a
+consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two
+"sentences" has a combined length of less than 512 tokens.
+
+The details of the masking procedure for each sentence are the following:
+- 15% of the tokens are masked.
+- In 80% of the cases, the masked tokens are replaced by `[MASK]`.
+- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace.
+- In the 10% remaining cases, the masked tokens are left as is.
+
+### Pretraining
+
+The model was trained on 8 16 GB V100 for 90 hours. See the
+[training code](https://github.com/huggingface/transformers/tree/master/examples/distillation) for all hyperparameters
+details.
+
+## Evaluation results
+
+When fine-tuned on downstream tasks, this model achieves the following results:
+
+Glue test results:
+
+| Task | MNLI | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  | Average |
+|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|:-------:|
+|      | 82.2 | 88.5 | 89.2 | 91.3  | 51.3 | 85.8  | 87.5 | 59.9 | 77.0    |
+
+
+### BibTeX entry and citation info
+
+```bibtex
+@article{Sanh2019DistilBERTAD,
+  title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter},
+  author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf},
+  journal={ArXiv},
+  year={2019},
+  volume={abs/1910.01108}
+}
+```
+
 <a href="https://huggingface.co/exbert/?model=distilbert-base-uncased">
 	<img width="300px" src="https://hf-dinosaur.huggingface.co/exbert/button.png">
 </a>

model_cards/google/reformer-enwik8/README.md

@@ -14,7 +14,7 @@ Therefore, this model does not need a tokenizer. The following function can inst
 import torch
 
 # Encoding
-def encode(list_of_strings, pad_to_max_length=True, pad_token_id=0):
+def encode(list_of_strings, pad_token_id=0):
     max_length = max([len(string) for string in list_of_strings])
 
     # create emtpy tensors

model_cards/gpt2-README.md

@@ -13,8 +13,9 @@ Pretrained model on English language using a causal language modeling (CLM) obje
 [this paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf)
 and first released at [this page](https://openai.com/blog/better-language-models/).
 
-Disclaimer: The team releasing GPT-2 did not write a model card for this model so this model card has been written by
-the Hugging Face team.
+Disclaimer: The team releasing GPT-2 also wrote a
+[model card](https://github.com/openai/gpt-2/blob/master/model_card.md) for their model. Content from this model card
+has been written by the Hugging Face team to complete the information they provided and give specific examples of bias.
 
 ## Model description
 
@@ -79,7 +80,19 @@ output = model(encoded_input)
 ### Limitations and bias
 
 The training data used for this model has not been released as a dataset one can browse. We know it contains a lot of
-unfiltered from the internet, which is far from neutral. Therefore, the model can have biased predictions:
+unfiltered content from the internet, which is far from neutral. As the openAI team themselves point out in their
+[model card](https://github.com/openai/gpt-2/blob/master/model_card.md#out-of-scope-use-cases):
+
+> Because large-scale language models like GPT-2 do not distinguish fact from fiction, we don’t support use-cases
+> that require the generated text to be true.
+>
+> Additionally, language models like GPT-2 reflect the biases inherent to the systems they were trained on, so we do
+> not recommend that they be deployed into systems that interact with humans > unless the deployers first carry out a
+> study of biases relevant to the intended use-case. We found no statistically significant difference in gender, race,
+> and religious bias probes between 774M and 1.5B, implying all versions of GPT-2 should be approached with similar
+> levels of caution around use cases that are sensitive to biases around human attributes.
+
+Here's an example of how the model can have biased predictions:
 
 ```python
 >>> from transformers import pipeline, set_seed
@@ -110,7 +123,8 @@ This bias will also affect all fine-tuned versions of this model.
 The OpenAI team wanted to train this model on a corpus as large as possible. To build it, they scraped all the web
 pages from outbound links on Reddit which received at least 3 karma. Note that all Wikipedia pages were removed from
 this dataset, so the model was not trained on any part of Wikipedia. The resulting dataset (called WebText) weights
-40GB of texts but has not been publicly released.
+40GB of texts but has not been publicly released. You can find a list of the top 1,000 domains present in WebText
+[here](https://github.com/openai/gpt-2/blob/master/domains.txt).
 
 ## Training procedure
 

model_cards/krevas/finance-koelectra-base-discriminator/README.md

@@ -0,0 +1,45 @@
+---
+language: korean
+---
+
+# 📈 Financial Korean ELECTRA model
+
+Pretrained ELECTRA Language Model for Korean (`finance-koelectra-base-discriminator`)
+
+> ELECTRA is a new method for self-supervised language representation learning. It can be used to
+> pre-train transformer networks using relatively little compute. ELECTRA models are trained to
+> distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to
+> the discriminator of a GAN.
+
+More details about ELECTRA can be found in the [ICLR paper](https://openreview.net/forum?id=r1xMH1BtvB)
+or in the [official ELECTRA repository](https://github.com/google-research/electra) on GitHub.
+
+## Stats
+
+The current version of the model is trained on a financial news data of Naver news.
+
+The final training corpus has a size of 25GB and 2.3B tokens.
+
+This model was trained a cased model on a TITAN RTX for 500k steps.
+
+## Usage
+
+```python
+from transformers import ElectraForPreTraining, ElectraTokenizer
+import torch
+discriminator = ElectraForPreTraining.from_pretrained("krevas/finance-koelectra-base-discriminator")
+tokenizer = ElectraTokenizer.from_pretrained("krevas/finance-koelectra-base-discriminator")
+sentence = "내일 해당 종목이 대폭 상승할 것이다"
+fake_sentence = "내일 해당 종목이 맛있게 상승할 것이다"
+fake_tokens = tokenizer.tokenize(fake_sentence)
+fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt")
+discriminator_outputs = discriminator(fake_inputs)
+predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2)
+[print("%7s" % token, end="") for token in fake_tokens]
+[print("%7s" % int(prediction), end="") for prediction in predictions.tolist()[1:-1]]
+print("fake token : %s" % fake_tokens[predictions.tolist()[1:-1].index(1)])
+```
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/krevas).

model_cards/krevas/finance-koelectra-base-generator/README.md

@@ -0,0 +1,41 @@
+---
+language: korean
+---
+
+# 📈 Financial Korean ELECTRA model
+
+Pretrained ELECTRA Language Model for Korean (`finance-koelectra-base-generator`)
+
+> ELECTRA is a new method for self-supervised language representation learning. It can be used to
+> pre-train transformer networks using relatively little compute. ELECTRA models are trained to
+> distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to
+> the discriminator of a GAN.
+
+More details about ELECTRA can be found in the [ICLR paper](https://openreview.net/forum?id=r1xMH1BtvB)
+or in the [official ELECTRA repository](https://github.com/google-research/electra) on GitHub.
+
+## Stats
+
+The current version of the model is trained on a financial news data of Naver news.
+
+The final training corpus has a size of 25GB and 2.3B tokens.
+
+This model was trained a cased model on a TITAN RTX for 500k steps.
+
+## Usage
+
+```python
+from transformers import pipeline
+
+fill_mask = pipeline(
+            "fill-mask",
+            model="krevas/finance-koelectra-base-generator",
+            tokenizer="krevas/finance-koelectra-base-generator"
+            )
+
+print(fill_mask(f"내일 해당 종목이 대폭 {fill_mask.tokenizer.mask_token}할 것이다."))
+```
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/krevas).

model_cards/krevas/finance-koelectra-small-discriminator/README.md

@@ -0,0 +1,45 @@
+---
+language: korean
+---
+
+# 📈 Financial Korean ELECTRA model
+
+Pretrained ELECTRA Language Model for Korean (`finance-koelectra-small-discriminator`)
+
+> ELECTRA is a new method for self-supervised language representation learning. It can be used to
+> pre-train transformer networks using relatively little compute. ELECTRA models are trained to
+> distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to
+> the discriminator of a GAN.
+
+More details about ELECTRA can be found in the [ICLR paper](https://openreview.net/forum?id=r1xMH1BtvB)
+or in the [official ELECTRA repository](https://github.com/google-research/electra) on GitHub.
+
+## Stats
+
+The current version of the model is trained on a financial news data of Naver news.
+
+The final training corpus has a size of 25GB and 2.3B tokens.
+
+This model was trained a cased model on a TITAN RTX for 500k steps.
+
+## Usage
+
+```python
+from transformers import ElectraForPreTraining, ElectraTokenizer
+import torch
+discriminator = ElectraForPreTraining.from_pretrained("krevas/finance-koelectra-small-discriminator")
+tokenizer = ElectraTokenizer.from_pretrained("krevas/finance-koelectra-small-discriminator")
+sentence = "내일 해당 종목이 대폭 상승할 것이다"
+fake_sentence = "내일 해당 종목이 맛있게 상승할 것이다"
+fake_tokens = tokenizer.tokenize(fake_sentence)
+fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt")
+discriminator_outputs = discriminator(fake_inputs)
+predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2)
+[print("%7s" % token, end="") for token in fake_tokens]
+[print("%7s" % int(prediction), end="") for prediction in predictions.tolist()[1:-1]]
+print("fake token : %s" % fake_tokens[predictions.tolist()[1:-1].index(1)])
+```
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/krevas).

model_cards/krevas/finance-koelectra-small-generator/README.md

@@ -0,0 +1,41 @@
+---
+language: korean
+---
+
+# 📈 Financial Korean ELECTRA model
+
+Pretrained ELECTRA Language Model for Korean (`finance-koelectra-small-generator`)
+
+> ELECTRA is a new method for self-supervised language representation learning. It can be used to
+> pre-train transformer networks using relatively little compute. ELECTRA models are trained to
+> distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to
+> the discriminator of a GAN.
+
+More details about ELECTRA can be found in the [ICLR paper](https://openreview.net/forum?id=r1xMH1BtvB)
+or in the [official ELECTRA repository](https://github.com/google-research/electra) on GitHub.
+
+## Stats
+
+The current version of the model is trained on a financial news data of Naver news.
+
+The final training corpus has a size of 25GB and 2.3B tokens.
+
+This model was trained a cased model on a TITAN RTX for 500k steps.
+
+## Usage
+
+```python
+from transformers import pipeline
+
+fill_mask = pipeline(
+            "fill-mask",
+            model="krevas/finance-koelectra-small-generator",
+            tokenizer="krevas/finance-koelectra-small-generator"
+            )
+
+print(fill_mask(f"내일 해당 종목이 대폭 {fill_mask.tokenizer.mask_token}할 것이다."))
+```
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/krevas).

model_cards/lserinol/bert-turkish-question-answering/README.md

@@ -43,7 +43,7 @@ questions = [
 ]
 
 for question in questions:
-    inputs = tokenizer.encode_plus(question, text, add_special_tokens=True, return_tensors="pt")
+    inputs = tokenizer(question, text, add_special_tokens=True, return_tensors="pt")
     input_ids = inputs["input_ids"].tolist()[0]
 
     text_tokens = tokenizer.convert_ids_to_tokens(input_ids)

model_cards/mrm8488/electra-base-finetuned-squadv1/README.md

@@ -0,0 +1,85 @@
+---
+language: english
+---
+
+# Electra base ⚡ + SQuAD v1 ❓
+
+[Electra-base-discriminator](https://huggingface.co/google/electra-base-discriminator) fine-tuned on [SQUAD v1.1 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) for **Q&A** downstream task.
+
+## Details of the downstream task (Q&A) - Model 🧠
+
+**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset.
+
+
+## Details of the downstream task (Q&A) - Dataset 📚
+
+**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.
+SQuAD v1.1 contains **100,000+** question-answer pairs on **500+** articles.
+
+## Model training 🏋️‍
+
+The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
+
+```bash
+python transformers/examples/question-answering/run_squad.py \
+  --model_type electra \
+  --model_name_or_path 'google/electra-base-discriminator' \
+  --do_eval \
+  --do_train \
+  --do_lower_case \
+  --train_file '/content/dataset/train-v1.1.json' \
+  --predict_file '/content/dataset/dev-v1.1.json' \
+  --per_gpu_train_batch_size 16 \
+  --learning_rate 3e-5 \
+  --num_train_epochs 10 \
+  --max_seq_length 384 \
+  --doc_stride 128 \
+  --output_dir '/content/output' \
+  --overwrite_output_dir \
+  --save_steps 1000
+```
+
+## Test set Results 🧾
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **83.03** |
+| **F1** | **90.77** |
+| **Size**| **+ 400 MB** |
+
+Very good metrics for such a "small" model!
+
+```json
+{
+'exact': 83.03689687795648,
+'f1': 90.77486052446231,
+'total': 10570,
+'HasAns_exact': 83.03689687795648,
+'HasAns_f1': 90.77486052446231,
+'HasAns_total': 10570,
+'best_exact': 83.03689687795648,
+'best_exact_thresh': 0.0,
+'best_f1': 90.77486052446231,
+'best_f1_thresh': 0.0
+}
+```
+
+### Model in action 🚀
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import pipeline
+
+QnA_pipeline = pipeline('question-answering', model='mrm8488/electra-base-finetuned-squadv1')
+
+QnA_pipeline({
+    'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.',
+    'question': 'What has been discovered by scientists from China ?'
+})
+# Output:
+{'answer': 'A new strain of flu', 'end': 19, 'score': 0.9995211430099182, 'start': 0}
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/electra-small-finetuned-squadv2/README.md

@@ -0,0 +1,86 @@
+---
+language: english
+---
+
+# Electra small ⚡ + SQuAD v2 ❓
+
+[Electra-small-discriminator](https://huggingface.co/google/electra-small-discriminator) fine-tuned on [SQUAD v2.0 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/) for **Q&A** downstream task.
+
+## Details of the downstream task (Q&A) - Model 🧠
+
+**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset.
+
+
+## Details of the downstream task (Q&A) - Dataset 📚
+
+**SQuAD2.0** combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering.
+
+## Model training 🏋️‍
+
+The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
+
+```bash
+python transformers/examples/question-answering/run_squad.py \
+  --model_type electra \
+  --model_name_or_path 'google/electra-small-discriminator' \
+  --do_eval \
+  --do_train \
+  --do_lower_case \
+  --train_file '/content/dataset/train-v2.0.json' \
+  --predict_file '/content/dataset/dev-v2.0.json' \
+  --per_gpu_train_batch_size 16 \
+  --learning_rate 3e-5 \
+  --num_train_epochs 10 \
+  --max_seq_length 384 \
+  --doc_stride 128 \
+  --output_dir '/content/output' \
+  --overwrite_output_dir \
+  --save_steps 1000 \
+  --version_2_with_negative
+```
+
+## Test set Results 🧾
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **69.71** |
+| **F1** | **73.44** |
+| **Size**| **50 MB** |
+
+
+```json
+{
+'exact': 69.71279373368147,
+'f1': 73.4439546123672,
+'total': 11873,
+'HasAns_exact': 69.92240215924427,
+'HasAns_f1': 77.39542393937836,
+'HasAns_total': 5928,
+'NoAns_exact': 69.50378469301934,
+'NoAns_f1': 69.50378469301934,
+'NoAns_total': 5945,
+'best_exact': 69.71279373368147,
+'best_exact_thresh': 0.0,
+'best_f1': 73.44395461236732,
+'best_f1_thresh': 0.0
+}
+```
+
+### Model in action 🚀
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import pipeline
+QnA_pipeline = pipeline('question-answering', model='mrm8488/electra-base-finetuned-squadv2')
+QnA_pipeline({
+    'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.',
+    'question': 'What has been discovered by scientists from China ?'
+})
+# Output:
+{'answer': 'A new strain of flu', 'end': 19, 'score': 0.8650811568752914, 'start': 0}
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/electricidad-small-finetuned-squadv1-es/README.md

@@ -0,0 +1,102 @@
+---
+language: spanish
+thumbnail: https://imgur.com/uxAvBfh
+---
+
+# Electricidad small + Spanish SQuAD v1 ⚡❓
+
+[Electricidad-small-discriminator](https://huggingface.co/mrm8488/electricidad-small-discriminator) fine-tuned on [Spanish SQUAD v1.1 dataset](https://github.com/ccasimiro88/TranslateAlignRetrieve/tree/master/SQuAD-es-v1.1) for **Q&A** downstream task.
+
+## Details of the downstream task (Q&A) - Dataset 📚
+
+[SQuAD-es-v1.1](https://github.com/ccasimiro88/TranslateAlignRetrieve/tree/master/SQuAD-es-v1.1)
+
+| Dataset split | # Samples |
+| ------------- | --------- |
+| Train         | 130 K     |
+| Test          | 11 K      |
+
+## Model training 🏋️‍
+
+The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
+
+```bash
+python /content/transformers/examples/question-answering/run_squad.py \
+  --model_type electra \
+  --model_name_or_path 'mrm8488/electricidad-small-discriminator' \
+  --do_eval \
+  --do_train \
+  --do_lower_case \
+  --train_file '/content/dataset/train-v1.1-es.json' \
+  --predict_file '/content/dataset/dev-v1.1-es.json' \
+  --per_gpu_train_batch_size 16 \
+  --learning_rate 3e-5 \
+  --num_train_epochs 10 \
+  --max_seq_length 384 \
+  --doc_stride 128 \
+  --output_dir '/content/electricidad-small-finetuned-squadv1-es' \
+  --overwrite_output_dir \
+  --save_steps 1000
+```
+
+## Test set Results 🧾
+
+| Metric | # Value   |
+| ------ | --------- |
+| **EM** | **46.82** |
+| **F1** | **64.79** |
+
+```json
+{
+'exact': 46.82119205298013,
+'f1': 64.79435260021918,
+'total': 10570,
+'HasAns_exact': 46.82119205298013,
+HasAns_f1': 64.79435260021918,
+'HasAns_total': 10570,
+'best_exact': 46.82119205298013,
+'best_exact_thresh': 0.0,
+'best_f1': 64.79435260021918,
+'best_f1_thresh': 0.0
+}
+```
+
+### Model in action 🚀
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import pipeline
+
+qa_pipeline = pipeline(
+    "question-answering",
+    model="mrm8488/electricidad-small-finetuned-squadv1-es",
+    tokenizer="mrm8488/electricidad-small-finetuned-squadv1-es"
+)
+
+context = "Manuel ha creado una versión del modelo Electra small en español que alcanza una puntuación F1 de 65 en el dataset SQUAD-es y sólo pesa 50 MB"
+
+q1 = "Cuál es su marcador F1?"
+q2 = "¿Cuál es el tamaño del modelo?"
+q3 = "¿Quién lo ha creado?"
+q4 = "¿Que es lo que ha hecho Manuel?"
+
+
+questions = [q1, q2, q3, q4]
+
+for question in questions:
+  result = qa_pipeline({
+    'context': context,
+    'question': question})
+  print(result)
+
+# Output:
+{'score': 0.14836778166355025, 'start': 98, 'end': 100, 'answer': '65'}
+{'score': 0.32219420810758237, 'start': 136, 'end': 140, 'answer': '50 MB'}
+{'score': 0.9672326951118713, 'start': 0, 'end': 6, 'answer': 'Manuel'}
+{'score': 0.23552458113848118, 'start': 10, 'end': 53, 'answer': 'creado una versión del modelo Electra small'}
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/longformer-base-4096-finetuned-squadv2/README.md

@@ -50,7 +50,7 @@ model = AutoModelForQuestionAnswering.from_pretrained("mrm8488/longformer-base-4
 
 text = "Huggingface has democratized NLP. Huge thanks to Huggingface for this."
 question = "What has Huggingface done ?"
-encoding = tokenizer.encode_plus(question, text, return_tensors="pt")
+encoding = tokenizer(question, text, return_tensors="pt")
 input_ids = encoding["input_ids"]
 
 # default is local attention everywhere

model_cards/mrm8488/t5-base-finetuned-emotion/README.md

@@ -1,4 +1,4 @@
---
+---
 language: english
 ---
 
@@ -13,7 +13,7 @@ The **T5** model was presented in [Exploring the Limits of Transfer Learning wit
 
 Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
 
-![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67)
+![model image](https://i.imgur.com/jVFMMWR.png)
 
 ## Details of the downstream task (Sentiment Recognition) - Dataset 📚
 
@@ -32,18 +32,18 @@ The training script is a slightly modified version of [this Colab Notebook](http
 
 ## Test set metrics 🧾
 
-               |precision | recall  | f1-score |support|
-    |----------|----------|---------|----------|-------|
-    |anger     |      0.93|     0.92|      0.93|    275|
-    |fear      |      0.91|     0.87|      0.89|    224|
-    |joy       |      0.97|     0.94|      0.95|    695|
-    |love      |      0.80|     0.91|      0.85|    159|
-    |sadness   |      0.97|     0.97|      0.97|    521|
-    |surpirse  |      0.73|     0.89|      0.80|     66|
-    |----------|----------|---------|----------|-------|
-    |accuracy|            |         |      0.93|   2000|
-    |macro avg|       0.89|     0.92|      0.90|   2000|
-    |weighted avg|    0.94|     0.93|      0.93|   2000|
+|          |precision | recall  | f1-score |support|
+|----------|----------|---------|----------|-------|
+|anger     |      0.93|     0.92|      0.93|    275|
+|fear      |      0.91|     0.87|      0.89|    224|
+|joy       |      0.97|     0.94|      0.95|    695|
+|love      |      0.80|     0.91|      0.85|    159|
+|sadness   |      0.97|     0.97|      0.97|    521|
+|surpirse  |      0.73|     0.89|      0.80|     66|
+|                                                  |
+|accuracy|            |         |      0.93|   2000|
+|macro avg|       0.89|     0.92|      0.90|   2000|
+|weighted avg|    0.94|     0.93|      0.93|   2000|
     
     
                  
@@ -73,7 +73,6 @@ def get_emotion(text):
  get_emotion("i feel as if i havent blogged in ages are at least truly blogged i am doing an update cute") # Output: 'joy'
  
  get_emotion("i have a feeling i kinda lost my best friend") # Output: 'sadness'
-
 ```
 
 > Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)

model_cards/mrm8488/t5-base-finetuned-sarcasm-twitter/README.md

@@ -0,0 +1,112 @@
+---
+language: english
+---
+
+# T5-base fine-tuned for Sarcasm Detection 🙄
+[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) base fine-tuned on [ Twitter Sarcasm Dataset](https://github.com/EducationalTestingService/sarcasm) for **Sequence classification (as text generation)** downstream task.
+
+## Details of T5
+
+The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract:
+
+Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
+
+![model image](https://i.imgur.com/jVFMMWR.png)
+## Details of the downstream task (Sequence Classification as Text generation) - Dataset 📚
+
+[ Twitter Sarcasm Dataset](https://github.com/EducationalTestingService/sarcasm)
+
+
+For Twitter training and testing datasets are provided for sarcasm detection tasks in jsonlines format. 
+
+Each line contains a JSON object with the following fields : 
+- ***label*** : `SARCASM` or `NOT_SARCASM`  
+	- **NOT** in test data
+- ***id***:  String identifier for sample. This id will be required when making submissions.
+	- **ONLY** in test data
+- ***response*** :  the sarcastic response, whether a sarcastic Tweet
+- ***context*** : the conversation context of the ***response***
+	- Note, the context is an ordered list of dialogue, i.e., if the context contains three elements, `c1`, `c2`, `c3`, in that order, then `c2` is a reply to `c1` and `c3` is a reply to `c2`. Further, if the sarcastic response is `r`, then `r` is a reply to `c3`.
+
+For instance, for the following training example : 
+
+`"label": "SARCASM", "response": "Did Kelly just call someone else messy? Baaaahaaahahahaha", "context": ["X is looking a First Lady should . #classact, "didn't think it was tailored enough it looked messy"]`
+
+The response tweet, "Did Kelly..." is a reply to its immediate context "didn't think it was tailored..." which is a reply to "X is looking...". Your goal is to predict the label of the "response" while also using the context (i.e, the immediate or the full context).
+
+***Dataset size statistics*** :
+
+|         | Train | Val  | Test |
+|---------|-------|------|------|
+| Twitter | 4050  | 450  | 500  |
+
+The datasets was preprocessed to convert it to a **text-to-text** (classfication as generation task).
+
+## Model fine-tuning 🏋️‍
+
+The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him!
+
+## Test set metrics 🧾
+
+|          | precision| recall  | f1-score |support|
+|----------|----------|---------|----------|-------|
+| derison  |    0.84  |   0.80  |    0.82  |  246  |
+| normal   |    0.82  |   0.85  |    0.83  |  254  | 
+|                                                  |
+|accuracy|            |         |      0.83|    500|
+|macro avg|       0.83|     0.83|      0.83|    500|
+|weighted avg|    0.83|     0.83|      0.83|    500|
+    
+
+
+## Model in Action 🚀
+
+```python
+from transformers import AutoTokenizer, AutoModelWithLMHead
+tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-sarcasm-twitter")
+
+model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-sarcasm-twitter")
+
+def eval_conversation(text):
+
+  input_ids = tokenizer.encode(text + '</s>', return_tensors='pt')
+
+  output = model.generate(input_ids=input_ids, max_length=3)
+  
+  dec = [tokenizer.decode(ids) for ids in output]
+
+  label = dec[0]
+
+  return label
+
+# For similarity with the training dataset we should replace users mentions in twits for @USER token and urls for URL token.
+
+twit1 = "Trump just suspended the visa program that allowed me to move to the US to start @USER!" +
+" Unfortunately, I won’t be able to vote in a few months but if you can, please vote him out, " +
+"he's destroying what made America great in so many different ways!"
+
+twit2 = "@USER @USER @USER We have far more cases than any other country, " +
+"so leaving remote workers in would be disastrous. Makes Trump sense."
+
+twit3 = "My worry is that i wouldn’t be surprised if half the country actually agrees with this move..."
+
+me = "Trump doing so??? It must be a mistake... XDDD"
+
+conversation = twit1 + twit2
+
+eval_conversation(conversation) #Output: 'derison'
+
+conversation = twit1 + twit3
+
+eval_conversation(conversation) #Output: 'normal'
+
+conversation = twit1 + me
+
+eval_conversation(conversation) #Output: 'derison'
+
+# We will get 'normal' when sarcasm is not detected and 'derison' when detected
+```
+
+> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
+
+> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/t5-base-finetuned-squadv2/README.md

@@ -14,14 +14,17 @@ The **T5** model was presented in [Exploring the Limits of Transfer Learning wit
 
 Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
 
+![model image](https://i.imgur.com/jVFMMWR.png)
+
 
 ## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓
 
 Dataset ID: ```squad_v2``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
+
 | Dataset  | Split | # samples |
 | -------- | ----- | --------- |
-| squad_v2 | train | 130319      |
-| squad_v2 | valid  | 11873     |
+| squad_v2 | train | 130319    |
+| squad_v2 | valid  | 11873    |
 
 How to load it from [nlp](https://github.com/huggingface/nlp)
 
@@ -55,7 +58,7 @@ model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-squadv2")
 
 def get_answer(question, context):
   input_text = "question: %s  context: %s </s>" % (question, context)
-  features = tokenizer.batch_encode_plus([input_text], return_tensors='pt')
+  features = tokenizer([input_text], return_tensors='pt')
 
   output = model.generate(input_ids=features['input_ids'], 
                attention_mask=features['attention_mask'])

model_cards/mrm8488/t5-base-finetuned-summarize-news/README.md

@@ -15,7 +15,7 @@ The **T5** model was presented in [Exploring the Limits of Transfer Learning wit
 
 Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
 
-![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67)
+![model image](https://i.imgur.com/jVFMMWR.png)
 
 ## Details of the downstream task (Summarization) - Dataset 📚
 

model_cards/oliverguhr/german-sentiment-bert/README.md

@@ -55,7 +55,7 @@ class SentimentModel():
     def predict_sentiment(self, texts: List[str])-> List[str]:
         texts = [self.clean_text(text) for text in texts]
         # Add special tokens takes care of adding [CLS], [SEP], <s>... tokens in the right way for each model.
-        input_ids = self.tokenizer.batch_encode_plus(texts,pad_to_max_length=True, add_special_tokens=True)
+        input_ids = self.tokenizer(texts, padding=True, truncation=True, add_special_tokens=True)
         input_ids = torch.tensor(input_ids["input_ids"])
 
         with torch.no_grad():

model_cards/pradhyra/AWSBlogBert/README.md

@@ -0,0 +1,8 @@
+This model is pre-trained on blog articles from AWS Blogs.
+
+## Pre-training corpora
+The input text contains around 3000 blog articles on [AWS Blogs website](https://aws.amazon.com/blogs/) technical subject matter including AWS products, tools and tutorials. 
+
+## Pre-training details
+I picked a Roberta architecture for masked language modeling (6-layer, 768-hidden, 12-heads, 82M parameters) and its corresponding ByteLevelBPE tokenization strategy. I then followed HuggingFace's Transformers [blog post](https://huggingface.co/blog/how-to-train) to train the model.
+I chose to follow the following training set-up: 28k training steps with batches of 64 sequences of length 512 with an initial learning rate 5e-5. The model acheived a training loss of 3.6 on the MLM task over 10 epochs.

model_cards/roberta-base-README.md

@@ -1,10 +1,234 @@
 ---
+language: english
 tags:
 - exbert
-
 license: mit
+datasets:
+- bookcorpus
+- wikipedia
 ---
 
+# RoBERTa base model
+
+Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in
+[this paper](https://arxiv.org/abs/1907.11692) and first released in
+[this repository](https://github.com/pytorch/fairseq/tree/master/examples/roberta). This model is case-sensitive: it
+makes a difference between english and English.
+
+Disclaimer: The team releasing RoBERTa did not write a model card for this model so this model card has been written by
+the Hugging Face team.
+
+## Model description
+
+RoBERTa is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means
+it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of
+publicly available data) with an automatic process to generate inputs and labels from those texts. 
+
+More precisely, it was pretrained with the Masked language modeling (MLM) objective. Taking a sentence, the model
+randomly masks 15% of the words in the input then run the entire masked sentence through the model and has to predict
+the masked words. This is different from traditional recurrent neural networks (RNNs) that usually see the words one
+after the other, or from autoregressive models like GPT which internally mask the future tokens. It allows the model to
+learn a bidirectional representation of the sentence.
+
+This way, the model learns an inner representation of the English language that can then be used to extract features
+useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard
+classifier using the features produced by the BERT model as inputs.
+
+## Intended uses & limitations
+
+You can use the raw model for masked language modeling, but it's mostly intended to be fine-tuned on a downstream task.
+See the [model hub](https://huggingface.co/models?filter=roberta) to look for fine-tuned versions on a task that
+interests you.
+
+Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked)
+to make decisions, such as sequence classification, token classification or question answering. For tasks such as text
+generation you should look at model like GPT2.
+
+### How to use
+
+You can use this model directly with a pipeline for masked language modeling:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='roberta-base')
+>>> unmasker("Hello I'm a <mask> model.")
+
+[{'sequence': "<s>Hello I'm a male model.</s>",
+  'score': 0.3306540250778198,
+  'token': 2943,
+  'token_str': 'Ġmale'},
+ {'sequence': "<s>Hello I'm a female model.</s>",
+  'score': 0.04655390977859497,
+  'token': 2182,
+  'token_str': 'Ġfemale'},
+ {'sequence': "<s>Hello I'm a professional model.</s>",
+  'score': 0.04232972860336304,
+  'token': 2038,
+  'token_str': 'Ġprofessional'},
+ {'sequence': "<s>Hello I'm a fashion model.</s>",
+  'score': 0.037216778844594955,
+  'token': 2734,
+  'token_str': 'Ġfashion'},
+ {'sequence': "<s>Hello I'm a Russian model.</s>",
+  'score': 0.03253649175167084,
+  'token': 1083,
+  'token_str': 'ĠRussian'}]
+```
+
+Here is how to use this model to get the features of a given text in PyTorch:
+
+```python
+from transformers import RobertaTokenizer, RobertaModel
+tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
+model = RobertaModel.from_pretrained('roberta-base')
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='pt')
+output = model(**encoded_input)
+```
+
+and in TensorFlow:
+
+```python
+from transformers import RobertaTokenizer, TFRobertaModel
+tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
+model = TFRobertaModel.from_pretrained('roberta-base')
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='tf')
+output = model(encoded_input)
+```
+
+### Limitations and bias
+
+The training data used for this model contains a lot of unfiltered content from the internet, which is far from
+neutral. Therefore, the model can have biased predictions:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='roberta-base')
+>>> unmasker("The man worked as a <mask>.")
+
+[{'sequence': '<s>The man worked as a mechanic.</s>',
+  'score': 0.08702439814805984,
+  'token': 25682,
+  'token_str': 'Ġmechanic'},
+ {'sequence': '<s>The man worked as a waiter.</s>',
+  'score': 0.0819653645157814,
+  'token': 38233,
+  'token_str': 'Ġwaiter'},
+ {'sequence': '<s>The man worked as a butcher.</s>',
+  'score': 0.073323555290699,
+  'token': 32364,
+  'token_str': 'Ġbutcher'},
+ {'sequence': '<s>The man worked as a miner.</s>',
+  'score': 0.046322137117385864,
+  'token': 18678,
+  'token_str': 'Ġminer'},
+ {'sequence': '<s>The man worked as a guard.</s>',
+  'score': 0.040150221437215805,
+  'token': 2510,
+  'token_str': 'Ġguard'}]
+
+>>> unmasker("The Black woman worked as a <mask>.")
+
+[{'sequence': '<s>The Black woman worked as a waitress.</s>',
+  'score': 0.22177888453006744,
+  'token': 35698,
+  'token_str': 'Ġwaitress'},
+ {'sequence': '<s>The Black woman worked as a prostitute.</s>',
+  'score': 0.19288744032382965,
+  'token': 36289,
+  'token_str': 'Ġprostitute'},
+ {'sequence': '<s>The Black woman worked as a maid.</s>',
+  'score': 0.06498628109693527,
+  'token': 29754,
+  'token_str': 'Ġmaid'},
+ {'sequence': '<s>The Black woman worked as a secretary.</s>',
+  'score': 0.05375480651855469,
+  'token': 2971,
+  'token_str': 'Ġsecretary'},
+ {'sequence': '<s>The Black woman worked as a nurse.</s>',
+  'score': 0.05245552211999893,
+  'token': 9008,
+  'token_str': 'Ġnurse'}]
+```
+
+This bias will also affect all fine-tuned versions of this model.
+
+## Training data
+
+The RoBERTa model was pretrained on the reunion of five datasets:
+- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books;
+- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers) ;
+- [CC-News](https://commoncrawl.org/2016/10/news-dataset-available/), a dataset containing 63 millions English news
+  articles crawled between September 2016 and February 2019.
+- [OpenWebText](https://github.com/jcpeterson/openwebtext), an opensource recreation of the WebText dataset used to
+  train GPT-2,
+- [Stories](https://arxiv.org/abs/1806.02847) a dataset containing a subset of CommonCrawl data filtered to match the
+  story-like style of Winograd schemas.
+
+Together theses datasets weight 160GB of text.
+
+## Training procedure
+
+### Preprocessing
+
+The texts are tokenized using a byte version of Byte-Pair Encoding (BPE) and a vocabulary size of 50,000. The inputs of
+the model take pieces of 512 contiguous token that may span over documents. The beginning of a new document is marked
+with `<s>` and the end of one by `</s>`
+
+The details of the masking procedure for each sentence are the following:
+- 15% of the tokens are masked.
+- In 80% of the cases, the masked tokens are replaced by `<mask>`.
+- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace.
+- In the 10% remaining cases, the masked tokens are left as is.
+
+Contrary to BERT, the masking is done dynamically during pretraining (e.g., it changes at each epoch and is not fixed).
+
+### Pretraining
+
+The model was trained on 1024 V100 GPUs for 500K steps with a batch size of 8K and a sequence length of 512. The
+optimizer used is Adam with a learning rate of 6e-4, \\(\beta_{1} = 0.9\\), \\(\beta_{2} = 0.98\\) and
+\\(\epsilon = 1e-6\\), a weight decay of 0.01, learning rate warmup for 24,000 steps and linear decay of the learning
+rate after.
+
+## Evaluation results
+
+When fine-tuned on downstream tasks, this model achieves the following results:
+
+Glue test results:
+
+| Task | MNLI | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  |
+|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|
+|      | 87.6 | 91.9 | 92.8 | 94.8  | 63.6 | 91.2  | 90.2 | 78.7 |
+
+
+### BibTeX entry and citation info
+
+```bibtex
+@article{DBLP:journals/corr/abs-1907-11692,
+  author    = {Yinhan Liu and
+               Myle Ott and
+               Naman Goyal and
+               Jingfei Du and
+               Mandar Joshi and
+               Danqi Chen and
+               Omer Levy and
+               Mike Lewis and
+               Luke Zettlemoyer and
+               Veselin Stoyanov},
+  title     = {RoBERTa: {A} Robustly Optimized {BERT} Pretraining Approach},
+  journal   = {CoRR},
+  volume    = {abs/1907.11692},
+  year      = {2019},
+  url       = {http://arxiv.org/abs/1907.11692},
+  archivePrefix = {arXiv},
+  eprint    = {1907.11692},
+  timestamp = {Thu, 01 Aug 2019 08:59:33 +0200},
+  biburl    = {https://dblp.org/rec/journals/corr/abs-1907-11692.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+```
+
 <a href="https://huggingface.co/exbert/?model=roberta-base">
 	<img width="300px" src="https://hf-dinosaur.huggingface.co/exbert/button.png">
 </a>

model_cards/roberta-large-README.md

@@ -0,0 +1,235 @@
+---
+language: english
+tags:
+- exbert
+license: mit
+datasets:
+- bookcorpus
+- wikipedia
+---
+
+# RoBERTa large model
+
+Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in
+[this paper](https://arxiv.org/abs/1907.11692) and first released in
+[this repository](https://github.com/pytorch/fairseq/tree/master/examples/roberta). This model is case-sensitive: it
+makes a difference between english and English.
+
+Disclaimer: The team releasing RoBERTa did not write a model card for this model so this model card has been written by
+the Hugging Face team.
+
+## Model description
+
+RoBERTa is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means
+it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of
+publicly available data) with an automatic process to generate inputs and labels from those texts. 
+
+More precisely, it was pretrained with the Masked language modeling (MLM) objective. Taking a sentence, the model
+randomly masks 15% of the words in the input then run the entire masked sentence through the model and has to predict
+the masked words. This is different from traditional recurrent neural networks (RNNs) that usually see the words one
+after the other, or from autoregressive models like GPT which internally mask the future tokens. It allows the model to
+learn a bidirectional representation of the sentence.
+
+This way, the model learns an inner representation of the English language that can then be used to extract features
+useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard
+classifier using the features produced by the BERT model as inputs.
+
+## Intended uses & limitations
+
+You can use the raw model for masked language modeling, but it's mostly intended to be fine-tuned on a downstream task.
+See the [model hub](https://huggingface.co/models?filter=roberta) to look for fine-tuned versions on a task that
+interests you.
+
+Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked)
+to make decisions, such as sequence classification, token classification or question answering. For tasks such as text
+generation you should look at model like GPT2.
+
+### How to use
+
+You can use this model directly with a pipeline for masked language modeling:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='roberta-large')
+>>> unmasker("Hello I'm a <mask> model.")
+
+[{'sequence': "<s>Hello I'm a male model.</s>",
+  'score': 0.3317350447177887,
+  'token': 2943,
+  'token_str': 'Ġmale'},
+ {'sequence': "<s>Hello I'm a fashion model.</s>",
+  'score': 0.14171843230724335,
+  'token': 2734,
+  'token_str': 'Ġfashion'},
+ {'sequence': "<s>Hello I'm a professional model.</s>",
+  'score': 0.04291723668575287,
+  'token': 2038,
+  'token_str': 'Ġprofessional'},
+ {'sequence': "<s>Hello I'm a freelance model.</s>",
+  'score': 0.02134818211197853,
+  'token': 18150,
+  'token_str': 'Ġfreelance'},
+ {'sequence': "<s>Hello I'm a young model.</s>",
+  'score': 0.021098261699080467,
+  'token': 664,
+  'token_str': 'Ġyoung'}]
+```
+
+Here is how to use this model to get the features of a given text in PyTorch:
+
+```python
+from transformers import RobertaTokenizer, RobertaModel
+tokenizer = RobertaTokenizer.from_pretrained('roberta-large')
+model = RobertaModel.from_pretrained('roberta-large')
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='pt')
+output = model(**encoded_input)
+```
+
+and in TensorFlow:
+
+```python
+from transformers import RobertaTokenizer, TFRobertaModel
+tokenizer = RobertaTokenizer.from_pretrained('roberta-large')
+model = TFRobertaModel.from_pretrained('roberta-large')
+text = "Replace me by any text you'd like."
+encoded_input = tokenizer(text, return_tensors='tf')
+output = model(encoded_input)
+```
+
+### Limitations and bias
+
+The training data used for this model contains a lot of unfiltered content from the internet, which is far from
+neutral. Therefore, the model can have biased predictions:
+
+```python
+>>> from transformers import pipeline
+>>> unmasker = pipeline('fill-mask', model='roberta-large')
+>>> unmasker("The man worked as a <mask>.")
+
+[{'sequence': '<s>The man worked as a mechanic.</s>',
+  'score': 0.08260300755500793,
+  'token': 25682,
+  'token_str': 'Ġmechanic'},
+ {'sequence': '<s>The man worked as a driver.</s>',
+  'score': 0.05736079439520836,
+  'token': 1393,
+  'token_str': 'Ġdriver'},
+ {'sequence': '<s>The man worked as a teacher.</s>',
+  'score': 0.04709019884467125,
+  'token': 3254,
+  'token_str': 'Ġteacher'},
+ {'sequence': '<s>The man worked as a bartender.</s>',
+  'score': 0.04641604796051979,
+  'token': 33080,
+  'token_str': 'Ġbartender'},
+ {'sequence': '<s>The man worked as a waiter.</s>',
+  'score': 0.04239227622747421,
+  'token': 38233,
+  'token_str': 'Ġwaiter'}]
+
+>>> unmasker("The woman worked as a <mask>.")
+
+[{'sequence': '<s>The woman worked as a nurse.</s>',
+  'score': 0.2667474150657654,
+  'token': 9008,
+  'token_str': 'Ġnurse'},
+ {'sequence': '<s>The woman worked as a waitress.</s>',
+  'score': 0.12280137836933136,
+  'token': 35698,
+  'token_str': 'Ġwaitress'},
+ {'sequence': '<s>The woman worked as a teacher.</s>',
+  'score': 0.09747499972581863,
+  'token': 3254,
+  'token_str': 'Ġteacher'},
+ {'sequence': '<s>The woman worked as a secretary.</s>',
+  'score': 0.05783602222800255,
+  'token': 2971,
+  'token_str': 'Ġsecretary'},
+ {'sequence': '<s>The woman worked as a cleaner.</s>',
+  'score': 0.05576248839497566,
+  'token': 16126,
+  'token_str': 'Ġcleaner'}]
+```
+
+This bias will also affect all fine-tuned versions of this model.
+
+## Training data
+
+The RoBERTa model was pretrained on the reunion of five datasets:
+- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books;
+- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers) ;
+- [CC-News](https://commoncrawl.org/2016/10/news-dataset-available/), a dataset containing 63 millions English news
+  articles crawled between September 2016 and February 2019.
+- [OpenWebText](https://github.com/jcpeterson/openwebtext), an opensource recreation of the WebText dataset used to
+  train GPT-2,
+- [Stories](https://arxiv.org/abs/1806.02847) a dataset containing a subset of CommonCrawl data filtered to match the
+  story-like style of Winograd schemas.
+
+Together theses datasets weight 160GB of text.
+
+## Training procedure
+
+### Preprocessing
+
+The texts are tokenized using a byte version of Byte-Pair Encoding (BPE) and a vocabulary size of 50,000. The inputs of
+the model take pieces of 512 contiguous token that may span over documents. The beginning of a new document is marked
+with `<s>` and the end of one by `</s>`
+
+The details of the masking procedure for each sentence are the following:
+- 15% of the tokens are masked.
+- In 80% of the cases, the masked tokens are replaced by `<mask>`.
+
+- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace.
+- In the 10% remaining cases, the masked tokens are left as is.
+
+Contrary to BERT, the masking is done dynamically during pretraining (e.g., it changes at each epoch and is not fixed).
+
+### Pretraining
+
+The model was trained on 1024 V100 GPUs for 500K steps with a batch size of 8K and a sequence length of 512. The
+optimizer used is Adam with a learning rate of 4e-4, \\(\beta_{1} = 0.9\\), \\(\beta_{2} = 0.98\\) and
+\\(\epsilon = 1e-6\\), a weight decay of 0.01, learning rate warmup for 30,000 steps and linear decay of the learning
+rate after.
+
+## Evaluation results
+
+When fine-tuned on downstream tasks, this model achieves the following results:
+
+Glue test results:
+
+| Task | MNLI | QQP  | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE  |
+|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|
+|      | 90.2 | 92.2 | 94.7 | 96.4  | 68.0 | 96.4  | 90.9 | 86.6 |
+
+
+### BibTeX entry and citation info
+
+```bibtex
+@article{DBLP:journals/corr/abs-1907-11692,
+  author    = {Yinhan Liu and
+               Myle Ott and
+               Naman Goyal and
+               Jingfei Du and
+               Mandar Joshi and
+               Danqi Chen and
+               Omer Levy and
+               Mike Lewis and
+               Luke Zettlemoyer and
+               Veselin Stoyanov},
+  title     = {RoBERTa: {A} Robustly Optimized {BERT} Pretraining Approach},
+  journal   = {CoRR},
+  volume    = {abs/1907.11692},
+  year      = {2019},
+  url       = {http://arxiv.org/abs/1907.11692},
+  archivePrefix = {arXiv},
+  eprint    = {1907.11692},
+  timestamp = {Thu, 01 Aug 2019 08:59:33 +0200},
+  biburl    = {https://dblp.org/rec/journals/corr/abs-1907-11692.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+```
+
+<a href="https://huggingface.co/exbert/?model=roberta-base">
+	<img width="300px" src="https://hf-dinosaur.huggingface.co/exbert/button.png">
+</a>

model_cards/roberta-large-mnli-README.md

@@ -0,0 +1,22 @@
+---
+license: mit
+widget:
+- text: "I like you. </s></s> I love you."
+---
+
+
+## roberta-large-mnli
+
+Trained by Facebook, [original source](https://github.com/pytorch/fairseq/tree/master/examples/roberta)
+
+```bibtex
+@article{liu2019roberta,
+    title = {RoBERTa: A Robustly Optimized BERT Pretraining Approach},
+    author = {Yinhan Liu and Myle Ott and Naman Goyal and Jingfei Du and
+              Mandar Joshi and Danqi Chen and Omer Levy and Mike Lewis and
+              Luke Zettlemoyer and Veselin Stoyanov},
+    journal={arXiv preprint arXiv:1907.11692},
+    year = {2019},
+}
+```
+

model_cards/schmidek/electra-small-cased/README.md

@@ -0,0 +1,11 @@
+---
+language: english
+license: apache-2.0
+---
+
+## ELECTRA-small-cased
+
+This is a cased version of `google/electra-small-discriminator`, trained on the
+[OpenWebText corpus](https://skylion007.github.io/OpenWebTextCorpus/).
+
+Uses the same tokenizer and vocab from `bert-base-cased`

model_cards/valhalla/bart-large-finetuned-squadv1/README.md

@@ -50,7 +50,7 @@ tokenizer = BartTokenizer.from_pretrained('valhalla/bart-large-finetuned-squadv1
 model = BartForQuestionAnswering.from_pretrained('valhalla/bart-large-finetuned-squadv1')
 
 question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
-encoding = tokenizer.encode_plus(question, text, return_tensors='pt')
+encoding = tokenizer(question, text, return_tensors='pt')
 input_ids = encoding['input_ids']
 attention_mask = encoding['attention_mask']
 

model_cards/valhalla/longformer-base-4096-finetuned-squadv1/README.md

@@ -33,7 +33,7 @@ model = AutoModelForQuestionAnswering.from_pretrained("valhalla/longformer-base-
 
 text = "Huggingface has democratized NLP. Huge thanks to Huggingface for this."
 question = "What has Huggingface done ?"
-encoding = tokenizer.encode_plus(question, text, return_tensors="pt")
+encoding = tokenizer(question, text, return_tensors="pt")
 input_ids = encoding["input_ids"]
 
 # default is local attention everywhere

model_cards/valhalla/t5-base-squad/README.md

@@ -19,7 +19,7 @@ model = AutoModelWithLMHead.from_pretrained("valhalla/t5-base-squad")
 
 def get_answer(question, context):
   input_text = "question: %s  context: %s </s>" % (question, context)
-  features = tokenizer.batch_encode_plus([input_text], return_tensors='pt')
+  features = tokenizer([input_text], return_tensors='pt')
 
   out = model.generate(input_ids=features['input_ids'], 
                attention_mask=features['attention_mask'])

notebooks/02-transformers.ipynb

@@ -255,7 +255,7 @@
     "# tokens_pt = torch.tensor([tokens_ids])\n",
     "\n",
     "# This code can be factored into one-line as follow\n",
-    "tokens_pt2 = tokenizer.encode_plus(\"This is an input example\", return_tensors=\"pt\")\n",
+    "tokens_pt2 = tokenizer(\"This is an input example\", return_tensors=\"pt\")\n",
     "\n",
     "for key, value in tokens_pt2.items():\n",
     "    print(\"{}:\\n\\t{}\".format(key, value))\n",
@@ -268,7 +268,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "As you can see above, the method `encode_plus` provides a convenient way to generate all the required parameters\n",
+    "As you can see above, calling the tokenizer provides a convenient way to generate all the required parameters\n",
     "that will go through the model. \n",
     "\n",
     "Moreover, you might have noticed it generated some additional tensors: \n",
@@ -302,10 +302,10 @@
    ],
    "source": [
     "# Single segment input\n",
-    "single_seg_input = tokenizer.encode_plus(\"This is a sample input\")\n",
+    "single_seg_input = tokenizer(\"This is a sample input\")\n",
     "\n",
     "# Multiple segment input\n",
-    "multi_seg_input = tokenizer.encode_plus(\"This is segment A\", \"This is segment B\")\n",
+    "multi_seg_input = tokenizer(\"This is segment A\", \"This is segment B\")\n",
     "\n",
     "print(\"Single segment token (str): {}\".format(tokenizer.convert_ids_to_tokens(single_seg_input['input_ids'])))\n",
     "print(\"Single segment token (int): {}\".format(single_seg_input['input_ids']))\n",
@@ -344,9 +344,9 @@
    ],
    "source": [
     "# Padding highlight\n",
-    "tokens = tokenizer.batch_encode_plus(\n",
+    "tokens = tokenizer(\n",
     "    [\"This is a sample\", \"This is another longer sample text\"], \n",
-    "    pad_to_max_length=True  # First sentence will have some PADDED tokens to match second sequence length\n",
+    "    padding=True  # First sentence will have some PADDED tokens to match second sequence length\n",
     ")\n",
     "\n",
     "for i in range(2):\n",
@@ -405,8 +405,8 @@
    ],
    "source": [
     "# transformers generates a ready to use dictionary with all the required parameters for the specific framework.\n",
-    "input_tf = tokenizer.encode_plus(\"This is a sample input\", return_tensors=\"tf\")\n",
-    "input_pt = tokenizer.encode_plus(\"This is a sample input\", return_tensors=\"pt\")\n",
+    "input_tf = tokenizer(\"This is a sample input\", return_tensors=\"tf\")\n",
+    "input_pt = tokenizer(\"This is a sample input\", return_tensors=\"pt\")\n",
     "\n",
     "# Let's compare the outputs\n",
     "output_tf, output_pt = model_tf(input_tf), model_pt(**input_pt)\n",
@@ -464,7 +464,7 @@
     "from transformers import DistilBertModel\n",
     "\n",
     "bert_distil = DistilBertModel.from_pretrained('distilbert-base-cased')\n",
-    "input_pt = tokenizer.encode_plus(\n",
+    "input_pt = tokenizer(\n",
     "    'This is a sample input to demonstrate performance of distiled models especially inference time', \n",
     "    return_tensors=\"pt\"\n",
     ")\n",
@@ -514,7 +514,7 @@
     "de_bert = BertModel.from_pretrained(\"dbmdz/bert-base-german-cased\")\n",
     "de_tokenizer = BertTokenizer.from_pretrained(\"dbmdz/bert-base-german-cased\")\n",
     "\n",
-    "de_input = de_tokenizer.encode_plus(\n",
+    "de_input = de_tokenizer(\n",
     "    \"Hugging Face ist eine französische Firma mit Sitz in New-York.\",\n",
     "    return_tensors=\"pt\"\n",
     ")\n",
@@ -559,4 +559,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

notebooks/04-onnx-export.ipynb

@@ -248,7 +248,7 @@
     "cpu_model = create_model_for_provider(\"onnx/bert-base-cased.onnx\", \"CPUExecutionProvider\")\n",
     "\n",
     "# Inputs are provided through numpy array\n",
-    "model_inputs = tokenizer.encode_plus(\"My name is Bert\", return_tensors=\"pt\")\n",
+    "model_inputs = tokenizer(\"My name is Bert\", return_tensors=\"pt\")\n",
     "inputs_onnx = {k: v.cpu().detach().numpy() for k, v in model_inputs.items()}\n",
     "\n",
     "# Run the model (None = get all the outputs)\n",

notebooks/05-benchmark.ipynb

@@ -289,7 +289,7 @@
         "\n",
         "Being able to accurately benchmark language models on both *speed* and *required memory* is therefore very important.\n",
         "\n",
-        "HuggingFace's Transformer library allows users to benchmark models for both Tensorflow 2 and PyTorch using the `PyTorchBenchmark` and `TensorflowBenchmark` classes.\n",
+        "HuggingFace's Transformer library allows users to benchmark models for both TensorFlow 2 and PyTorch using the `PyTorchBenchmark` and `TensorFlowBenchmark` classes.\n",
         "\n",
         "The currently available features for `PyTorchBenchmark` are summarized in the following table.\n",
         "\n",
@@ -306,7 +306,7 @@
         "\n",
         "*   *torchscript* corresponds to PyTorch's torchscript format, see [here](https://pytorch.org/docs/stable/jit.html).\n",
         "\n",
-        "The currently available features for `TensorflowBenchmark` are summarized in the following table.\n",
+        "The currently available features for `TensorFlowBenchmark` are summarized in the following table.\n",
         "\n",
         "| | CPU | CPU + eager execution | GPU | GPU + eager execution | GPU + XLA | GPU + FP16 | TPU |\n",
         ":-- | :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n",
@@ -315,16 +315,16 @@
         "**Speed - Train** | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ |\n",
         "**Memory - Train** | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ | ✘ |\n",
         "\n",
-        "*   *eager execution* means that the function is run in the eager execution environment of Tensorflow 2, see [here](https://www.tensorflow.org/guide/eager).\n",
+        "*   *eager execution* means that the function is run in the eager execution environment of TensorFlow 2, see [here](https://www.tensorflow.org/guide/eager).\n",
         "\n",
-        "*   *XLA* stands for Tensorflow's Accelerated Linear Algebra (XLA) compiler, see [here](https://www.tensorflow.org/xla)\n",
+        "*   *XLA* stands for TensorFlow's Accelerated Linear Algebra (XLA) compiler, see [here](https://www.tensorflow.org/xla)\n",
         "\n",
-        "*   *FP16* stands for Tensorflow's mixed-precision package and is analogous to PyTorch's FP16 feature, see [here](https://www.tensorflow.org/guide/mixed_precision).\n",
+        "*   *FP16* stands for TensorFlow's mixed-precision package and is analogous to PyTorch's FP16 feature, see [here](https://www.tensorflow.org/guide/mixed_precision).\n",
         "\n",
-        "***Note***: In ~1,2 weeks it will also be possible to benchmark training in Tensorflow.\n",
+        "***Note***: In ~1,2 weeks it will also be possible to benchmark training in TensorFlow.\n",
         "\n",
         "\n",
-        "This notebook will show the user how to use `PyTorchBenchmark` and `TensorflowBenchmark` for two different scenarios:\n",
+        "This notebook will show the user how to use `PyTorchBenchmark` and `TensorFlowBenchmark` for two different scenarios:\n",
         "\n",
         "1. **Inference - Pre-trained Model Comparison** - *A user wants to implement a pre-trained model in production for inference. She wants to compare different models on speed and required memory.*\n",
         "\n",
@@ -443,7 +443,7 @@
       "source": [
         "Looks good! Now we import `transformers` and download the scripts `run_benchmark.py`, `run_benchmark_tf.py`, and `plot_csv_file.py` which can be found under `transformers/examples/benchmarking`.\n",
         "\n",
-        "`run_benchmark_tf.py` and `run_benchmark.py` are very simple scripts leveraging the `PyTorchBenchmark` and `TensorflowBenchmark` classes, respectively."
+        "`run_benchmark_tf.py` and `run_benchmark.py` are very simple scripts leveraging the `PyTorchBenchmark` and `TensorFlowBenchmark` classes, respectively."
       ]
     },
     {
@@ -482,7 +482,7 @@
         "colab_type": "text"
       },
       "source": [
-        "Information about the input arguments to the *run_benchmark* scripts can be accessed by running `!python run_benchmark.py --help` for PyTorch and `!python run_benchmark_tf.py --help` for Tensorflow."
+        "Information about the input arguments to the *run_benchmark* scripts can be accessed by running `!python run_benchmark.py --help` for PyTorch and `!python run_benchmark_tf.py --help` for TensorFlow."
       ]
     },
     {
@@ -1130,7 +1130,7 @@
       },
       "source": [
         "At this point, it is important to understand how the peak memory is measured. The benchmarking tools measure the peak memory usage the same way the command `nvidia-smi` does - see [here](https://developer.nvidia.com/nvidia-system-management-interface) for more information. \n",
-        "In short, all memory that is allocated for a given *model identifier*, *batch size* and *sequence length* is measured in a separate process. This way it can be ensured that there is no previously unreleased memory falsely included in the measurement. One should also note that the measured memory even includes the memory allocated by the CUDA driver to load PyTorch and Tensorflow and is, therefore, higher than library-specific memory measurement function, *e.g.* this one for [PyTorch](https://pytorch.org/docs/stable/cuda.html#torch.cuda.max_memory_allocated).\n",
+        "In short, all memory that is allocated for a given *model identifier*, *batch size* and *sequence length* is measured in a separate process. This way it can be ensured that there is no previously unreleased memory falsely included in the measurement. One should also note that the measured memory even includes the memory allocated by the CUDA driver to load PyTorch and TensorFlow and is, therefore, higher than library-specific memory measurement function, *e.g.* this one for [PyTorch](https://pytorch.org/docs/stable/cuda.html#torch.cuda.max_memory_allocated).\n",
         "\n",
         "Alright, let's analyze the results. It can be noted that the models `aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2` and `deepset/roberta-base-squad2` require significantly less memory than the other three models. Besides `mrm8488/longformer-base-4096-finetuned-squadv2` all models more or less follow the same memory consumption pattern with `aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2` seemingly being able to better scale to larger sequence lengths. \n",
         "`mrm8488/longformer-base-4096-finetuned-squadv2` is a *Longformer* model, which makes use of *LocalAttention* (check this blog post to learn more about local attention) so that the model scales much better to longer input sequences.\n",
@@ -1256,7 +1256,7 @@
       "source": [
         "Interesting! `aodiniz/bert_uncased_L-10_H-51` clearly scales better for higher batch sizes and does not even run out of memory for 512 tokens.\n",
         "\n",
-        "For comparison, let's run the same benchmarking on Tensorflow."
+        "For comparison, let's run the same benchmarking on TensorFlow."
       ]
     },
     {
@@ -1341,7 +1341,7 @@
         "colab_type": "text"
       },
       "source": [
-        "Let's see the same plot for Tensorflow."
+        "Let's see the same plot for TensorFlow."
       ]
     },
     {
@@ -1394,7 +1394,7 @@
         "colab_type": "text"
       },
       "source": [
-        "The model implemented in Tensorflow requires more memory than the one implemented in PyTorch. Let's say for whatever reason we have decided to use Tensorflow instead of PyTorch. \n",
+        "The model implemented in TensorFlow requires more memory than the one implemented in PyTorch. Let's say for whatever reason we have decided to use TensorFlow instead of PyTorch. \n",
         "\n",
         "The next step is to measure the inference time of these two models. Instead of disabling time measurement with `--no_speed`, we will now disable memory measurement with `--no_memory`."
       ]
@@ -1499,7 +1499,7 @@
       "source": [
         "Ok, this took some time... time measurements take much longer than memory measurements because the forward pass is called multiple times for stable results. Timing measurements leverage Python's [timeit module](https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat) and run 10 times the value given to the `--repeat` argument (defaults to 3), so in our case 30 times.\n",
         "\n",
-        "Let's focus on the resulting plot. It becomes obvious that `aodiniz/bert_uncased_L-10_H-51` is around twice as fast as `deepset/roberta-base-squad2`. Given that the model is also more memory efficient and assuming that the model performs reasonably well, for the sake of this notebook we will settle on `aodiniz/bert_uncased_L-10_H-51`. Our model should be able to process input sequences of up to 512 tokens. Latency time of around 2 seconds might be too long though, so let's compare the time for different batch sizes and using Tensorflows XLA package for more speed."
+        "Let's focus on the resulting plot. It becomes obvious that `aodiniz/bert_uncased_L-10_H-51` is around twice as fast as `deepset/roberta-base-squad2`. Given that the model is also more memory efficient and assuming that the model performs reasonably well, for the sake of this notebook we will settle on `aodiniz/bert_uncased_L-10_H-51`. Our model should be able to process input sequences of up to 512 tokens. Latency time of around 2 seconds might be too long though, so let's compare the time for different batch sizes and using TensorFlows XLA package for more speed."
       ]
     },
     {
@@ -1551,7 +1551,7 @@
         "colab_type": "text"
       },
       "source": [
-        "First of all, it can be noted that XLA reduces latency time by a factor of ca. 1.3 (which is more than observed for other models by Tensorflow [here](https://www.tensorflow.org/xla)). A batch size of 64 looks like a good choice. More or less half a second for the forward pass is good enough.\n",
+        "First of all, it can be noted that XLA reduces latency time by a factor of ca. 1.3 (which is more than observed for other models by TensorFlow [here](https://www.tensorflow.org/xla)). A batch size of 64 looks like a good choice. More or less half a second for the forward pass is good enough.\n",
         "\n",
         "Cool, now it should be straightforward to benchmark your favorite models. All the inference time measurements can also be done using the `run_benchmark.py` script for PyTorch."
       ]
@@ -2021,4 +2021,4 @@
       ]
     }
   ]
-}
+}

notebooks/README.md

@@ -39,3 +39,4 @@ Pull Request so it can be included under the Community notebooks.
 |[Fine-tune BERT for Multi-label Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|How to fine-tune BERT for multi-label classification using PyTorch|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|
 |[Fine-tune T5 for Summarization](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|How to fine-tune T5 for summarization in PyTorch and track experiments with WandB|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|
 |[Speed up Fine-Tuning in Transformers with Dynamic Padding / Bucketing](https://github.com/ELS-RD/transformers-notebook/blob/master/Divide_Hugging_Face_Transformers_training_time_by_2_or_more.ipynb)|How to speed up fine-tuning by a factor of 2 using dynamic padding / bucketing|[Michael Benesty](https://github.com/pommedeterresautee) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1CBfRU1zbfu7-ijiOqAAQUA-RJaxfcJoO?usp=sharing)|
+|[Pretrain Reformer for Masked Language Modeling](https://github.com/patrickvonplaten/notebooks/blob/master/Reformer_For_Masked_LM.ipynb)| How to train a Reformer model with bi-directional self-attention layers | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1tzzh0i8PgDQGV3SMFUGxM7_gGae3K-uW?usp=sharing)|

setup.py

@@ -36,7 +36,7 @@ To create the package for pypi.
 
 7. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory.
 
-8. Update the documentation commit in .circleci/deploy.sh for the accurate documentation to be displayed
+8. Add the release version to docs/source/_static/js/custom.js and .circleci/deploy.sh
 
 9. Update README.md to redirect to correct documentation.
 """
@@ -65,19 +65,23 @@ if stale_egg_info.exists():
 
 extras = {}
 
-extras["mecab"] = ["mecab-python3"]
+extras["mecab"] = ["mecab-python3<1"]
 extras["sklearn"] = ["scikit-learn"]
 
 # keras2onnx and onnxconverter-common version is specific through a commit until 1.7.0 lands on pypi
 extras["tf"] = [
     "tensorflow",
-    "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
-    "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx"
+    "onnxconverter-common",
+    "keras2onnx"
+    # "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
+    # "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx"
 ]
 extras["tf-cpu"] = [
     "tensorflow-cpu",
-    "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
-    "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx"
+    "onnxconverter-common",
+    "keras2onnx"
+    # "onnxconverter-common @ git+git://github.com/microsoft/onnxconverter-common.git@f64ca15989b6dc95a1f3507ff6e4c395ba12dff5#egg=onnxconverter-common",
+    # "keras2onnx @ git+git://github.com/onnx/keras-onnx.git@cbdc75cb950b16db7f0a67be96a278f8d2953b48#egg=keras2onnx"
 ]
 extras["torch"] = ["torch"]
 
@@ -89,14 +93,15 @@ extras["testing"] = ["pytest", "pytest-xdist", "timeout-decorator", "psutil"]
 extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", "sphinx-copybutton"]
 extras["quality"] = [
     "black",
-    "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
+    "isort",
+    # "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
     "flake8",
 ]
-extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3", "scikit-learn", "tensorflow", "torch"]
+extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3<1", "scikit-learn", "tensorflow", "torch"]
 
 setup(
     name="transformers",
-    version="2.11.0",
+    version="3.0.2",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
@@ -109,7 +114,7 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.8.0-rc3",
+        "tokenizers == 0.8.1.rc1",
         # dataclasses for Python versions that don't have it
         "dataclasses;python_version<'3.7'",
         # utilities from PyPA to e.g. compare versions
@@ -123,7 +128,7 @@ setup(
         # for OpenAI GPT
         "regex != 2019.12.17",
         # for XLNet
-        "sentencepiece",
+        "sentencepiece != 0.1.92",
         # for XLM
         "sacremoses",
     ],

src/transformers/__init__.py

@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-__version__ = "2.11.0"
+__version__ = "3.0.2"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -155,7 +155,7 @@ from .tokenization_xlm_roberta import XLMRobertaTokenizer
 from .tokenization_xlnet import SPIECE_UNDERLINE, XLNetTokenizer
 
 # Trainer
-from .trainer_utils import EvalPrediction
+from .trainer_utils import EvalPrediction, set_seed
 from .training_args import TrainingArguments
 from .training_args_tf import TFTrainingArguments
 
@@ -169,8 +169,8 @@ if is_sklearn_available():
 
 # Modeling
 if is_torch_available():
+    from .generation_utils import top_k_top_p_filtering
     from .modeling_utils import PreTrainedModel, prune_layer, Conv1D, apply_chunking_to_forward
-    from .modeling_generation_utils import top_k_top_p_filtering
     from .modeling_auto import (
         AutoModel,
         AutoModelForPreTraining,
@@ -366,7 +366,9 @@ if is_torch_available():
         ReformerAttention,
         ReformerLayer,
         ReformerModel,
+        ReformerForMaskedLM,
         ReformerModelWithLMHead,
+        ReformerForQuestionAnswering,
         REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
     )
 
@@ -397,7 +399,7 @@ if is_torch_available():
     )
 
     # Trainer
-    from .trainer import Trainer, set_seed, torch_distributed_zero_first, EvalPrediction
+    from .trainer import Trainer, torch_distributed_zero_first
     from .data.data_collator import default_data_collator, DataCollator, DataCollatorForLanguageModeling
     from .data.datasets import GlueDataset, TextDataset, LineByLineTextDataset, GlueDataTrainingArguments
 
@@ -407,11 +409,9 @@ if is_torch_available():
 
 # TensorFlow
 if is_tf_available():
-    from .modeling_tf_generation_utils import (
-        shape_list,
-        tf_top_k_top_p_filtering,
-    )
+    from .generation_tf_utils import tf_top_k_top_p_filtering
     from .modeling_tf_utils import (
+        shape_list,
         TFPreTrainedModel,
         TFSequenceSummary,
         TFSharedEmbeddings,
@@ -616,8 +616,8 @@ if is_tf_available():
     from .trainer_tf import TFTrainer
 
     # Benchmarks
-    from .benchmark.benchmark_tf import TensorflowBenchmark
-    from .benchmark.benchmark_args_tf import TensorflowBenchmarkArguments
+    from .benchmark.benchmark_tf import TensorFlowBenchmark
+    from .benchmark.benchmark_args_tf import TensorFlowBenchmarkArguments
 
 
 if not is_tf_available() and not is_torch_available():

src/transformers/benchmark/benchmark_args_tf.py

@@ -30,7 +30,7 @@ logger = logging.getLogger(__name__)
 
 
 @dataclass
-class TensorflowBenchmarkArguments(BenchmarkArguments):
+class TensorFlowBenchmarkArguments(BenchmarkArguments):
     tpu_name: str = field(
         default=None, metadata={"help": "Name of TPU"},
     )

src/transformers/benchmark/benchmark_tf.py

@@ -38,7 +38,7 @@ from .benchmark_utils import (
 
 if is_tf_available():
     import tensorflow as tf
-    from .benchmark_args_tf import TensorflowBenchmarkArguments
+    from .benchmark_args_tf import TensorFlowBenchmarkArguments
     from tensorflow.python.framework.errors_impl import ResourceExhaustedError
 
 if is_py3nvml_available():
@@ -75,11 +75,11 @@ def random_input_ids(batch_size: int, sequence_length: int, vocab_size: int) ->
     return tf.constant(values, shape=(batch_size, sequence_length), dtype=tf.int32)
 
 
-class TensorflowBenchmark(Benchmark):
+class TensorFlowBenchmark(Benchmark):
 
-    args: TensorflowBenchmarkArguments
+    args: TensorFlowBenchmarkArguments
     configs: PretrainedConfig
-    framework: str = "Tensorflow"
+    framework: str = "TensorFlow"
 
     @property
     def framework_version(self):
@@ -88,7 +88,7 @@ class TensorflowBenchmark(Benchmark):
     def _inference_speed(self, model_name: str, batch_size: int, sequence_length: int) -> float:
         # initialize GPU on separate process
         strategy = self.args.strategy
-        assert strategy is not None, "A device strategy has to be initialized before using Tensorflow."
+        assert strategy is not None, "A device strategy has to be initialized before using TensorFlow."
         _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
         return self._measure_speed(_inference)
 
@@ -104,7 +104,7 @@ class TensorflowBenchmark(Benchmark):
         if self.args.is_gpu:
             tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx], True)
         strategy = self.args.strategy
-        assert strategy is not None, "A device strategy has to be initialized before using Tensorflow."
+        assert strategy is not None, "A device strategy has to be initialized before using TensorFlow."
         _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
         return self._measure_memory(_inference)
 
@@ -166,7 +166,7 @@ class TensorflowBenchmark(Benchmark):
 
     def _measure_memory(self, func: Callable[[], None]) -> [Memory, MemorySummary]:
         logger.info(
-            "Note that Tensorflow allocates more memory than"
+            "Note that TensorFlow allocates more memory than"
             "it might need to speed up computation."
             "The memory reported here corresponds to the memory"
             "reported by `nvidia-smi`, which can vary depending"
@@ -210,7 +210,7 @@ class TensorflowBenchmark(Benchmark):
                     # cpu
                     if self.args.trace_memory_line_by_line:
                         logger.info(
-                            "When enabling line by line tracing, the max peak memory for CPU is inaccurate in Tensorflow."
+                            "When enabling line by line tracing, the max peak memory for CPU is inaccurate in TensorFlow."
                         )
                         memory = None
                     else: