Checking output format + check raises ValueError (#8986)

* Slightly increase tolerance between pytorch and flax output

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* test_multiple_sentences doesn't require torch

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Simplify parameterization on "jit" to use boolean rather than str

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Use `require_torch` on `test_multiple_sentences` because we pull the weight from the hub.

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Rename "jit" parameter to "use_jit" for (hopefully) making it self-documenting.

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Remove pytest.mark.parametrize which seems to fail in some circumstances

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Fix unused imports.

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Fix style.

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Give default parameters values for traced model.

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Review comment: Change sentences to sequences

Signed-off-by: Morgan Funtowicz <funtowiczmo@gmail.com>

* Apply suggestions from code review

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

.circleci/config.yml

@@ -88,7 +88,7 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - skip-job-on-doc-only-changes
+            # - skip-job-on-doc-only-changes
             - restore_cache:
                   keys:
                       - v0.4-torch_and_tf-{{ checksum "setup.py" }}
@@ -115,7 +115,7 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - skip-job-on-doc-only-changes
+            # - skip-job-on-doc-only-changes
             - restore_cache:
                   keys:
                       - v0.4-torch-{{ checksum "setup.py" }}
@@ -142,7 +142,7 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - skip-job-on-doc-only-changes
+            # - skip-job-on-doc-only-changes
             - restore_cache:
                   keys:
                       - v0.4-tf-{{ checksum "setup.py" }}
@@ -169,7 +169,7 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - skip-job-on-doc-only-changes
+            # - skip-job-on-doc-only-changes
             - restore_cache:
                 keys:
                     - v0.4-flax-{{ checksum "setup.py" }}
@@ -196,7 +196,7 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - skip-job-on-doc-only-changes
+            # - skip-job-on-doc-only-changes
             - restore_cache:
                   keys:
                       - v0.4-torch-{{ checksum "setup.py" }}
@@ -223,7 +223,7 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - skip-job-on-doc-only-changes
+            # - skip-job-on-doc-only-changes
             - restore_cache:
                   keys:
                       - v0.4-tf-{{ checksum "setup.py" }}
@@ -248,7 +248,7 @@ jobs:
             RUN_CUSTOM_TOKENIZERS: yes
         steps:
             - checkout
-            - skip-job-on-doc-only-changes
+            # - skip-job-on-doc-only-changes
             - restore_cache:
                   keys:
                       - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
@@ -276,7 +276,7 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - skip-job-on-doc-only-changes
+            # - skip-job-on-doc-only-changes
             - restore_cache:
                   keys:
                       - v0.4-torch_examples-{{ checksum "setup.py" }}

.circleci/deploy.sh

@@ -52,4 +52,5 @@ deploy_doc "4b3ee9c" v3.1.0
 deploy_doc "3ebb1b3" v3.2.0
 deploy_doc "0613f05" v3.3.1
 deploy_doc "eb0e0ce" v3.4.0
-deploy_doc "818878d" # v3.5.1 Latest stable release
+deploy_doc "818878d" v3.5.1
+deploy_doc "c781171" # v4.0.0 Latest stable release

README.md

@@ -198,6 +198,8 @@ ultilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/
 1. **[Other community models](https://huggingface.co/models)**, contributed by the [community](https://huggingface.co/users).
 1. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
 
+To cehck if each model has an implementation in PyTorch/TensorFlow/Flax or has an associated tokenizer backed by the 🤗 Tokenizers library, refer to [this table](https://huggingface.co/transformers/index.html#bigtable)
+
 These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations. You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html).
 
 

docs/source/_static/js/custom.js

@@ -1,14 +1,15 @@
 // These two things need to be updated at each release for the version selector.
 // Last stable version
-const stableVersion = "v3.5.0"
+const stableVersion = "v4.0.0"
 // Dictionary doc folder to label
 const versionMapping = {
     "master": "master",
-    "": "v3.5.0/v3.5.1",
+    "v4.0.0": "v4.0.0",
+    "v3.5.1": "v3.5.0/v3.5.1",
     "v3.4.0": "v3.4.0",
     "v3.3.1": "v3.3.0/v3.3.1",
     "v3.2.0": "v3.2.0",
-    "v3.1.0": "v3.1.0 (stable)",
+    "v3.1.0": "v3.1.0",
     "v3.0.2": "v3.0.0/v3.0.1/v3.0.2",
     "v2.11.0": "v2.11.0",
     "v2.10.0": "v2.10.0",

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'3.5.0'
+release = u'4.0.0'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/index.rst

@@ -172,6 +172,8 @@ and conversion utilities for the following models:
     <https://huggingface.co/users>`__.
 
 
+.. _bigtable:
+
 The table below represents the current support in the library for each of those models, whether they have a Python
 tokenizer (called "slow"). A "fast" tokenizer backed by the 🤗 Tokenizers library, whether they have support in PyTorch,
 TensorFlow and/or Flax.

docs/source/model_doc/conv_bert.rst

@@ -1,81 +0,0 @@
-CONV_BERT
------------------------------------------------------------------------------------------------------------------------
-
-Overview
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The ConvBERT model was proposed in `<INSERT PAPER NAME HERE> <<INSERT PAPER LINK HERE>>`__ by <INSERT AUTHORS HERE>.
-<INSERT SHORT SUMMARY HERE>
-
-The abstract from the paper is the following:
-
-*<INSERT PAPER ABSTRACT HERE>*
-
-Tips:
-
-<INSERT TIPS ABOUT MODEL HERE>
-
-ConvBertConfig
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ConvBertConfig
-    :members:
-
-
-ConvBertTokenizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ConvBertTokenizer
-    :members: build_inputs_with_special_tokens, get_special_tokens_mask,
-        create_token_type_ids_from_sequences, save_vocabulary
-
-
-ConvBertTokenizerFast
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ConvBertTokenizerFast
-    :members: build_inputs_with_special_tokens, get_special_tokens_mask,
-        create_token_type_ids_from_sequences, save_vocabulary
-
-
-ConvBertModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ConvBertModel
-    :members: forward
-
-
-ConvBertForMaskedLM
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ConvBertForMaskedLM
-    :members: forward
-
-
-ConvBertForSequenceClassification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ConvBertForSequenceClassification
-    :members: forward
-
-
-ConvBertForMultipleChoice
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ConvBertForMultipleChoice
-    :members:
-
-
-ConvBertForTokenClassification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ConvBertForTokenClassification
-    :members: forward
-
-
-ConvBertForQuestionAnswering
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.ConvBertForQuestionAnswering
-    :members: forward
-

examples/seq2seq/builtin_trainer/finetune.sh

@@ -3,7 +3,8 @@
 python finetune_trainer.py \
     --learning_rate=3e-5 \
     --fp16 \
-    --do_train --do_eval --do_predict --evaluate_during_training \
+    --do_train --do_eval --do_predict \
+    --evaluation_strategy steps \
     --predict_with_generate \
     --n_val 1000 \
     "$@"

examples/seq2seq/builtin_trainer/finetune_tpu.sh

@@ -5,7 +5,8 @@ export TPU_NUM_CORES=8
 python xla_spawn.py --num_cores $TPU_NUM_CORES \
     finetune_trainer.py \
     --learning_rate=3e-5 \
-    --do_train --do_eval --evaluate_during_training \
+    --do_train --do_eval \
+    --evaluation_strategy steps \
     --prediction_loss_only \
     --n_val 1000 \
     "$@"

examples/seq2seq/builtin_trainer/train_distil_marian_enro.sh

@@ -16,7 +16,8 @@ python finetune_trainer.py \
     --num_train_epochs=6 \
     --save_steps 3000 --eval_steps 3000 \
     --max_source_length $MAX_LEN --max_target_length $MAX_LEN --val_max_target_length $MAX_LEN --test_max_target_length $MAX_LEN \
-    --do_train --do_eval --do_predict --evaluate_during_training\
+    --do_train --do_eval --do_predict \
+    --evaluation_strategy steps \
     --predict_with_generate --logging_first_step \
     --task translation --label_smoothing 0.1 \
     "$@"

examples/seq2seq/builtin_trainer/train_distil_marian_enro_tpu.sh

@@ -17,7 +17,8 @@ python xla_spawn.py --num_cores $TPU_NUM_CORES \
     --save_steps 500 --eval_steps 500 \
     --logging_first_step --logging_steps 200 \
     --max_source_length $MAX_LEN --max_target_length $MAX_LEN --val_max_target_length $MAX_LEN --test_max_target_length $MAX_LEN \
-    --do_train --do_eval --evaluate_during_training \
+    --do_train --do_eval \
+    --evaluation_strategy steps \
     --prediction_loss_only \
     --task translation --label_smoothing 0.1 \
     "$@"

examples/seq2seq/builtin_trainer/train_distilbart_cnn.sh

@@ -19,6 +19,7 @@ python finetune_trainer.py \
     --save_steps 3000 --eval_steps 3000 \
     --logging_first_step \
     --max_target_length 56 --val_max_target_length $MAX_TGT_LEN --test_max_target_length $MAX_TGT_LEN \
-    --do_train --do_eval --do_predict --evaluate_during_training \
+    --do_train --do_eval --do_predict \
+    --evaluation_strategy steps \
     --predict_with_generate --sortish_sampler \
     "$@"

examples/seq2seq/builtin_trainer/train_mbart_cc25_enro.sh

@@ -15,7 +15,8 @@ python finetune_trainer.py \
     --sortish_sampler \
     --num_train_epochs 6 \
     --save_steps 25000 --eval_steps 25000 --logging_steps 1000 \
-    --do_train --do_eval --do_predict --evaluate_during_training \
+    --do_train --do_eval --do_predict \
+    --evaluation_strategy steps \
     --predict_with_generate --logging_first_step \
     --task translation \
     "$@"

examples/token-classification/run_ner.py

@@ -369,7 +369,7 @@ def main():
         ]
 
         output_test_results_file = os.path.join(training_args.output_dir, "test_results.txt")
-        if trainer.is_world_master():
+        if trainer.is_world_process_zero():
             with open(output_test_results_file, "w") as writer:
                 for key, value in metrics.items():
                     logger.info(f"  {key} = {value}")
@@ -377,7 +377,7 @@ def main():
 
         # Save predictions
         output_test_predictions_file = os.path.join(training_args.output_dir, "test_predictions.txt")
-        if trainer.is_world_master():
+        if trainer.is_world_process_zero():
             with open(output_test_predictions_file, "w") as writer:
                 for prediction in true_predictions:
                     writer.write(" ".join(prediction) + "\n")

examples/token-classification/run_ner_old.py

@@ -291,7 +291,7 @@ def main():
         preds_list, _ = align_predictions(predictions, label_ids)
 
         output_test_results_file = os.path.join(training_args.output_dir, "test_results.txt")
-        if trainer.is_world_master():
+        if trainer.is_world_process_zero():
             with open(output_test_results_file, "w") as writer:
                 for key, value in metrics.items():
                     logger.info("  %s = %s", key, value)
@@ -299,7 +299,7 @@ def main():
 
         # Save predictions
         output_test_predictions_file = os.path.join(training_args.output_dir, "test_predictions.txt")
-        if trainer.is_world_master():
+        if trainer.is_world_process_zero():
             with open(output_test_predictions_file, "w") as writer:
                 with open(os.path.join(data_args.data_dir, "test.txt"), "r") as f:
                     token_classification_task.write_predictions_to_file(writer, f, preds_list)

setup.py

@@ -230,7 +230,7 @@ install_requires = [
 
 setup(
     name="transformers",
-    version="4.0.0-rc-1",
+    version="4.1.0.dev0",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Sylvain Gugger, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",

src/transformers/__init__.py

@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-__version__ = "4.0.0-rc-1"
+__version__ = "4.1.0.dev0"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -113,7 +113,6 @@ from .models.blenderbot import (
     BlenderbotTokenizer,
 )
 from .models.camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
-from .models.conv_bert import CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig
 from .models.ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig, CTRLTokenizer
 from .models.deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig, DebertaTokenizer
 from .models.distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig, DistilBertTokenizer
@@ -392,18 +391,6 @@ if is_torch_available():
         CamembertForTokenClassification,
         CamembertModel,
     )
-    from .models.conv_bert import (
-        CONV_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        ConvBertForMaskedLM,
-        ConvBertForMultipleChoice,
-        ConvBertForQuestionAnswering,
-        ConvBertForSequenceClassification,
-        ConvBertForTokenClassification,
-        ConvBertLayer,
-        ConvBertModel,
-        ConvBertPreTrainedModel,
-        load_tf_weights_in_conv_bert,
-    )
     from .models.ctrl import CTRL_PRETRAINED_MODEL_ARCHIVE_LIST, CTRLLMHeadModel, CTRLModel, CTRLPreTrainedModel
     from .models.deberta import (
         DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,

src/transformers/convert_pytorch_checkpoint_to_tf2.py

@@ -302,7 +302,7 @@ def convert_pt_checkpoint_to_tf(
         pytorch_checkpoint_url = hf_bucket_url(pytorch_checkpoint_path, filename=WEIGHTS_NAME)
         pytorch_checkpoint_path = cached_path(pytorch_checkpoint_url, force_download=not use_cached_models)
     # Load PyTorch checkpoint in tf2 model:
-    tf_model = load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, config)
+    tf_model = load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path)
 
     if compare_with_pt_model:
         tfo = tf_model(tf_model.dummy_inputs, training=False)  # build the network

src/transformers/integrations.py

@@ -2,6 +2,7 @@
 import math
 import os
 
+from .trainer_utils import EvaluationStrategy
 from .utils import logging
 
 
@@ -212,13 +213,13 @@ def run_hp_search_ray(trainer, n_trials: int, direction: str, **kwargs) -> BestR
         # Check for `do_eval` and `eval_during_training` for schedulers that require intermediate reporting.
         if isinstance(
             kwargs["scheduler"], (ASHAScheduler, MedianStoppingRule, HyperBandForBOHB, PopulationBasedTraining)
-        ) and (not trainer.args.do_eval or not trainer.args.evaluate_during_training):
+        ) and (not trainer.args.do_eval or trainer.args.evaluation_strategy == EvaluationStrategy.NO):
             raise RuntimeError(
                 "You are using {cls} as a scheduler but you haven't enabled evaluation during training. "
                 "This means your trials will not report intermediate results to Ray Tune, and "
                 "can thus not be stopped early or used to exploit other trials parameters. "
                 "If this is what you want, do not use {cls}. If you would like to use {cls}, "
-                "make sure you pass `do_eval=True` and `evaluate_during_training=True` in the "
+                "make sure you pass `do_eval=True` and `evaluation_strategy='steps'` in the "
                 "Trainer `args`.".format(cls=type(kwargs["scheduler"]).__name__)
             )
 

src/transformers/modeling_tf_pytorch_utils.py

@@ -43,13 +43,6 @@ def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove="")
           other
     """
     tf_name = tf_name.replace(":0", "")  # device ids
-
-    # support for ConvBERT model
-    tf_name_split = tf_name.split("/")
-    if tf_name_split[-1] == "depthwise_kernel":
-        tf_name = "/".join(tf_name_split[:-1]) + "/depthwise/weight"
-    if tf_name_split[-1] == "pointwise_kernel":
-        tf_name = "/".join(tf_name_split[:-1]) + "/pointwise/weight"
     tf_name = re.sub(
         r"/[^/]*___([^/]*)/", r"/\1/", tf_name
     )  # '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch)
@@ -73,6 +66,7 @@ def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove="")
     tf_name = ".".join(tf_name)
     if start_prefix_to_remove:
         tf_name = tf_name.replace(start_prefix_to_remove, "", 1)
+
     return tf_name, transpose
 
 
@@ -81,9 +75,7 @@ def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove="")
 #####################
 
 
-def load_pytorch_checkpoint_in_tf2_model(
-    tf_model, pytorch_checkpoint_path, config, tf_inputs=None, allow_missing_keys=False
-):
+def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=None, allow_missing_keys=False):
     """Load pytorch checkpoints in a TF 2.0 model"""
     try:
         import tensorflow as tf  # noqa: F401
@@ -102,7 +94,7 @@ def load_pytorch_checkpoint_in_tf2_model(
     logger.info("PyTorch checkpoint contains {:,} parameters".format(sum(t.numel() for t in pt_state_dict.values())))
 
     return load_pytorch_weights_in_tf2_model(
-        tf_model, pt_state_dict, config=config, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys
+        tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys
     )
 
 
@@ -115,7 +107,7 @@ def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=None, allow_mi
     )
 
 
-def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, config=None, tf_inputs=None, allow_missing_keys=False):
+def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, allow_missing_keys=False):
     """Load pytorch state_dict in a TF 2.0 model."""
     try:
         import tensorflow as tf  # noqa: F401
@@ -181,25 +173,6 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, config=None, tf_i
 
         array = pt_state_dict[name].numpy()
 
-        if config.model_type == "conv_bert":
-            if name.endswith("depthwise.weight"):
-                array = numpy.transpose(array, axes=(2, 0, 1))
-                transpose = False
-
-            if name.endswith("pointwise.weight"):
-                array = numpy.transpose(array, axes=(2, 1, 0))
-                transpose = False
-
-            if name.endswith("conv_attn_key.bias"):
-                array = numpy.squeeze(array)
-                transpose = False
-
-            if config.num_groups > 1:
-                if name.endswith("intermediate.dense.weight") or name.endswith("output.dense.weight"):
-                    if not name.endswith("attention.output.dense.weight"):
-                        print(name)
-                        transpose = False
-
         if transpose:
             array = numpy.transpose(array)
 

src/transformers/modeling_tf_utils.py

@@ -841,7 +841,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin):
             from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model
 
             # Load from a PyTorch checkpoint
-            return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, config, allow_missing_keys=True)
+            return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, allow_missing_keys=True)
 
         model(model.dummy_inputs, training=False)  # build the network with dummy inputs
 

src/transformers/models/auto/configuration_auto.py

@@ -24,7 +24,6 @@ from ..bert.configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertCo
 from ..bert_generation.configuration_bert_generation import BertGenerationConfig
 from ..blenderbot.configuration_blenderbot import BLENDERBOT_PRETRAINED_CONFIG_ARCHIVE_MAP, BlenderbotConfig
 from ..camembert.configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
-from ..conv_bert.configuration_conv_bert import CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig
 from ..ctrl.configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
 from ..deberta.configuration_deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig
 from ..distilbert.configuration_distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig
@@ -65,7 +64,6 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
     (key, value)
     for pretrained_map in [
         # Add archive maps here
-        CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         BART_PRETRAINED_CONFIG_ARCHIVE_MAP,
         BLENDERBOT_PRETRAINED_CONFIG_ARCHIVE_MAP,
@@ -103,7 +101,6 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
 CONFIG_MAPPING = OrderedDict(
     [
         # Add configs here
-        ("conv_bert", ConvBertConfig),
         ("retribert", RetriBertConfig),
         ("mt5", MT5Config),
         ("t5", T5Config),
@@ -147,7 +144,6 @@ CONFIG_MAPPING = OrderedDict(
 MODEL_NAMES_MAPPING = OrderedDict(
     [
         # Add full (and cased) model names here
-        ("conv_bert", "ConvBert"),
         ("retribert", "RetriBERT"),
         ("t5", "T5"),
         ("mobilebert", "MobileBERT"),

src/transformers/models/auto/modeling_auto.py

@@ -21,6 +21,8 @@ from collections import OrderedDict
 from ...configuration_utils import PretrainedConfig
 from ...file_utils import add_start_docstrings
 from ...utils import logging
+
+# Add modeling imports here
 from ..albert.modeling_albert import (
     AlbertForMaskedLM,
     AlbertForMultipleChoice,
@@ -58,14 +60,6 @@ from ..camembert.modeling_camembert import (
     CamembertForTokenClassification,
     CamembertModel,
 )
-from ..conv_bert.modeling_conv_bert import (
-    ConvBertForMaskedLM,
-    ConvBertForMultipleChoice,
-    ConvBertForQuestionAnswering,
-    ConvBertForSequenceClassification,
-    ConvBertForTokenClassification,
-    ConvBertModel,
-)
 from ..ctrl.modeling_ctrl import CTRLLMHeadModel, CTRLModel
 from ..deberta.modeling_deberta import DebertaForSequenceClassification, DebertaModel
 from ..distilbert.modeling_distilbert import (
@@ -202,7 +196,6 @@ from .configuration_auto import (
     BertGenerationConfig,
     BlenderbotConfig,
     CamembertConfig,
-    ConvBertConfig,
     CTRLConfig,
     DebertaConfig,
     DistilBertConfig,
@@ -237,17 +230,12 @@ from .configuration_auto import (
 )
 
 
-# Add modeling imports here
-
-
-
 logger = logging.get_logger(__name__)
 
 
 MODEL_MAPPING = OrderedDict(
     [
         # Base model mapping
-        (ConvBertConfig, ConvBertModel),
         (RetriBertConfig, RetriBertModel),
         (MT5Config, MT5Model),
         (T5Config, T5Model),
@@ -315,7 +303,6 @@ MODEL_FOR_PRETRAINING_MAPPING = OrderedDict(
 MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
     [
         # Model with LM heads mapping
-        (ConvBertConfig, ConvBertForMaskedLM),
         (LayoutLMConfig, LayoutLMForMaskedLM),
         (T5Config, T5ForConditionalGeneration),
         (DistilBertConfig, DistilBertForMaskedLM),
@@ -370,7 +357,6 @@ MODEL_FOR_CAUSAL_LM_MAPPING = OrderedDict(
 MODEL_FOR_MASKED_LM_MAPPING = OrderedDict(
     [
         # Model for Masked LM mapping
-        (ConvBertConfig, ConvBertForMaskedLM),
         (LayoutLMConfig, LayoutLMForMaskedLM),
         (DistilBertConfig, DistilBertForMaskedLM),
         (AlbertConfig, AlbertForMaskedLM),
@@ -410,7 +396,6 @@ MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
 MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
     [
         # Model for Sequence Classification mapping
-        (ConvBertConfig, ConvBertForSequenceClassification),
         (DistilBertConfig, DistilBertForSequenceClassification),
         (AlbertConfig, AlbertForSequenceClassification),
         (CamembertConfig, CamembertForSequenceClassification),
@@ -436,7 +421,6 @@ MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
 MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
     [
         # Model for Question Answering mapping
-        (ConvBertConfig, ConvBertForQuestionAnswering),
         (DistilBertConfig, DistilBertForQuestionAnswering),
         (AlbertConfig, AlbertForQuestionAnswering),
         (CamembertConfig, CamembertForQuestionAnswering),
@@ -460,7 +444,6 @@ MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
 MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = OrderedDict(
     [
         # Model for Token Classification mapping
-        (ConvBertConfig, ConvBertForTokenClassification),
         (LayoutLMConfig, LayoutLMForTokenClassification),
         (DistilBertConfig, DistilBertForTokenClassification),
         (CamembertConfig, CamembertForTokenClassification),
@@ -483,7 +466,6 @@ MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = OrderedDict(
 MODEL_FOR_MULTIPLE_CHOICE_MAPPING = OrderedDict(
     [
         # Model for Multiple Choice mapping
-        (ConvBertConfig, ConvBertForMultipleChoice),
         (CamembertConfig, CamembertForMultipleChoice),
         (ElectraConfig, ElectraForMultipleChoice),
         (XLMRobertaConfig, XLMRobertaForMultipleChoice),

src/transformers/models/conv_bert/__init__.py

@@ -1,21 +0,0 @@
-# flake8: noqa
-# There's no way to ignore "F401 '...' imported but unused" warnings in this
-# module, but to preserve other warnings. So, don't check this module at all.
-from ...file_utils import is_torch_available
-from .configuration_conv_bert import CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig
-from .tokenization_conv_bert import ConvBertTokenizer
-
-
-if is_torch_available():
-    from .modeling_conv_bert import (
-        CONV_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
-        ConvBertForMaskedLM,
-        ConvBertForMultipleChoice,
-        ConvBertForQuestionAnswering,
-        ConvBertForSequenceClassification,
-        ConvBertForTokenClassification,
-        ConvBertLayer,
-        ConvBertModel,
-        ConvBertPreTrainedModel,
-        load_tf_weights_in_conv_bert,
-    )

src/transformers/models/conv_bert/configuration_conv_bert.py

@@ -1,126 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" ConvBERT model configuration """
-
-from ...configuration_utils import PretrainedConfig
-from ...utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "convbert-base-uncased": "https://huggingface.co/convbert-base-uncased/resolve/main/config.json",
-    # See all ConvBERT models at https://huggingface.co/models?filter=conv_bert
-}
-
-
-class ConvBertConfig(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a :class:`~transformers.ConvBertModel`. It is used to
-    instantiate an ConvBERT model according to the specified arguments, defining the model architecture. Instantiating
-    a configuration with the defaults will yield a similar configuration to that of the ConvBERT `convbert-base-uncased
-    <https://huggingface.co/convbert-base-uncased>`__ architecture.
-
-    Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model
-    outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information.
-
-
-    Args:
-        vocab_size (:obj:`int`, `optional`, defaults to 30522):
-            Vocabulary size of the ConvBERT model. Defines the number of different tokens that can be represented by
-            the :obj:`inputs_ids` passed when calling :class:`~transformers.ConvBertModel` or
-            :class:`~transformers.TFConvBertModel`. Vocabulary size of the model. Defines the different tokens that can
-            be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.ConvBertModel`.
-        hidden_size (:obj:`int`, `optional`, defaults to 768):
-            Dimensionality of the encoder layers and the pooler layer.
-        num_hidden_layers (:obj:`int`, `optional`, defaults to 12):
-            Number of hidden layers in the Transformer encoder.
-        num_attention_heads (:obj:`int`, `optional`, defaults to 12):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        intermediate_size (:obj:`int`, `optional`, defaults to 3072):
-            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
-        hidden_act (:obj:`str` or :obj:`function`, `optional`, defaults to :obj:`"gelu"`):
-            The non-linear activation function (function or string) in the encoder and pooler. If string,
-            :obj:`"gelu"`, :obj:`"relu"`, :obj:`"selu"` and :obj:`"gelu_new"` are supported.
-        hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1):
-            The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
-        attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1):
-            The dropout ratio for the attention probabilities.
-        max_position_embeddings (:obj:`int`, `optional`, defaults to 512):
-            The maximum sequence length that this model might ever be used with. Typically set this to something large
-            just in case (e.g., 512 or 1024 or 2048).
-        type_vocab_size (:obj:`int`, `optional`, defaults to 2):
-            The vocabulary size of the :obj:`token_type_ids` passed when calling :class:`~transformers.ConvBertModel`
-            or :class:`~transformers.TFConvBertModel`.
-        initializer_range (:obj:`float`, `optional`, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        layer_norm_eps (:obj:`float`, `optional`, defaults to 1e-12):
-            The epsilon used by the layer normalization layers.
-
-    Example::
-
-        >>> from transformers import ConvBertModel, ConvBertConfig
-
-        >>> # Initializing a ConvBERT convbert-base-uncased style configuration
-        >>> configuration = ConvBertConfig()
-
-        >>> # Initializing a model from the convbert-base-uncased style configuration
-        >>> model = ConvBertModel(configuration)
-
-        >>> # Accessing the model configuration
-        >>> configuration = model.config
-    """
-    model_type = "conv_bert"
-
-    def __init__(
-        self,
-        vocab_size=30522,
-        hidden_size=768,
-        is_encoder_decoder=False,
-        num_hidden_layers=12,
-        num_attention_heads=12,
-        intermediate_size=3072,
-        hidden_act="gelu",
-        hidden_dropout_prob=0.1,
-        attention_probs_dropout_prob=0.1,
-        max_position_embeddings=512,
-        type_vocab_size=2,
-        initializer_range=0.02,
-        layer_norm_eps=1e-12,
-        pad_token_id=1,
-        bos_token_id=0,
-        eos_token_id=2,
-        **kwargs
-    ):
-        super().__init__(
-            pad_token_id=pad_token_id,
-            is_encoder_decoder=is_encoder_decoder,
-            bos_token_id=bos_token_id,
-            eos_token_id=eos_token_id,
-            **kwargs,
-        )
-
-        self.vocab_size = vocab_size
-        self.hidden_size = hidden_size
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.intermediate_size = intermediate_size
-        self.hidden_act = hidden_act
-        self.hidden_dropout_prob = hidden_dropout_prob
-        self.attention_probs_dropout_prob = attention_probs_dropout_prob
-        self.max_position_embeddings = max_position_embeddings
-        self.type_vocab_size = type_vocab_size
-        self.initializer_range = initializer_range
-        self.layer_norm_eps = layer_norm_eps

src/transformers/models/conv_bert/convert_tf1.py

@@ -1,25 +0,0 @@
-import tensorflow as tf
-import torch
-
-from .modeling_tf_conv_bert import ConvBertConfig, TFConvBertModel
-from .tokenization_conv_bert import ConvBertTokenizer
-
-
-tf_checkpoint_path = "/home/abhishek/huggingface/models/convbert/model.ckpt"
-conf = ConvBertConfig.from_json_file("/home/abhishek/huggingface/models/convbert/config.json")
-tokenizer = ConvBertTokenizer.from_pretrained("/home/abhishek/huggingface/models/convbert/")
-
-
-if __name__ == "__main__":
-    init_vars = tf.train.list_variables(tf_checkpoint_path)
-    model = TFConvBertModel(conf)
-    tf_inputs = model.dummy_inputs
-    model(tf_inputs, training=False)
-    print(model.base_model_prefix)
-
-    weight_dict = {}
-    for name, shape in init_vars:
-        array = tf.train.load_variable(tf_checkpoint_path, name)
-        weight_dict[name] = array
-
-    print(model.summary())

src/transformers/models/conv_bert/param_mapping.py

@@ -1,85 +0,0 @@
-def fetch_mapping(config):
-    PARAM_MAPPING = {
-        "embeddings.word_embeddings.weight": "electra/embeddings/word_embeddings",
-        "embeddings.position_embeddings.weight": "electra/embeddings/position_embeddings",
-        "embeddings.token_type_embeddings.weight": "electra/embeddings/token_type_embeddings",
-        "embeddings.LayerNorm.weight": "electra/embeddings/LayerNorm/gamma",
-        "embeddings.LayerNorm.bias": "electra/embeddings/LayerNorm/beta",
-        "embeddings_project.weight": "electra/embeddings_project/kernel",
-        "embeddings_project.bias": "electra/embeddings_project/bias",
-    }
-    if config.num_groups > 1:
-        group_dense_name = "g_dense"
-    else:
-        group_dense_name = "dense"
-
-    for j in range(config.num_hidden_layers):
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.query.weight"
-        ] = f"electra/encoder/layer_{j}/attention/self/query/kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.query.bias"
-        ] = f"electra/encoder/layer_{j}/attention/self/query/bias"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.key.weight"
-        ] = f"electra/encoder/layer_{j}/attention/self/key/kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.key.bias"
-        ] = f"electra/encoder/layer_{j}/attention/self/key/bias"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.value.weight"
-        ] = f"electra/encoder/layer_{j}/attention/self/value/kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.value.bias"
-        ] = f"electra/encoder/layer_{j}/attention/self/value/bias"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.key_conv_attn_layer.depthwise.weight"
-        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_key/depthwise_kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.key_conv_attn_layer.pointwise.weight"
-        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_key/pointwise_kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.key_conv_attn_layer.bias"
-        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_key/bias"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.conv_kernel_layer.weight"
-        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_kernel/kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.conv_kernel_layer.bias"
-        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_kernel/bias"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.conv_out_layer.weight"
-        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_point/kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.self.conv_out_layer.bias"
-        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_point/bias"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.output.dense.weight"
-        ] = f"electra/encoder/layer_{j}/attention/output/dense/kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.output.LayerNorm.weight"
-        ] = f"electra/encoder/layer_{j}/attention/output/LayerNorm/gamma"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.output.dense.bias"
-        ] = f"electra/encoder/layer_{j}/attention/output/dense/bias"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.attention.output.LayerNorm.bias"
-        ] = f"electra/encoder/layer_{j}/attention/output/LayerNorm/beta"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.intermediate.dense.weight"
-        ] = f"electra/encoder/layer_{j}/intermediate/{group_dense_name}/kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.intermediate.dense.bias"
-        ] = f"electra/encoder/layer_{j}/intermediate/{group_dense_name}/bias"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.output.dense.weight"
-        ] = f"electra/encoder/layer_{j}/output/{group_dense_name}/kernel"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.output.dense.bias"
-        ] = f"electra/encoder/layer_{j}/output/{group_dense_name}/bias"
-        PARAM_MAPPING[
-            f"encoder.layer.{j}.output.LayerNorm.weight"
-        ] = f"electra/encoder/layer_{j}/output/LayerNorm/gamma"
-        PARAM_MAPPING[f"encoder.layer.{j}.output.LayerNorm.bias"] = f"electra/encoder/layer_{j}/output/LayerNorm/beta"
-
-    return PARAM_MAPPING

src/transformers/models/conv_bert/run_bert.py

@@ -1,31 +0,0 @@
-import os
-
-import tensorflow as tf
-import torch
-
-from ..bert.modeling_bert import BertConfig, BertModel, load_tf_weights_in_bert
-from ..bert.modeling_tf_bert import TFBertModel
-from ..bert.tokenization_bert import BertTokenizer
-
-
-if __name__ == "__main__":
-    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
-    # tf_checkpoint_path = os.path.join(model_path, "model.ckpt")
-    model = BertModel.from_pretrained("bert-base-uncased")
-
-    # model = load_tf_weights_in_conv_bert(model, conf, tf_checkpoint_path)
-    # model.save_pretrained(model_path)
-
-    tf_model = TFBertModel.from_pretrained("bert-base-uncased")
-    # tf_model.save_pretrained(model_path)
-
-    print(model.dummy_inputs)
-    print(model(**model.dummy_inputs))
-
-    # tf_model = TFConvBertModel(conf)
-
-    print(tf_model.dummy_inputs)
-    print(tf_model(tf_model.dummy_inputs))
-
-    # last_hidden_states = outputs.last_hidden_state
-    # print(last_hidden_states)

src/transformers/models/conv_bert/run_pt.py

@@ -1,26 +0,0 @@
-import os
-
-import tensorflow as tf
-import torch
-
-from .modeling_conv_bert import ConvBertConfig, ConvBertModel, load_tf_weights_in_conv_bert
-from .modeling_tf_conv_bert import TFConvBertModel
-from .tokenization_conv_bert import ConvBertTokenizer
-
-
-model_path = "/home/abhishek/huggingface/models/convbert_models/convbert_medium_small"
-
-if __name__ == "__main__":
-    conf = ConvBertConfig.from_json_file(os.path.join(model_path, "config.json"))
-    tokenizer = ConvBertTokenizer.from_pretrained(model_path)
-    tf_checkpoint_path = os.path.join(model_path, "model.ckpt")
-    model = ConvBertModel(conf)
-
-    model = load_tf_weights_in_conv_bert(model, conf, tf_checkpoint_path)
-    # model.save_pretrained(model_path)
-    model.eval()
-    # for p in model.named_parameters():
-    #     print(p)
-
-    print(model.dummy_inputs)
-    print(model(**model.dummy_inputs).last_hidden_state)

src/transformers/models/conv_bert/run_test.py

@@ -1,40 +0,0 @@
-import os
-
-import tensorflow as tf
-import torch
-
-from .modeling_conv_bert import ConvBertConfig, ConvBertForSequenceClassification, load_tf_weights_in_conv_bert
-from .modeling_tf_conv_bert import TFConvBertForSequenceClassification
-from .tokenization_conv_bert import ConvBertTokenizer
-
-
-model_path = "/home/abhishek/huggingface/models/convbert_models/convbert_medium_small"
-
-if __name__ == "__main__":
-    tokenizer = ConvBertTokenizer.from_pretrained(model_path)
-    model = ConvBertForSequenceClassification.from_pretrained(model_path)
-    tf_model = TFConvBertForSequenceClassification.from_pretrained(model_path)
-
-    print(model.dummy_inputs)
-    print(model(**model.dummy_inputs))
-
-    # tf_model = TFConvBertModel(conf)
-
-    print(tf_model.dummy_inputs)
-    print(tf_model(tf_model.dummy_inputs))
-
-    # last_hidden_states = outputs.last_hidden_state
-    # print(last_hidden_states)
-
-    import torch
-
-    from transformers import BertForSequenceClassification, BertTokenizer
-
-    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
-    model = BertForSequenceClassification.from_pretrained("bert-base-uncased", return_dict=True)
-    inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-    labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
-    outputs = model(**inputs, labels=labels)
-    loss = outputs.loss
-    logits = outputs.logits
-    print(logits)

src/transformers/models/conv_bert/run_tf.py

@@ -1,32 +0,0 @@
-import os
-
-import tensorflow as tf
-import torch
-
-from .modeling_conv_bert import ConvBertConfig, ConvBertModel, load_tf_weights_in_conv_bert
-from .modeling_tf_conv_bert import TFConvBertModel
-from .tokenization_conv_bert import ConvBertTokenizer
-
-
-model_path = "/home/abhishek/huggingface/models/convbert_models/convbert_medium_small"
-
-if __name__ == "__main__":
-    conf = ConvBertConfig.from_json_file(os.path.join(model_path, "config.json"))
-    tokenizer = ConvBertTokenizer.from_pretrained(model_path)
-    tf_checkpoint_path = os.path.join(model_path, "model.ckpt")
-    model = ConvBertModel(conf)
-
-    model = load_tf_weights_in_conv_bert(model, conf, tf_checkpoint_path)
-    # model.save_pretrained(model_path)
-
-    tf_model = TFConvBertModel.from_pretrained(model_path, from_pt=True)
-    tf_model.trainable = False
-    # tf_model.save_pretrained(model_path)
-
-    # symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights
-    # for symbolic_weight in symbolic_weights:
-    #     sw_name = symbolic_weight.name
-    #     print(symbolic_weight)
-
-    print(tf_model.dummy_inputs)
-    print(tf_model(tf_model.dummy_inputs).last_hidden_state)

src/transformers/models/conv_bert/tokenization_conv_bert.py

@@ -1,72 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Tokenization classes for ConvBERT."""
-from ...utils import logging
-from ..bert.tokenization_bert import BertTokenizer
-from ..bert.tokenization_bert_fast import BertTokenizerFast
-
-
-logger = logging.get_logger(__name__)
-
-VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
-
-PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {
-        "conv-bert-base-uncased": "https://huggingface.co/conv-bert-base-uncased/resolve/main/vocab.txt",
-    }
-}
-
-PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
-    "conv-bert-base-uncased": 512,
-}
-
-
-PRETRAINED_INIT_CONFIGURATION = {
-    "conv-bert-base-uncased": {"do_lower_case": False},
-}
-
-
-class ConvBertTokenizer(BertTokenizer):
-    r"""
-    Construct a ConvBERT tokenizer.
-
-    :class:`~transformers.ConvBertTokenizer` is identical to :class:`~transformers.BertTokenizer` and runs end-to-end
-    tokenization: punctuation splitting and wordpiece.
-
-    Refer to superclass :class:`~transformers.BertTokenizer` for usage examples and documentation concerning
-    parameters.
-    """
-
-    vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-
-
-class ConvBertTokenizerFast(BertTokenizerFast):
-    r"""
-    Construct a "fast" ConvBERT tokenizer (backed by HuggingFace's `tokenizers` library).
-
-    :class:`~transformers.ConvBertTokenizerFast` is identical to :class:`~transformers.BertTokenizerFast` and runs
-    end-to-end tokenization: punctuation splitting and wordpiece.
-
-    Refer to superclass :class:`~transformers.BertTokenizerFast` for usage examples and documentation concerning
-    parameters.
-    """
-
-    vocab_files_names = VOCAB_FILES_NAMES
-    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
-    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION

src/transformers/models/electra/modeling_tf_electra.py

@@ -136,8 +136,6 @@ class TFElectraSelfAttention(tf.keras.layers.Layer):
         context_layer = tf.reshape(
             context_layer, (batch_size, -1, self.all_head_size)
         )  # (batch_size, seq_len_q, all_head_size)
-        print(context_layer.shape)
-        print(attention_probs.shape)
         outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
 
         return outputs

src/transformers/models/longformer/modeling_longformer.py

@@ -459,6 +459,7 @@ class LongformerEmbeddings(nn.Module):
 
         # position_ids (1, len position emb) is contiguous in memory and exported when serialized
         self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)))
+        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
 
         self.padding_idx = config.pad_token_id
         self.position_embeddings = nn.Embedding(

src/transformers/pipelines.py

@@ -516,7 +516,7 @@ class Pipeline(_ScikitCompat):
         if framework is None:
             framework = get_framework(model)
 
-        self.task = task
+        self.task = task or getattr(self, "task", "")
         self.model = model
         self.tokenizer = tokenizer
         self.modelcard = modelcard
@@ -530,8 +530,8 @@ class Pipeline(_ScikitCompat):
 
         # Update config with task specific parameters
         task_specific_params = self.model.config.task_specific_params
-        if task_specific_params is not None and task in task_specific_params:
-            self.model.config.update(task_specific_params.get(task))
+        if task_specific_params is not None and self.task in task_specific_params:
+            self.model.config.update(task_specific_params.get(self.task))
 
     def save_pretrained(self, save_directory: str):
         """
@@ -926,6 +926,14 @@ class TextGenerationPipeline(Pipeline):
     r"""
         return_all_scores (:obj:`bool`, `optional`, defaults to :obj:`False`):
             Whether to return all prediction scores or just the one of the predicted class.
+        function_to_apply (:obj:`str`, `optional`, defaults to :obj:`"default"`):
+            The function to apply to the model outputs in order to retrieve the scores. Accepts four different values:
+
+            - :obj:`"default"`: if the model has a single label, will apply the sigmoid function on the output. If the
+              model has several labels, will apply the softmax function on the output.
+            - :obj:`"sigmoid"`: Applies the sigmoid function on the output.
+            - :obj:`"softmax"`: Applies the softmax function on the output.
+            - :obj:`"none"`: Does not apply any function on the output.
     """,
 )
 class TextClassificationPipeline(Pipeline):
@@ -945,7 +953,9 @@ class TextClassificationPipeline(Pipeline):
     <https://huggingface.co/models?filter=text-classification>`__.
     """
 
-    def __init__(self, return_all_scores: bool = False, **kwargs):
+    task = "text-classification"
+
+    def __init__(self, return_all_scores: bool = None, function_to_apply: str = None, **kwargs):
         super().__init__(**kwargs)
 
         self.check_model_type(
@@ -954,15 +964,33 @@ class TextClassificationPipeline(Pipeline):
             else MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
         )
 
-        self.return_all_scores = return_all_scores
+        if hasattr(self.model.config, "return_all_scores") and return_all_scores is None:
+            return_all_scores = self.model.config.return_all_scores
 
-    def __call__(self, *args, **kwargs):
+        if hasattr(self.model.config, "function_to_apply") and function_to_apply is None:
+            function_to_apply = self.model.config.function_to_apply
+
+        self.return_all_scores = return_all_scores if return_all_scores is not None else False
+        self.function_to_apply = function_to_apply if function_to_apply is not None else "default"
+
+    def __call__(self, *args, return_all_scores=None, function_to_apply=None, **kwargs):
         """
         Classify the text(s) given as inputs.
 
         Args:
             args (:obj:`str` or :obj:`List[str]`):
                 One or several texts (or one list of prompts) to classify.
+            return_all_scores (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                Whether to return scores for all labels.
+            function_to_apply (:obj:`str`, `optional`, defaults to :obj:`"default"`):
+                The function to apply to the model outputs in order to retrieve the scores. Accepts four different
+                values:
+
+                - :obj:`"default"`: if the model has a single label, will apply the sigmoid function on the output. If
+                  the model has several labels, will apply the softmax function on the output.
+                - :obj:`"sigmoid"`: Applies the sigmoid function on the output.
+                - :obj:`"softmax"`: Applies the softmax function on the output.
+                - :obj:`"none"`: Does not apply any function on the output.
 
         Return:
             A list or a list of list of :obj:`dict`: Each result comes as list of dictionaries with the following keys:
@@ -974,11 +1002,30 @@ class TextClassificationPipeline(Pipeline):
         """
         outputs = super().__call__(*args, **kwargs)
 
-        if self.model.config.num_labels == 1:
-            scores = 1.0 / (1.0 + np.exp(-outputs))
+        return_all_scores = return_all_scores if return_all_scores is not None else self.return_all_scores
+        function_to_apply = function_to_apply if function_to_apply is not None else self.function_to_apply
+
+        def sigmoid(_outputs):
+            return 1.0 / (1.0 + np.exp(-_outputs))
+
+        def softmax(_outputs):
+            return np.exp(_outputs) / np.exp(_outputs).sum(-1, keepdims=True)
+
+        if function_to_apply == "default":
+            if self.model.config.num_labels == 1:
+                scores = sigmoid(outputs)
+            else:
+                scores = softmax(outputs)
+        elif function_to_apply == "sigmoid":
+            scores = sigmoid(outputs)
+        elif function_to_apply == "softmax":
+            scores = softmax(outputs)
+        elif function_to_apply.lower() == "none":
+            scores = outputs
         else:
-            scores = np.exp(outputs) / np.exp(outputs).sum(-1, keepdims=True)
-        if self.return_all_scores:
+            raise ValueError(f"Unrecognized `function_to_apply` argument: {function_to_apply}")
+
+        if return_all_scores:
             return [
                 [{"label": self.model.config.id2label[i], "score": score.item()} for i, score in enumerate(item)]
                 for item in scores
@@ -1040,6 +1087,8 @@ class ZeroShotClassificationPipeline(Pipeline):
     of available models on `huggingface.co/models <https://huggingface.co/models?search=nli>`__.
     """
 
+    task = "zero-shot"
+
     def __init__(self, args_parser=ZeroShotClassificationArgumentHandler(), *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._args_parser = args_parser
@@ -1172,6 +1221,8 @@ class FillMaskPipeline(Pipeline):
         This pipeline only works for inputs with exactly one token masked.
     """
 
+    task = "fill-mask"
+
     def __init__(
         self,
         model: Union["PreTrainedModel", "TFPreTrainedModel"],
@@ -1361,6 +1412,7 @@ class TokenClassificationPipeline(Pipeline):
     <https://huggingface.co/models?filter=token-classification>`__.
     """
 
+    task = "token-classification"
     default_input_names = "sequences"
 
     def __init__(
@@ -1667,6 +1719,7 @@ class QuestionAnsweringPipeline(Pipeline):
     <https://huggingface.co/models?filter=question-answering>`__.
     """
 
+    task = "question-answering"
     default_input_names = "question,context"
 
     def __init__(
@@ -2067,6 +2120,8 @@ class SummarizationPipeline(Pipeline):
         summarizer("Sam Shleifer writes the best docstring examples in the whole world.", min_length=5, max_length=20)
     """
 
+    task = "summarization"
+
     def __init__(self, *args, **kwargs):
         kwargs.update(task="summarization")
         super().__init__(*args, **kwargs)
@@ -2188,6 +2243,8 @@ class TranslationPipeline(Pipeline):
         en_fr_translator("How old are you?")
     """
 
+    task = "translation"
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -2297,6 +2354,8 @@ class Text2TextGenerationPipeline(Pipeline):
         text2text_generator("question: What is 42 ? context: 42 is the answer to life, the universe and everything")
     """
 
+    task = "text2text-generation"
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -2522,6 +2581,8 @@ class ConversationalPipeline(Pipeline):
         conversational_pipeline([conversation_1, conversation_2])
     """
 
+    task = "conversational"
+
     def __init__(self, min_length_for_response=32, *args, **kwargs):
         super().__init__(*args, **kwargs)
 

src/transformers/tokenization_utils_base.py

@@ -2604,7 +2604,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
 
                 Instead of :obj:`List[int]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow tensors),
                 see the note above for the return type.
-            padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`False`):
+            padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
                  Select a strategy to pad the returned sequences (according to the model's padding side and padding
                  index) among:
 

src/transformers/trainer.py

@@ -808,8 +808,8 @@ class Trainer:
             logger.info(
                 f"Loading best model from {self.state.best_model_checkpoint} (score: {self.state.best_metric})."
             )
-            if isinstance(model, PreTrainedModel):
-                self.model = model.from_pretrained(self.state.best_model_checkpoint)
+            if isinstance(self.model, PreTrainedModel):
+                self.model = self.model.from_pretrained(self.state.best_model_checkpoint)
                 if not self.args.model_parallel:
                     self.model = self.model.to(self.args.device)
             else:

src/transformers/trainer_tf.py

@@ -19,7 +19,7 @@ from tensorflow.python.distribute.values import PerReplica
 
 from .modeling_tf_utils import TFPreTrainedModel
 from .optimization_tf import GradientAccumulator, create_optimizer
-from .trainer_utils import PREFIX_CHECKPOINT_DIR, EvalPrediction, PredictionOutput, set_seed
+from .trainer_utils import PREFIX_CHECKPOINT_DIR, EvalPrediction, EvaluationStrategy, PredictionOutput, set_seed
 from .training_args_tf import TFTrainingArguments
 from .utils import logging
 
@@ -561,7 +561,7 @@ class TFTrainer:
 
                     if (
                         self.args.eval_steps > 0
-                        and self.args.evaluate_during_training
+                        and self.args.evaluate_strategy == EvaluationStrategy.STEPS
                         and self.global_step % self.args.eval_steps == 0
                     ):
                         self.evaluate()

src/transformers/training_args_tf.py

@@ -34,8 +34,12 @@ class TFTrainingArguments(TrainingArguments):
             Whether to run evaluation on the dev set or not.
         do_predict (:obj:`bool`, `optional`, defaults to :obj:`False`):
             Whether to run predictions on the test set or not.
-        evaluate_during_training (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            Whether to run evaluation during training at each logging step or not.
+        evaluation_strategy (:obj:`str` or :class:`~transformers.trainer_utils.EvaluationStrategy`, `optional`, defaults to :obj:`"no"`):
+            The evaluation strategy to adopt during training. Possible values are:
+
+                * :obj:`"no"`: No evaluation is done during training.
+                * :obj:`"steps"`: Evaluation is done (and logged) every :obj:`eval_steps`.
+
         per_device_train_batch_size (:obj:`int`, `optional`, defaults to 8):
             The batch size per GPU/TPU core/CPU for training.
         per_device_eval_batch_size (:obj:`int`, `optional`, defaults to 8):

tests/test_modeling_conv_bert.py

@@ -1,265 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Google AI Language Team Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" Testing suite for the PyTorch ConvBERT model. """
-
-
-import unittest
-
-from transformers import is_torch_available
-from transformers.testing_utils import require_torch, slow, torch_device
-
-from .test_configuration_common import ConfigTester
-from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
-
-
-if is_torch_available():
-    from transformers import (
-        ConvBertConfig,
-        ConvBertForMaskedLM,
-        ConvBertForMultipleChoice,
-        ConvBertForQuestionAnswering,
-        ConvBertForSequenceClassification,
-        ConvBertForTokenClassification,
-        ConvBertModel,
-    )
-    from transformers.models.conv_bert.modeling_conv_bert import CONV_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
-
-
-class ConvBertModelTester:
-    def __init__(
-        self,
-        parent,
-        batch_size=13,
-        seq_length=7,
-        is_training=True,
-        use_input_mask=True,
-        use_token_type_ids=True,
-        use_labels=True,
-        vocab_size=99,
-        hidden_size=32,
-        num_hidden_layers=5,
-        num_attention_heads=4,
-        intermediate_size=37,
-        hidden_act="gelu",
-        hidden_dropout_prob=0.1,
-        attention_probs_dropout_prob=0.1,
-        max_position_embeddings=512,
-        type_vocab_size=16,
-        type_sequence_label_size=2,
-        initializer_range=0.02,
-        num_labels=3,
-        num_choices=4,
-        scope=None,
-    ):
-        self.parent = parent
-        self.batch_size = batch_size
-        self.seq_length = seq_length
-        self.is_training = is_training
-        self.use_input_mask = use_input_mask
-        self.use_token_type_ids = use_token_type_ids
-        self.use_labels = use_labels
-        self.vocab_size = vocab_size
-        self.hidden_size = hidden_size
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.intermediate_size = intermediate_size
-        self.hidden_act = hidden_act
-        self.hidden_dropout_prob = hidden_dropout_prob
-        self.attention_probs_dropout_prob = attention_probs_dropout_prob
-        self.max_position_embeddings = max_position_embeddings
-        self.type_vocab_size = type_vocab_size
-        self.type_sequence_label_size = type_sequence_label_size
-        self.initializer_range = initializer_range
-        self.num_labels = num_labels
-        self.num_choices = num_choices
-        self.scope = scope
-
-    def prepare_config_and_inputs(self):
-        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
-
-        input_mask = None
-        if self.use_input_mask:
-            input_mask = random_attention_mask([self.batch_size, self.seq_length])
-
-        token_type_ids = None
-        if self.use_token_type_ids:
-            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)
-
-        sequence_labels = None
-        token_labels = None
-        choice_labels = None
-        if self.use_labels:
-            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
-            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
-            choice_labels = ids_tensor([self.batch_size], self.num_choices)
-
-        config = ConvBertConfig(
-            vocab_size=self.vocab_size,
-            hidden_size=self.hidden_size,
-            num_hidden_layers=self.num_hidden_layers,
-            num_attention_heads=self.num_attention_heads,
-            intermediate_size=self.intermediate_size,
-            hidden_act=self.hidden_act,
-            hidden_dropout_prob=self.hidden_dropout_prob,
-            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
-            max_position_embeddings=self.max_position_embeddings,
-            type_vocab_size=self.type_vocab_size,
-            is_decoder=False,
-            initializer_range=self.initializer_range,
-        )
-
-        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
-
-    def create_and_check_model(
-        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
-    ):
-        model = ConvBertModel(config=config)
-        model.to(torch_device)
-        model.eval()
-        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
-        result = model(input_ids, token_type_ids=token_type_ids)
-        result = model(input_ids)
-        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
-
-    def create_and_check_for_masked_lm(
-        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
-    ):
-        model = ConvBertForMaskedLM(config=config)
-        model.to(torch_device)
-        model.eval()
-        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
-        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
-
-    def create_and_check_for_question_answering(
-        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
-    ):
-        model = ConvBertForQuestionAnswering(config=config)
-        model.to(torch_device)
-        model.eval()
-        result = model(
-            input_ids,
-            attention_mask=input_mask,
-            token_type_ids=token_type_ids,
-            start_positions=sequence_labels,
-            end_positions=sequence_labels,
-        )
-        self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
-        self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))
-
-    def create_and_check_for_sequence_classification(
-        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
-    ):
-        config.num_labels = self.num_labels
-        model = ConvBertForSequenceClassification(config)
-        model.to(torch_device)
-        model.eval()
-        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels)
-        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))
-
-    def create_and_check_for_token_classification(
-        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
-    ):
-        config.num_labels = self.num_labels
-        model = ConvBertForTokenClassification(config=config)
-        model.to(torch_device)
-        model.eval()
-        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
-        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
-
-    def create_and_check_for_multiple_choice(
-        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
-    ):
-        config.num_choices = self.num_choices
-        model = ConvBertForMultipleChoice(config=config)
-        model.to(torch_device)
-        model.eval()
-        multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
-        multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
-        multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
-        result = model(
-            multiple_choice_inputs_ids,
-            attention_mask=multiple_choice_input_mask,
-            token_type_ids=multiple_choice_token_type_ids,
-            labels=choice_labels,
-        )
-        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))
-
-    def prepare_config_and_inputs_for_common(self):
-        config_and_inputs = self.prepare_config_and_inputs()
-        (
-            config,
-            input_ids,
-            token_type_ids,
-            input_mask,
-            sequence_labels,
-            token_labels,
-            choice_labels,
-        ) = config_and_inputs
-        inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
-        return config, inputs_dict
-
-
-@require_torch
-class ConvBertModelTest(ModelTesterMixin, unittest.TestCase):
-
-    all_model_classes = (
-        (
-            ConvBertModel,
-            ConvBertForMaskedLM,
-            ConvBertForMultipleChoice,
-            ConvBertForQuestionAnswering,
-            ConvBertForSequenceClassification,
-            ConvBertForTokenClassification,
-        )
-        if is_torch_available()
-        else ()
-    )
-
-    def setUp(self):
-        self.model_tester = ConvBertModelTester(self)
-        self.config_tester = ConfigTester(self, config_class=ConvBertConfig, hidden_size=37)
-
-    def test_config(self):
-        self.config_tester.run_common_tests()
-
-    def test_model(self):
-        config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_model(*config_and_inputs)
-
-    def test_for_masked_lm(self):
-        config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_masked_lm(*config_and_inputs)
-
-    def test_for_multiple_choice(self):
-        config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs)
-
-    def test_for_question_answering(self):
-        config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_question_answering(*config_and_inputs)
-
-    def test_for_sequence_classification(self):
-        config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs)
-
-    def test_for_token_classification(self):
-        config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_token_classification(*config_and_inputs)
-
-    @slow
-    def test_model_from_pretrained(self):
-        for model_name in CONV_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
-            model = ConvBertModel.from_pretrained(model_name)
-            self.assertIsNotNone(model)

tests/test_modeling_flax_bert.py

@@ -1,6 +1,5 @@
 import unittest
 
-import pytest
 from numpy import ndarray
 
 from transformers import BertTokenizerFast, TensorType, is_flax_available, is_torch_available
@@ -24,6 +23,10 @@ if is_torch_available():
 @require_flax
 @require_torch
 class FlaxBertModelTest(unittest.TestCase):
+    def assert_almost_equals(self, a: ndarray, b: ndarray, tol: float):
+        diff = (a - b).sum()
+        self.assertLessEqual(diff, tol, f"Difference between torch and flax is {diff} (>= {tol})")
+
     def test_from_pytorch(self):
         with torch.no_grad():
             with self.subTest("bert-base-cased"):
@@ -40,32 +43,27 @@ class FlaxBertModelTest(unittest.TestCase):
                 self.assertEqual(len(fx_outputs), len(pt_outputs), "Output lengths differ between Flax and PyTorch")
 
                 for fx_output, pt_output in zip(fx_outputs, pt_outputs):
-                    self.assert_almost_equals(fx_output, pt_output.numpy(), 5e-4)
+                    self.assert_almost_equals(fx_output, pt_output.numpy(), 5e-3)
 
-    def assert_almost_equals(self, a: ndarray, b: ndarray, tol: float):
-        diff = (a - b).sum()
-        self.assertLessEqual(diff, tol, "Difference between torch and flax is {} (>= {})".format(diff, tol))
+    def test_multiple_sequences(self):
+        tokenizer = BertTokenizerFast.from_pretrained("bert-base-cased")
+        model = FlaxBertModel.from_pretrained("bert-base-cased")
 
+        sequences = ["this is an example sentence", "this is another", "and a third one"]
+        encodings = tokenizer(sequences, return_tensors=TensorType.JAX, padding=True, truncation=True)
 
-@require_flax
-@require_torch
-@pytest.mark.parametrize("jit", ["disable_jit", "enable_jit"])
-def test_multiple_sentences(jit):
-    tokenizer = BertTokenizerFast.from_pretrained("bert-base-cased")
-    model = FlaxBertModel.from_pretrained("bert-base-cased")
+        @jax.jit
+        def model_jitted(input_ids, attention_mask=None, token_type_ids=None):
+            return model(input_ids, attention_mask, token_type_ids)
 
-    sentences = ["this is an example sentence", "this is another", "and a third one"]
-    encodings = tokenizer(sentences, return_tensors=TensorType.JAX, padding=True, truncation=True)
+        with self.subTest("JIT Disabled"):
+            with jax.disable_jit():
+                tokens, pooled = model_jitted(**encodings)
+                self.assertEqual(tokens.shape, (3, 7, 768))
+                self.assertEqual(pooled.shape, (3, 768))
 
-    @jax.jit
-    def model_jitted(input_ids, attention_mask, token_type_ids):
-        return model(input_ids, attention_mask, token_type_ids)
+        with self.subTest("JIT Enabled"):
+            jitted_tokens, jitted_pooled = model_jitted(**encodings)
 
-    if jit == "disable_jit":
-        with jax.disable_jit():
-            tokens, pooled = model_jitted(**encodings)
-    else:
-        tokens, pooled = model_jitted(**encodings)
-
-    assert tokens.shape == (3, 7, 768)
-    assert pooled.shape == (3, 768)
+            self.assertEqual(jitted_tokens.shape, (3, 7, 768))
+            self.assertEqual(jitted_pooled.shape, (3, 768))

tests/test_modeling_flax_roberta.py

@@ -1,6 +1,5 @@
 import unittest
 
-import pytest
 from numpy import ndarray
 
 from transformers import RobertaTokenizerFast, TensorType, is_flax_available, is_torch_available
@@ -24,6 +23,10 @@ if is_torch_available():
 @require_flax
 @require_torch
 class FlaxRobertaModelTest(unittest.TestCase):
+    def assert_almost_equals(self, a: ndarray, b: ndarray, tol: float):
+        diff = (a - b).sum()
+        self.assertLessEqual(diff, tol, f"Difference between torch and flax is {diff} (>= {tol})")
+
     def test_from_pytorch(self):
         with torch.no_grad():
             with self.subTest("roberta-base"):
@@ -40,32 +43,27 @@ class FlaxRobertaModelTest(unittest.TestCase):
                 self.assertEqual(len(fx_outputs), len(pt_outputs), "Output lengths differ between Flax and PyTorch")
 
                 for fx_output, pt_output in zip(fx_outputs, pt_outputs.to_tuple()):
-                    self.assert_almost_equals(fx_output, pt_output.numpy(), 5e-4)
+                    self.assert_almost_equals(fx_output, pt_output.numpy(), 6e-4)
 
-    def assert_almost_equals(self, a: ndarray, b: ndarray, tol: float):
-        diff = (a - b).sum()
-        self.assertLessEqual(diff, tol, "Difference between torch and flax is {} (>= {})".format(diff, tol))
+    def test_multiple_sequences(self):
+        tokenizer = RobertaTokenizerFast.from_pretrained("roberta-base")
+        model = FlaxRobertaModel.from_pretrained("roberta-base")
 
+        sequences = ["this is an example sentence", "this is another", "and a third one"]
+        encodings = tokenizer(sequences, return_tensors=TensorType.JAX, padding=True, truncation=True)
 
-@require_flax
-@require_torch
-@pytest.mark.parametrize("jit", ["disable_jit", "enable_jit"])
-def test_multiple_sentences(jit):
-    tokenizer = RobertaTokenizerFast.from_pretrained("roberta-base")
-    model = FlaxRobertaModel.from_pretrained("roberta-base")
+        @jax.jit
+        def model_jitted(input_ids, attention_mask=None, token_type_ids=None):
+            return model(input_ids, attention_mask, token_type_ids)
 
-    sentences = ["this is an example sentence", "this is another", "and a third one"]
-    encodings = tokenizer(sentences, return_tensors=TensorType.JAX, padding=True, truncation=True)
+        with self.subTest("JIT Disabled"):
+            with jax.disable_jit():
+                tokens, pooled = model_jitted(**encodings)
+                self.assertEqual(tokens.shape, (3, 7, 768))
+                self.assertEqual(pooled.shape, (3, 768))
 
-    @jax.jit
-    def model_jitted(input_ids, attention_mask):
-        return model(input_ids, attention_mask)
+        with self.subTest("JIT Enabled"):
+            jitted_tokens, jitted_pooled = model_jitted(**encodings)
 
-    if jit == "disable_jit":
-        with jax.disable_jit():
-            tokens, pooled = model_jitted(**encodings)
-    else:
-        tokens, pooled = model_jitted(**encodings)
-
-    assert tokens.shape == (3, 7, 768)
-    assert pooled.shape == (3, 768)
+            self.assertEqual(jitted_tokens.shape, (3, 7, 768))
+            self.assertEqual(jitted_pooled.shape, (3, 768))

tests/test_pipelines_common.py

@@ -197,6 +197,7 @@ class MonoInputPipelineCommonMixin(CustomInputPipelineCommonMixin):
 
     def _test_pipeline(self, nlp: Pipeline):
         self.assertIsNotNone(nlp)
+        self.assertIsNotNone(nlp.task)
 
         mono_result = nlp(self.valid_inputs[0], **self.pipeline_running_kwargs)
         self.assertIsInstance(mono_result, list)

tests/test_pipelines_sentiment_analysis.py

@@ -1,12 +1,117 @@
 import unittest
 
+import numpy as np
+
+from transformers import AutoTokenizer, DistilBertConfig, DistilBertForSequenceClassification, pipeline
+from transformers.testing_utils import slow
+
 from .test_pipelines_common import MonoInputPipelineCommonMixin
 
 
+VALID_INPUTS = ["I really disagree with what you've said.", ["I love you."]]
+
+
 class SentimentAnalysisPipelineTests(MonoInputPipelineCommonMixin, unittest.TestCase):
     pipeline_task = "sentiment-analysis"
-    small_models = [
-        "sshleifer/tiny-distilbert-base-uncased-finetuned-sst-2-english"
-    ]  # Default model - Models tested without the @slow decorator
+    small_models = ["distilbert-base-cased"]  # Default model - Models tested without the @slow decorator
     large_models = [None]  # Models tested with the @slow decorator
     mandatory_keys = {"label", "score"}  # Keys which should be in the output
+
+    @slow
+    def test_function_to_apply(self):
+        for model_name in self.small_models:
+            string_input, string_list_input = VALID_INPUTS
+            tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
+
+            model = DistilBertForSequenceClassification(DistilBertConfig.from_pretrained(model_name))
+            model.eval()
+            classifier = pipeline(task="sentiment-analysis", model=model, tokenizer=tokenizer)
+
+            def check_output(output):
+                # This model does not have a sequence-classification head, so results are random
+                self.assertTrue(isinstance(output, list))
+                self.assertEqual(len(output), 1)
+                d = string_output[0]
+                self.assertEqual(set(d.keys()), {"label", "score"})
+                self.assertEqual(type(d["label"]), str)
+                self.assertEqual(type(d["score"]), float)
+
+            def pipeline_call_argument(argument=None):
+                _string_output = classifier(string_input, function_to_apply=argument)
+                _string_list_output = classifier(string_list_input, function_to_apply=argument)
+                check_output(_string_output)
+                return _string_output, _string_list_output
+
+            def pipeline_init_argument(argument=None):
+                _classifier = pipeline(
+                    task="sentiment-analysis", model=model, tokenizer=tokenizer, function_to_apply=argument
+                )
+                _string_output = _classifier(string_input)
+                _string_list_output = _classifier(string_list_input)
+                check_output(_string_output)
+                return _string_output, _string_list_output
+
+            def pipeline_model_argument(argument=None):
+                model.config.task_specific_params = {"sentiment-analysis": {"function_to_apply": argument}}
+                _classifier = pipeline(task="sentiment-analysis", model=model, tokenizer=tokenizer)
+                _string_output = _classifier(string_input)
+                _string_list_output = _classifier(string_list_input)
+                check_output(_string_output)
+                return _string_output, _string_list_output
+
+            string_output = classifier(string_input)
+            string_list_output = classifier(string_list_input)
+
+            string_output_default = pipeline_call_argument("default")
+            string_output_sigmoid = pipeline_call_argument("sigmoid")
+            string_output_softmax = pipeline_call_argument("softmax")
+            string_output_none = pipeline_call_argument("none")
+
+            string_output_init_default = pipeline_init_argument("default")
+            string_output_init_sigmoid = pipeline_init_argument("sigmoid")
+            string_output_init_softmax = pipeline_init_argument("softmax")
+            string_output_init_none = pipeline_init_argument("none")
+
+            string_output_model_default = pipeline_model_argument("default")
+            string_output_model_sigmoid = pipeline_model_argument("sigmoid")
+            string_output_model_softmax = pipeline_model_argument("softmax")
+            string_output_model_none = pipeline_model_argument("none")
+
+            should_be_equal = [
+                (
+                    (string_output, string_list_output),
+                    string_output_default,
+                    string_output_init_default,
+                    string_output_model_default,
+                ),
+                (string_output_sigmoid, string_output_init_sigmoid, string_output_model_sigmoid),
+                (string_output_softmax, string_output_init_softmax, string_output_model_softmax),
+                (string_output_none, string_output_init_none, string_output_model_none),
+            ]
+
+            for tuples_containing_equal_values in should_be_equal:
+                # Retrieve each tuple from the list
+                for tuple_value_0 in tuples_containing_equal_values:
+                    for tuple_value_1 in tuples_containing_equal_values:
+                        if tuple_value_0 is not tuple_value_1:
+                            # Compare each tuple value with all the others, as long as they're not the same object
+                            for pipeline_output_0, pipeline_output_1 in zip(tuple_value_0, tuple_value_1):
+                                # Compare all outputs (call, init, model argument)
+                                for example_result_0, example_result_1 in zip(pipeline_output_0, pipeline_output_1):
+                                    # Iterate through the results
+                                    self.assertTrue(
+                                        np.allclose(example_result_0["score"], example_result_1["score"], atol=1e-6)
+                                    )
+
+    @slow
+    def test_function_to_apply_error(self):
+        for model_name in self.small_models:
+            string_input, _ = VALID_INPUTS
+            tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
+
+            model = DistilBertForSequenceClassification(DistilBertConfig.from_pretrained(model_name))
+            model.eval()
+            classifier = pipeline(task="sentiment-analysis", model=model, tokenizer=tokenizer)
+
+            with self.assertRaises(ValueError):
+                classifier(string_input, function_to_apply="logits")

utils/check_copies.py

@@ -282,7 +282,7 @@ def check_model_list_copy(overwrite=False, max_per_line=119):
     rst_list, start_index, end_index, lines = _find_text_in_file(
         filename=os.path.join(PATH_TO_DOCS, "index.rst"),
         start_prompt="    This list is updated automatically from the README",
-        end_prompt="The table below represents the current support",
+        end_prompt=".. _bigtable:",
     )
     md_list = get_model_list()
     converted_list = convert_to_rst(md_list, max_per_line=max_per_line)