make style

docs/source/model_doc/conv_bert.rst

@@ -0,0 +1,81 @@
+CONV_BERT
+-----------------------------------------------------------------------------------------------------------------------
+
+Overview
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ConvBERT model was proposed in `<INSERT PAPER NAME HERE> <<INSERT PAPER LINK HERE>>`__ by <INSERT AUTHORS HERE>.
+<INSERT SHORT SUMMARY HERE>
+
+The abstract from the paper is the following:
+
+*<INSERT PAPER ABSTRACT HERE>*
+
+Tips:
+
+<INSERT TIPS ABOUT MODEL HERE>
+
+ConvBertConfig
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ConvBertConfig
+    :members:
+
+
+ConvBertTokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ConvBertTokenizer
+    :members: build_inputs_with_special_tokens, get_special_tokens_mask,
+        create_token_type_ids_from_sequences, save_vocabulary
+
+
+ConvBertTokenizerFast
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ConvBertTokenizerFast
+    :members: build_inputs_with_special_tokens, get_special_tokens_mask,
+        create_token_type_ids_from_sequences, save_vocabulary
+
+
+ConvBertModel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ConvBertModel
+    :members: forward
+
+
+ConvBertForMaskedLM
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ConvBertForMaskedLM
+    :members: forward
+
+
+ConvBertForSequenceClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ConvBertForSequenceClassification
+    :members: forward
+
+
+ConvBertForMultipleChoice
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ConvBertForMultipleChoice
+    :members:
+
+
+ConvBertForTokenClassification
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ConvBertForTokenClassification
+    :members: forward
+
+
+ConvBertForQuestionAnswering
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.ConvBertForQuestionAnswering
+    :members: forward
+

src/transformers/__init__.py

@@ -113,6 +113,7 @@ from .models.blenderbot import (
     BlenderbotTokenizer,
 )
 from .models.camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
+from .models.conv_bert import CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig
 from .models.ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig, CTRLTokenizer
 from .models.deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig, DebertaTokenizer
 from .models.distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig, DistilBertTokenizer
@@ -391,6 +392,18 @@ if is_torch_available():
         CamembertForTokenClassification,
         CamembertModel,
     )
+    from .models.conv_bert import (
+        CONV_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        ConvBertForMaskedLM,
+        ConvBertForMultipleChoice,
+        ConvBertForQuestionAnswering,
+        ConvBertForSequenceClassification,
+        ConvBertForTokenClassification,
+        ConvBertLayer,
+        ConvBertModel,
+        ConvBertPreTrainedModel,
+        load_tf_weights_in_conv_bert,
+    )
     from .models.ctrl import CTRL_PRETRAINED_MODEL_ARCHIVE_LIST, CTRLLMHeadModel, CTRLModel, CTRLPreTrainedModel
     from .models.deberta import (
         DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,

src/transformers/convert_pytorch_checkpoint_to_tf2.py

@@ -302,7 +302,7 @@ def convert_pt_checkpoint_to_tf(
         pytorch_checkpoint_url = hf_bucket_url(pytorch_checkpoint_path, filename=WEIGHTS_NAME)
         pytorch_checkpoint_path = cached_path(pytorch_checkpoint_url, force_download=not use_cached_models)
     # Load PyTorch checkpoint in tf2 model:
-    tf_model = load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path)
+    tf_model = load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, config)
 
     if compare_with_pt_model:
         tfo = tf_model(tf_model.dummy_inputs, training=False)  # build the network

src/transformers/modeling_tf_pytorch_utils.py

@@ -43,6 +43,13 @@ def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove="")
           other
     """
     tf_name = tf_name.replace(":0", "")  # device ids
+
+    # support for ConvBERT model
+    tf_name_split = tf_name.split("/")
+    if tf_name_split[-1] == "depthwise_kernel":
+        tf_name = "/".join(tf_name_split[:-1]) + "/depthwise/weight"
+    if tf_name_split[-1] == "pointwise_kernel":
+        tf_name = "/".join(tf_name_split[:-1]) + "/pointwise/weight"
     tf_name = re.sub(
         r"/[^/]*___([^/]*)/", r"/\1/", tf_name
     )  # '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch)
@@ -66,7 +73,6 @@ def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove="")
     tf_name = ".".join(tf_name)
     if start_prefix_to_remove:
         tf_name = tf_name.replace(start_prefix_to_remove, "", 1)
-
     return tf_name, transpose
 
 
@@ -75,7 +81,9 @@ def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove="")
 #####################
 
 
-def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=None, allow_missing_keys=False):
+def load_pytorch_checkpoint_in_tf2_model(
+    tf_model, pytorch_checkpoint_path, config, tf_inputs=None, allow_missing_keys=False
+):
     """Load pytorch checkpoints in a TF 2.0 model"""
     try:
         import tensorflow as tf  # noqa: F401
@@ -94,7 +102,7 @@ def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_i
     logger.info("PyTorch checkpoint contains {:,} parameters".format(sum(t.numel() for t in pt_state_dict.values())))
 
     return load_pytorch_weights_in_tf2_model(
-        tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys
+        tf_model, pt_state_dict, config=config, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys
     )
 
 
@@ -107,7 +115,7 @@ def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=None, allow_mi
     )
 
 
-def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, allow_missing_keys=False):
+def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, config=None, tf_inputs=None, allow_missing_keys=False):
     """Load pytorch state_dict in a TF 2.0 model."""
     try:
         import tensorflow as tf  # noqa: F401
@@ -173,6 +181,25 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a
 
         array = pt_state_dict[name].numpy()
 
+        if config.model_type == "conv_bert":
+            if name.endswith("depthwise.weight"):
+                array = numpy.transpose(array, axes=(2, 0, 1))
+                transpose = False
+
+            if name.endswith("pointwise.weight"):
+                array = numpy.transpose(array, axes=(2, 1, 0))
+                transpose = False
+
+            if name.endswith("conv_attn_key.bias"):
+                array = numpy.squeeze(array)
+                transpose = False
+
+            if config.num_groups > 1:
+                if name.endswith("intermediate.dense.weight") or name.endswith("output.dense.weight"):
+                    if not name.endswith("attention.output.dense.weight"):
+                        print(name)
+                        transpose = False
+
         if transpose:
             array = numpy.transpose(array)
 

src/transformers/modeling_tf_utils.py

@@ -841,7 +841,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin):
             from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model
 
             # Load from a PyTorch checkpoint
-            return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, allow_missing_keys=True)
+            return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, config, allow_missing_keys=True)
 
         model(model.dummy_inputs, training=False)  # build the network with dummy inputs
 

src/transformers/models/auto/configuration_auto.py

@@ -24,6 +24,7 @@ from ..bert.configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertCo
 from ..bert_generation.configuration_bert_generation import BertGenerationConfig
 from ..blenderbot.configuration_blenderbot import BLENDERBOT_PRETRAINED_CONFIG_ARCHIVE_MAP, BlenderbotConfig
 from ..camembert.configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
+from ..conv_bert.configuration_conv_bert import CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig
 from ..ctrl.configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
 from ..deberta.configuration_deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig
 from ..distilbert.configuration_distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig
@@ -64,6 +65,7 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
     (key, value)
     for pretrained_map in [
         # Add archive maps here
+        CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         BART_PRETRAINED_CONFIG_ARCHIVE_MAP,
         BLENDERBOT_PRETRAINED_CONFIG_ARCHIVE_MAP,
@@ -101,6 +103,7 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
 CONFIG_MAPPING = OrderedDict(
     [
         # Add configs here
+        ("conv_bert", ConvBertConfig),
         ("retribert", RetriBertConfig),
         ("mt5", MT5Config),
         ("t5", T5Config),
@@ -144,6 +147,7 @@ CONFIG_MAPPING = OrderedDict(
 MODEL_NAMES_MAPPING = OrderedDict(
     [
         # Add full (and cased) model names here
+        ("conv_bert", "ConvBert"),
         ("retribert", "RetriBERT"),
         ("t5", "T5"),
         ("mobilebert", "MobileBERT"),

src/transformers/models/auto/modeling_auto.py

@@ -21,8 +21,6 @@ from collections import OrderedDict
 from ...configuration_utils import PretrainedConfig
 from ...file_utils import add_start_docstrings
 from ...utils import logging
-
-# Add modeling imports here
 from ..albert.modeling_albert import (
     AlbertForMaskedLM,
     AlbertForMultipleChoice,
@@ -60,6 +58,14 @@ from ..camembert.modeling_camembert import (
     CamembertForTokenClassification,
     CamembertModel,
 )
+from ..conv_bert.modeling_conv_bert import (
+    ConvBertForMaskedLM,
+    ConvBertForMultipleChoice,
+    ConvBertForQuestionAnswering,
+    ConvBertForSequenceClassification,
+    ConvBertForTokenClassification,
+    ConvBertModel,
+)
 from ..ctrl.modeling_ctrl import CTRLLMHeadModel, CTRLModel
 from ..deberta.modeling_deberta import DebertaForSequenceClassification, DebertaModel
 from ..distilbert.modeling_distilbert import (
@@ -196,6 +202,7 @@ from .configuration_auto import (
     BertGenerationConfig,
     BlenderbotConfig,
     CamembertConfig,
+    ConvBertConfig,
     CTRLConfig,
     DebertaConfig,
     DistilBertConfig,
@@ -230,12 +237,17 @@ from .configuration_auto import (
 )
 
 
+# Add modeling imports here
+
+
+
 logger = logging.get_logger(__name__)
 
 
 MODEL_MAPPING = OrderedDict(
     [
         # Base model mapping
+        (ConvBertConfig, ConvBertModel),
         (RetriBertConfig, RetriBertModel),
         (MT5Config, MT5Model),
         (T5Config, T5Model),
@@ -303,6 +315,7 @@ MODEL_FOR_PRETRAINING_MAPPING = OrderedDict(
 MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
     [
         # Model with LM heads mapping
+        (ConvBertConfig, ConvBertForMaskedLM),
         (LayoutLMConfig, LayoutLMForMaskedLM),
         (T5Config, T5ForConditionalGeneration),
         (DistilBertConfig, DistilBertForMaskedLM),
@@ -357,6 +370,7 @@ MODEL_FOR_CAUSAL_LM_MAPPING = OrderedDict(
 MODEL_FOR_MASKED_LM_MAPPING = OrderedDict(
     [
         # Model for Masked LM mapping
+        (ConvBertConfig, ConvBertForMaskedLM),
         (LayoutLMConfig, LayoutLMForMaskedLM),
         (DistilBertConfig, DistilBertForMaskedLM),
         (AlbertConfig, AlbertForMaskedLM),
@@ -396,6 +410,7 @@ MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
 MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
     [
         # Model for Sequence Classification mapping
+        (ConvBertConfig, ConvBertForSequenceClassification),
         (DistilBertConfig, DistilBertForSequenceClassification),
         (AlbertConfig, AlbertForSequenceClassification),
         (CamembertConfig, CamembertForSequenceClassification),
@@ -421,6 +436,7 @@ MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
 MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
     [
         # Model for Question Answering mapping
+        (ConvBertConfig, ConvBertForQuestionAnswering),
         (DistilBertConfig, DistilBertForQuestionAnswering),
         (AlbertConfig, AlbertForQuestionAnswering),
         (CamembertConfig, CamembertForQuestionAnswering),
@@ -444,6 +460,7 @@ MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
 MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = OrderedDict(
     [
         # Model for Token Classification mapping
+        (ConvBertConfig, ConvBertForTokenClassification),
         (LayoutLMConfig, LayoutLMForTokenClassification),
         (DistilBertConfig, DistilBertForTokenClassification),
         (CamembertConfig, CamembertForTokenClassification),
@@ -466,6 +483,7 @@ MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = OrderedDict(
 MODEL_FOR_MULTIPLE_CHOICE_MAPPING = OrderedDict(
     [
         # Model for Multiple Choice mapping
+        (ConvBertConfig, ConvBertForMultipleChoice),
         (CamembertConfig, CamembertForMultipleChoice),
         (ElectraConfig, ElectraForMultipleChoice),
         (XLMRobertaConfig, XLMRobertaForMultipleChoice),

src/transformers/models/conv_bert/__init__.py

@@ -0,0 +1,21 @@
+# flake8: noqa
+# There's no way to ignore "F401 '...' imported but unused" warnings in this
+# module, but to preserve other warnings. So, don't check this module at all.
+from ...file_utils import is_torch_available
+from .configuration_conv_bert import CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, ConvBertConfig
+from .tokenization_conv_bert import ConvBertTokenizer
+
+
+if is_torch_available():
+    from .modeling_conv_bert import (
+        CONV_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
+        ConvBertForMaskedLM,
+        ConvBertForMultipleChoice,
+        ConvBertForQuestionAnswering,
+        ConvBertForSequenceClassification,
+        ConvBertForTokenClassification,
+        ConvBertLayer,
+        ConvBertModel,
+        ConvBertPreTrainedModel,
+        load_tf_weights_in_conv_bert,
+    )

src/transformers/models/conv_bert/configuration_conv_bert.py

@@ -0,0 +1,126 @@
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" ConvBERT model configuration """
+
+from ...configuration_utils import PretrainedConfig
+from ...utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+CONV_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "convbert-base-uncased": "https://huggingface.co/convbert-base-uncased/resolve/main/config.json",
+    # See all ConvBERT models at https://huggingface.co/models?filter=conv_bert
+}
+
+
+class ConvBertConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a :class:`~transformers.ConvBertModel`. It is used to
+    instantiate an ConvBERT model according to the specified arguments, defining the model architecture. Instantiating
+    a configuration with the defaults will yield a similar configuration to that of the ConvBERT `convbert-base-uncased
+    <https://huggingface.co/convbert-base-uncased>`__ architecture.
+
+    Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model
+    outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information.
+
+
+    Args:
+        vocab_size (:obj:`int`, `optional`, defaults to 30522):
+            Vocabulary size of the ConvBERT model. Defines the number of different tokens that can be represented by
+            the :obj:`inputs_ids` passed when calling :class:`~transformers.ConvBertModel` or
+            :class:`~transformers.TFConvBertModel`. Vocabulary size of the model. Defines the different tokens that can
+            be represented by the `inputs_ids` passed to the forward method of :class:`~transformers.ConvBertModel`.
+        hidden_size (:obj:`int`, `optional`, defaults to 768):
+            Dimensionality of the encoder layers and the pooler layer.
+        num_hidden_layers (:obj:`int`, `optional`, defaults to 12):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (:obj:`int`, `optional`, defaults to 12):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        intermediate_size (:obj:`int`, `optional`, defaults to 3072):
+            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        hidden_act (:obj:`str` or :obj:`function`, `optional`, defaults to :obj:`"gelu"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string,
+            :obj:`"gelu"`, :obj:`"relu"`, :obj:`"selu"` and :obj:`"gelu_new"` are supported.
+        hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1):
+            The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
+        attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1):
+            The dropout ratio for the attention probabilities.
+        max_position_embeddings (:obj:`int`, `optional`, defaults to 512):
+            The maximum sequence length that this model might ever be used with. Typically set this to something large
+            just in case (e.g., 512 or 1024 or 2048).
+        type_vocab_size (:obj:`int`, `optional`, defaults to 2):
+            The vocabulary size of the :obj:`token_type_ids` passed when calling :class:`~transformers.ConvBertModel`
+            or :class:`~transformers.TFConvBertModel`.
+        initializer_range (:obj:`float`, `optional`, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        layer_norm_eps (:obj:`float`, `optional`, defaults to 1e-12):
+            The epsilon used by the layer normalization layers.
+
+    Example::
+
+        >>> from transformers import ConvBertModel, ConvBertConfig
+
+        >>> # Initializing a ConvBERT convbert-base-uncased style configuration
+        >>> configuration = ConvBertConfig()
+
+        >>> # Initializing a model from the convbert-base-uncased style configuration
+        >>> model = ConvBertModel(configuration)
+
+        >>> # Accessing the model configuration
+        >>> configuration = model.config
+    """
+    model_type = "conv_bert"
+
+    def __init__(
+        self,
+        vocab_size=30522,
+        hidden_size=768,
+        is_encoder_decoder=False,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        intermediate_size=3072,
+        hidden_act="gelu",
+        hidden_dropout_prob=0.1,
+        attention_probs_dropout_prob=0.1,
+        max_position_embeddings=512,
+        type_vocab_size=2,
+        initializer_range=0.02,
+        layer_norm_eps=1e-12,
+        pad_token_id=1,
+        bos_token_id=0,
+        eos_token_id=2,
+        **kwargs
+    ):
+        super().__init__(
+            pad_token_id=pad_token_id,
+            is_encoder_decoder=is_encoder_decoder,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs,
+        )
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.hidden_act = hidden_act
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.type_vocab_size = type_vocab_size
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps

src/transformers/models/conv_bert/convert_tf1.py

@@ -0,0 +1,25 @@
+import tensorflow as tf
+import torch
+
+from .modeling_tf_conv_bert import ConvBertConfig, TFConvBertModel
+from .tokenization_conv_bert import ConvBertTokenizer
+
+
+tf_checkpoint_path = "/home/abhishek/huggingface/models/convbert/model.ckpt"
+conf = ConvBertConfig.from_json_file("/home/abhishek/huggingface/models/convbert/config.json")
+tokenizer = ConvBertTokenizer.from_pretrained("/home/abhishek/huggingface/models/convbert/")
+
+
+if __name__ == "__main__":
+    init_vars = tf.train.list_variables(tf_checkpoint_path)
+    model = TFConvBertModel(conf)
+    tf_inputs = model.dummy_inputs
+    model(tf_inputs, training=False)
+    print(model.base_model_prefix)
+
+    weight_dict = {}
+    for name, shape in init_vars:
+        array = tf.train.load_variable(tf_checkpoint_path, name)
+        weight_dict[name] = array
+
+    print(model.summary())

src/transformers/models/conv_bert/param_mapping.py

@@ -0,0 +1,85 @@
+def fetch_mapping(config):
+    PARAM_MAPPING = {
+        "embeddings.word_embeddings.weight": "electra/embeddings/word_embeddings",
+        "embeddings.position_embeddings.weight": "electra/embeddings/position_embeddings",
+        "embeddings.token_type_embeddings.weight": "electra/embeddings/token_type_embeddings",
+        "embeddings.LayerNorm.weight": "electra/embeddings/LayerNorm/gamma",
+        "embeddings.LayerNorm.bias": "electra/embeddings/LayerNorm/beta",
+        "embeddings_project.weight": "electra/embeddings_project/kernel",
+        "embeddings_project.bias": "electra/embeddings_project/bias",
+    }
+    if config.num_groups > 1:
+        group_dense_name = "g_dense"
+    else:
+        group_dense_name = "dense"
+
+    for j in range(config.num_hidden_layers):
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.query.weight"
+        ] = f"electra/encoder/layer_{j}/attention/self/query/kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.query.bias"
+        ] = f"electra/encoder/layer_{j}/attention/self/query/bias"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.key.weight"
+        ] = f"electra/encoder/layer_{j}/attention/self/key/kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.key.bias"
+        ] = f"electra/encoder/layer_{j}/attention/self/key/bias"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.value.weight"
+        ] = f"electra/encoder/layer_{j}/attention/self/value/kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.value.bias"
+        ] = f"electra/encoder/layer_{j}/attention/self/value/bias"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.key_conv_attn_layer.depthwise.weight"
+        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_key/depthwise_kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.key_conv_attn_layer.pointwise.weight"
+        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_key/pointwise_kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.key_conv_attn_layer.bias"
+        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_key/bias"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.conv_kernel_layer.weight"
+        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_kernel/kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.conv_kernel_layer.bias"
+        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_kernel/bias"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.conv_out_layer.weight"
+        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_point/kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.self.conv_out_layer.bias"
+        ] = f"electra/encoder/layer_{j}/attention/self/conv_attn_point/bias"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.output.dense.weight"
+        ] = f"electra/encoder/layer_{j}/attention/output/dense/kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.output.LayerNorm.weight"
+        ] = f"electra/encoder/layer_{j}/attention/output/LayerNorm/gamma"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.output.dense.bias"
+        ] = f"electra/encoder/layer_{j}/attention/output/dense/bias"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.attention.output.LayerNorm.bias"
+        ] = f"electra/encoder/layer_{j}/attention/output/LayerNorm/beta"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.intermediate.dense.weight"
+        ] = f"electra/encoder/layer_{j}/intermediate/{group_dense_name}/kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.intermediate.dense.bias"
+        ] = f"electra/encoder/layer_{j}/intermediate/{group_dense_name}/bias"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.output.dense.weight"
+        ] = f"electra/encoder/layer_{j}/output/{group_dense_name}/kernel"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.output.dense.bias"
+        ] = f"electra/encoder/layer_{j}/output/{group_dense_name}/bias"
+        PARAM_MAPPING[
+            f"encoder.layer.{j}.output.LayerNorm.weight"
+        ] = f"electra/encoder/layer_{j}/output/LayerNorm/gamma"
+        PARAM_MAPPING[f"encoder.layer.{j}.output.LayerNorm.bias"] = f"electra/encoder/layer_{j}/output/LayerNorm/beta"
+
+    return PARAM_MAPPING

src/transformers/models/conv_bert/run_bert.py

@@ -0,0 +1,31 @@
+import os
+
+import tensorflow as tf
+import torch
+
+from ..bert.modeling_bert import BertConfig, BertModel, load_tf_weights_in_bert
+from ..bert.modeling_tf_bert import TFBertModel
+from ..bert.tokenization_bert import BertTokenizer
+
+
+if __name__ == "__main__":
+    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+    # tf_checkpoint_path = os.path.join(model_path, "model.ckpt")
+    model = BertModel.from_pretrained("bert-base-uncased")
+
+    # model = load_tf_weights_in_conv_bert(model, conf, tf_checkpoint_path)
+    # model.save_pretrained(model_path)
+
+    tf_model = TFBertModel.from_pretrained("bert-base-uncased")
+    # tf_model.save_pretrained(model_path)
+
+    print(model.dummy_inputs)
+    print(model(**model.dummy_inputs))
+
+    # tf_model = TFConvBertModel(conf)
+
+    print(tf_model.dummy_inputs)
+    print(tf_model(tf_model.dummy_inputs))
+
+    # last_hidden_states = outputs.last_hidden_state
+    # print(last_hidden_states)

src/transformers/models/conv_bert/run_pt.py

@@ -0,0 +1,26 @@
+import os
+
+import tensorflow as tf
+import torch
+
+from .modeling_conv_bert import ConvBertConfig, ConvBertModel, load_tf_weights_in_conv_bert
+from .modeling_tf_conv_bert import TFConvBertModel
+from .tokenization_conv_bert import ConvBertTokenizer
+
+
+model_path = "/home/abhishek/huggingface/models/convbert_models/convbert_medium_small"
+
+if __name__ == "__main__":
+    conf = ConvBertConfig.from_json_file(os.path.join(model_path, "config.json"))
+    tokenizer = ConvBertTokenizer.from_pretrained(model_path)
+    tf_checkpoint_path = os.path.join(model_path, "model.ckpt")
+    model = ConvBertModel(conf)
+
+    model = load_tf_weights_in_conv_bert(model, conf, tf_checkpoint_path)
+    # model.save_pretrained(model_path)
+    model.eval()
+    # for p in model.named_parameters():
+    #     print(p)
+
+    print(model.dummy_inputs)
+    print(model(**model.dummy_inputs).last_hidden_state)

src/transformers/models/conv_bert/run_test.py

@@ -0,0 +1,40 @@
+import os
+
+import tensorflow as tf
+import torch
+
+from .modeling_conv_bert import ConvBertConfig, ConvBertForSequenceClassification, load_tf_weights_in_conv_bert
+from .modeling_tf_conv_bert import TFConvBertForSequenceClassification
+from .tokenization_conv_bert import ConvBertTokenizer
+
+
+model_path = "/home/abhishek/huggingface/models/convbert_models/convbert_medium_small"
+
+if __name__ == "__main__":
+    tokenizer = ConvBertTokenizer.from_pretrained(model_path)
+    model = ConvBertForSequenceClassification.from_pretrained(model_path)
+    tf_model = TFConvBertForSequenceClassification.from_pretrained(model_path)
+
+    print(model.dummy_inputs)
+    print(model(**model.dummy_inputs))
+
+    # tf_model = TFConvBertModel(conf)
+
+    print(tf_model.dummy_inputs)
+    print(tf_model(tf_model.dummy_inputs))
+
+    # last_hidden_states = outputs.last_hidden_state
+    # print(last_hidden_states)
+
+    import torch
+
+    from transformers import BertForSequenceClassification, BertTokenizer
+
+    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
+    model = BertForSequenceClassification.from_pretrained("bert-base-uncased", return_dict=True)
+    inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
+    labels = torch.tensor([1]).unsqueeze(0)  # Batch size 1
+    outputs = model(**inputs, labels=labels)
+    loss = outputs.loss
+    logits = outputs.logits
+    print(logits)

src/transformers/models/conv_bert/run_tf.py

@@ -0,0 +1,32 @@
+import os
+
+import tensorflow as tf
+import torch
+
+from .modeling_conv_bert import ConvBertConfig, ConvBertModel, load_tf_weights_in_conv_bert
+from .modeling_tf_conv_bert import TFConvBertModel
+from .tokenization_conv_bert import ConvBertTokenizer
+
+
+model_path = "/home/abhishek/huggingface/models/convbert_models/convbert_medium_small"
+
+if __name__ == "__main__":
+    conf = ConvBertConfig.from_json_file(os.path.join(model_path, "config.json"))
+    tokenizer = ConvBertTokenizer.from_pretrained(model_path)
+    tf_checkpoint_path = os.path.join(model_path, "model.ckpt")
+    model = ConvBertModel(conf)
+
+    model = load_tf_weights_in_conv_bert(model, conf, tf_checkpoint_path)
+    # model.save_pretrained(model_path)
+
+    tf_model = TFConvBertModel.from_pretrained(model_path, from_pt=True)
+    tf_model.trainable = False
+    # tf_model.save_pretrained(model_path)
+
+    # symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights
+    # for symbolic_weight in symbolic_weights:
+    #     sw_name = symbolic_weight.name
+    #     print(symbolic_weight)
+
+    print(tf_model.dummy_inputs)
+    print(tf_model(tf_model.dummy_inputs).last_hidden_state)

src/transformers/models/conv_bert/tokenization_conv_bert.py

@@ -0,0 +1,72 @@
+# coding=utf-8
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for ConvBERT."""
+from ...utils import logging
+from ..bert.tokenization_bert import BertTokenizer
+from ..bert.tokenization_bert_fast import BertTokenizerFast
+
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "vocab_file": {
+        "conv-bert-base-uncased": "https://huggingface.co/conv-bert-base-uncased/resolve/main/vocab.txt",
+    }
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "conv-bert-base-uncased": 512,
+}
+
+
+PRETRAINED_INIT_CONFIGURATION = {
+    "conv-bert-base-uncased": {"do_lower_case": False},
+}
+
+
+class ConvBertTokenizer(BertTokenizer):
+    r"""
+    Construct a ConvBERT tokenizer.
+
+    :class:`~transformers.ConvBertTokenizer` is identical to :class:`~transformers.BertTokenizer` and runs end-to-end
+    tokenization: punctuation splitting and wordpiece.
+
+    Refer to superclass :class:`~transformers.BertTokenizer` for usage examples and documentation concerning
+    parameters.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
+
+
+class ConvBertTokenizerFast(BertTokenizerFast):
+    r"""
+    Construct a "fast" ConvBERT tokenizer (backed by HuggingFace's `tokenizers` library).
+
+    :class:`~transformers.ConvBertTokenizerFast` is identical to :class:`~transformers.BertTokenizerFast` and runs
+    end-to-end tokenization: punctuation splitting and wordpiece.
+
+    Refer to superclass :class:`~transformers.BertTokenizerFast` for usage examples and documentation concerning
+    parameters.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION

src/transformers/models/electra/modeling_tf_electra.py

@@ -136,6 +136,8 @@ class TFElectraSelfAttention(tf.keras.layers.Layer):
         context_layer = tf.reshape(
             context_layer, (batch_size, -1, self.all_head_size)
         )  # (batch_size, seq_len_q, all_head_size)
+        print(context_layer.shape)
+        print(attention_probs.shape)
         outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
 
         return outputs

tests/test_modeling_conv_bert.py

@@ -0,0 +1,265 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Testing suite for the PyTorch ConvBERT model. """
+
+
+import unittest
+
+from transformers import is_torch_available
+from transformers.testing_utils import require_torch, slow, torch_device
+
+from .test_configuration_common import ConfigTester
+from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
+
+
+if is_torch_available():
+    from transformers import (
+        ConvBertConfig,
+        ConvBertForMaskedLM,
+        ConvBertForMultipleChoice,
+        ConvBertForQuestionAnswering,
+        ConvBertForSequenceClassification,
+        ConvBertForTokenClassification,
+        ConvBertModel,
+    )
+    from transformers.models.conv_bert.modeling_conv_bert import CONV_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
+
+
+class ConvBertModelTester:
+    def __init__(
+        self,
+        parent,
+        batch_size=13,
+        seq_length=7,
+        is_training=True,
+        use_input_mask=True,
+        use_token_type_ids=True,
+        use_labels=True,
+        vocab_size=99,
+        hidden_size=32,
+        num_hidden_layers=5,
+        num_attention_heads=4,
+        intermediate_size=37,
+        hidden_act="gelu",
+        hidden_dropout_prob=0.1,
+        attention_probs_dropout_prob=0.1,
+        max_position_embeddings=512,
+        type_vocab_size=16,
+        type_sequence_label_size=2,
+        initializer_range=0.02,
+        num_labels=3,
+        num_choices=4,
+        scope=None,
+    ):
+        self.parent = parent
+        self.batch_size = batch_size
+        self.seq_length = seq_length
+        self.is_training = is_training
+        self.use_input_mask = use_input_mask
+        self.use_token_type_ids = use_token_type_ids
+        self.use_labels = use_labels
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.hidden_act = hidden_act
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.type_vocab_size = type_vocab_size
+        self.type_sequence_label_size = type_sequence_label_size
+        self.initializer_range = initializer_range
+        self.num_labels = num_labels
+        self.num_choices = num_choices
+        self.scope = scope
+
+    def prepare_config_and_inputs(self):
+        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
+
+        input_mask = None
+        if self.use_input_mask:
+            input_mask = random_attention_mask([self.batch_size, self.seq_length])
+
+        token_type_ids = None
+        if self.use_token_type_ids:
+            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)
+
+        sequence_labels = None
+        token_labels = None
+        choice_labels = None
+        if self.use_labels:
+            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
+            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
+            choice_labels = ids_tensor([self.batch_size], self.num_choices)
+
+        config = ConvBertConfig(
+            vocab_size=self.vocab_size,
+            hidden_size=self.hidden_size,
+            num_hidden_layers=self.num_hidden_layers,
+            num_attention_heads=self.num_attention_heads,
+            intermediate_size=self.intermediate_size,
+            hidden_act=self.hidden_act,
+            hidden_dropout_prob=self.hidden_dropout_prob,
+            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
+            max_position_embeddings=self.max_position_embeddings,
+            type_vocab_size=self.type_vocab_size,
+            is_decoder=False,
+            initializer_range=self.initializer_range,
+        )
+
+        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+
+    def create_and_check_model(
+        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+    ):
+        model = ConvBertModel(config=config)
+        model.to(torch_device)
+        model.eval()
+        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
+        result = model(input_ids, token_type_ids=token_type_ids)
+        result = model(input_ids)
+        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
+
+    def create_and_check_for_masked_lm(
+        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+    ):
+        model = ConvBertForMaskedLM(config=config)
+        model.to(torch_device)
+        model.eval()
+        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
+        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
+
+    def create_and_check_for_question_answering(
+        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+    ):
+        model = ConvBertForQuestionAnswering(config=config)
+        model.to(torch_device)
+        model.eval()
+        result = model(
+            input_ids,
+            attention_mask=input_mask,
+            token_type_ids=token_type_ids,
+            start_positions=sequence_labels,
+            end_positions=sequence_labels,
+        )
+        self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
+        self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))
+
+    def create_and_check_for_sequence_classification(
+        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+    ):
+        config.num_labels = self.num_labels
+        model = ConvBertForSequenceClassification(config)
+        model.to(torch_device)
+        model.eval()
+        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels)
+        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))
+
+    def create_and_check_for_token_classification(
+        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+    ):
+        config.num_labels = self.num_labels
+        model = ConvBertForTokenClassification(config=config)
+        model.to(torch_device)
+        model.eval()
+        result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
+        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
+
+    def create_and_check_for_multiple_choice(
+        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
+    ):
+        config.num_choices = self.num_choices
+        model = ConvBertForMultipleChoice(config=config)
+        model.to(torch_device)
+        model.eval()
+        multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+        multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+        multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
+        result = model(
+            multiple_choice_inputs_ids,
+            attention_mask=multiple_choice_input_mask,
+            token_type_ids=multiple_choice_token_type_ids,
+            labels=choice_labels,
+        )
+        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))
+
+    def prepare_config_and_inputs_for_common(self):
+        config_and_inputs = self.prepare_config_and_inputs()
+        (
+            config,
+            input_ids,
+            token_type_ids,
+            input_mask,
+            sequence_labels,
+            token_labels,
+            choice_labels,
+        ) = config_and_inputs
+        inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
+        return config, inputs_dict
+
+
+@require_torch
+class ConvBertModelTest(ModelTesterMixin, unittest.TestCase):
+
+    all_model_classes = (
+        (
+            ConvBertModel,
+            ConvBertForMaskedLM,
+            ConvBertForMultipleChoice,
+            ConvBertForQuestionAnswering,
+            ConvBertForSequenceClassification,
+            ConvBertForTokenClassification,
+        )
+        if is_torch_available()
+        else ()
+    )
+
+    def setUp(self):
+        self.model_tester = ConvBertModelTester(self)
+        self.config_tester = ConfigTester(self, config_class=ConvBertConfig, hidden_size=37)
+
+    def test_config(self):
+        self.config_tester.run_common_tests()
+
+    def test_model(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_model(*config_and_inputs)
+
+    def test_for_masked_lm(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_for_masked_lm(*config_and_inputs)
+
+    def test_for_multiple_choice(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs)
+
+    def test_for_question_answering(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_for_question_answering(*config_and_inputs)
+
+    def test_for_sequence_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs)
+
+    def test_for_token_classification(self):
+        config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.model_tester.create_and_check_for_token_classification(*config_and_inputs)
+
+    @slow
+    def test_model_from_pretrained(self):
+        for model_name in CONV_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
+            model = ConvBertModel.from_pretrained(model_name)
+            self.assertIsNotNone(model)