Fixing doc style.

warnings.

docs/source/model_doc/blenderbot.rst

@@ -95,11 +95,3 @@ See :obj:`transformers.BartForConditionalGeneration` for arguments to `forward`
 
 .. autoclass:: transformers.BlenderbotForConditionalGeneration
     :members:
-
-
-TFBlenderbotForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-See :obj:`transformers.TFBartForConditionalGeneration` for arguments to `forward` and `generate`
-
-.. autoclass:: transformers.TFBlenderbotForConditionalGeneration
-    :members:

docs/source/model_doc/marian.rst

@@ -129,9 +129,3 @@ MarianMTModel
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.MarianMTModel
-
-
-TFMarianMTModel
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFMarianMTModel

docs/source/model_doc/mbart.rst

@@ -79,11 +79,4 @@ MBartForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.MBartForConditionalGeneration
-    :members:
-
-
-TFMBartForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFMBartForConditionalGeneration
-    :members:
+    :members: forward

docs/source/model_doc/pegasus.rst

@@ -95,9 +95,3 @@ PegasusForConditionalGeneration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.PegasusForConditionalGeneration
-
-
-TFPegasusForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFPegasusForConditionalGeneration

src/transformers/__init__.py

@@ -670,7 +670,6 @@ if is_tf_available():
         TFBertModel,
         TFBertPreTrainedModel,
     )
-    from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
     from .modeling_tf_camembert import (
         TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFCamembertForMaskedLM,
@@ -751,8 +750,6 @@ if is_tf_available():
         TFLxmertPreTrainedModel,
         TFLxmertVisualFeatureEncoder,
     )
-    from .modeling_tf_marian import TFMarianMTModel
-    from .modeling_tf_mbart import TFMBartForConditionalGeneration
     from .modeling_tf_mobilebert import (
         TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFMobileBertForMaskedLM,
@@ -774,7 +771,6 @@ if is_tf_available():
         TFOpenAIGPTModel,
         TFOpenAIGPTPreTrainedModel,
     )
-    from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
     from .modeling_tf_roberta import (
         TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFRobertaForMaskedLM,

src/transformers/file_utils.py

@@ -651,7 +651,7 @@ TF_TOKEN_CLASSIFICATION_SAMPLE = r"""
         >>> import tensorflow as tf
 
         >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True))
+        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True)
 
         >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")
         >>> input_ids = inputs["input_ids"]
@@ -669,7 +669,7 @@ TF_QUESTION_ANSWERING_SAMPLE = r"""
         >>> import tensorflow as tf
 
         >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True))
+        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True)
 
         >>> question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
         >>> input_dict = tokenizer(question, text, return_tensors='tf')
@@ -688,7 +688,7 @@ TF_SEQUENCE_CLASSIFICATION_SAMPLE = r"""
         >>> import tensorflow as tf
 
         >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True))
+        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True)
 
         >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")
         >>> inputs["labels"] = tf.reshape(tf.constant(1), (-1, 1)) # Batch size 1
@@ -705,7 +705,7 @@ TF_MASKED_LM_SAMPLE = r"""
         >>> import tensorflow as tf
 
         >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True))
+        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True)
 
         >>> inputs = tokenizer("The capital of France is {mask}.", return_tensors="tf")
         >>> inputs["labels"] = tokenizer("The capital of France is Paris.", return_tensors="tf")["input_ids"]
@@ -722,7 +722,7 @@ TF_BASE_MODEL_SAMPLE = r"""
         >>> import tensorflow as tf
 
         >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True))
+        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True)
 
         >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")
         >>> outputs = model(inputs)
@@ -737,7 +737,7 @@ TF_MULTIPLE_CHOICE_SAMPLE = r"""
         >>> import tensorflow as tf
 
         >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True))
+        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True)
 
         >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
         >>> choice0 = "It is eaten with a fork and a knife."
@@ -758,7 +758,7 @@ TF_CAUSAL_LM_SAMPLE = r"""
         >>> import tensorflow as tf
 
         >>> tokenizer = {tokenizer_class}.from_pretrained('{checkpoint}')
-        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True))
+        >>> model = {model_class}.from_pretrained('{checkpoint}', return_dict=True)
 
         >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="tf")
         >>> outputs = model(inputs)

src/transformers/integrations.py

@@ -2,16 +2,24 @@
 import math
 import os
 
+from .utils import logging
 
-# Import 3rd-party integrations first:
+
+logger = logging.get_logger(__name__)
+
+
+# Import 3rd-party integrations before ML frameworks:
 
 try:
     # Comet needs to be imported before any ML frameworks
     import comet_ml  # noqa: F401
 
-    # XXX: there should be comet_ml.ensure_configured(), like `wandb`, for now emulate it
-    comet_ml.Experiment(project_name="ensure_configured")
-    _has_comet = True
+    if comet_ml.config.get_config("comet.api_key"):
+        _has_comet = True
+    else:
+        if os.getenv("COMET_MODE", "").upper() != "DISABLED":
+            logger.warning("comet_ml is installed but `COMET_API_KEY` is not set.")
+        _has_comet = False
 except (ImportError, ValueError):
     _has_comet = False
 
@@ -63,13 +71,9 @@ except ImportError:
 
 # No transformer imports above this point
 
-from .file_utils import is_torch_tpu_available
-from .trainer_callback import TrainerCallback
-from .trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun
-from .utils import logging
-
-
-logger = logging.get_logger(__name__)
+from .file_utils import is_torch_tpu_available  # noqa: E402
+from .trainer_callback import TrainerCallback  # noqa: E402
+from .trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun  # noqa: E402
 
 
 # Integration functions:

src/transformers/modeling_bart.py

@@ -427,6 +427,7 @@ class DecoderLayer(nn.Module):
         output_attentions=False,
     ):
         residual = x
+
         if layer_state is None:
             layer_state = {}
         if self.normalize_before:
@@ -446,7 +447,7 @@ class DecoderLayer(nn.Module):
         if not self.normalize_before:
             x = self.self_attn_layer_norm(x)
 
-        # Cross-Attention Block
+        # Cross attention
         residual = x
         assert self.encoder_attn.cache_key != self.self_attn.cache_key
         if self.normalize_before:
@@ -627,6 +628,7 @@ class BartDecoder(nn.Module):
         encoder_hidden_states = encoder_hidden_states.transpose(0, 1)
 
         next_cache = next_decoder_cache if use_cache else None
+
         if not return_dict:
             return tuple(v for v in [x, next_cache, all_hidden_states, all_self_attns] if v is not None)
         return BaseModelOutputWithPast(

src/transformers/modeling_tf_auto.py

@@ -41,10 +41,6 @@ from .configuration_auto import (
     XLNetConfig,
     replace_list_option_in_docstrings,
 )
-from .configuration_blenderbot import BlenderbotConfig
-from .configuration_marian import MarianConfig
-from .configuration_mbart import MBartConfig
-from .configuration_pegasus import PegasusConfig
 from .configuration_utils import PretrainedConfig
 from .file_utils import add_start_docstrings
 from .modeling_tf_albert import (
@@ -67,7 +63,6 @@ from .modeling_tf_bert import (
     TFBertLMHeadModel,
     TFBertModel,
 )
-from .modeling_tf_blenderbot import TFBlenderbotForConditionalGeneration
 from .modeling_tf_camembert import (
     TFCamembertForMaskedLM,
     TFCamembertForMultipleChoice,
@@ -113,8 +108,6 @@ from .modeling_tf_funnel import (
 )
 from .modeling_tf_gpt2 import TFGPT2LMHeadModel, TFGPT2Model
 from .modeling_tf_longformer import TFLongformerForMaskedLM, TFLongformerForQuestionAnswering, TFLongformerModel
-from .modeling_tf_marian import TFMarianMTModel
-from .modeling_tf_mbart import TFMBartForConditionalGeneration
 from .modeling_tf_mobilebert import (
     TFMobileBertForMaskedLM,
     TFMobileBertForMultipleChoice,
@@ -125,7 +118,6 @@ from .modeling_tf_mobilebert import (
     TFMobileBertModel,
 )
 from .modeling_tf_openai import TFOpenAIGPTLMHeadModel, TFOpenAIGPTModel
-from .modeling_tf_pegasus import TFPegasusForConditionalGeneration
 from .modeling_tf_roberta import (
     TFRobertaForMaskedLM,
     TFRobertaForMultipleChoice,
@@ -218,7 +210,6 @@ TF_MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
         (T5Config, TFT5ForConditionalGeneration),
         (DistilBertConfig, TFDistilBertForMaskedLM),
         (AlbertConfig, TFAlbertForMaskedLM),
-        (MarianConfig, TFMarianMTModel),
         (BartConfig, TFBartForConditionalGeneration),
         (CamembertConfig, TFCamembertForMaskedLM),
         (XLMRobertaConfig, TFXLMRobertaForMaskedLM),
@@ -270,16 +261,8 @@ TF_MODEL_FOR_MASKED_LM_MAPPING = OrderedDict(
     ]
 )
 
-
 TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
-    [
-        (T5Config, TFT5ForConditionalGeneration),
-        (MarianConfig, TFMarianMTModel),
-        (MBartConfig, TFMBartForConditionalGeneration),
-        (PegasusConfig, TFPegasusForConditionalGeneration),
-        (BlenderbotConfig, TFBlenderbotForConditionalGeneration),
-        (BartConfig, TFBartForConditionalGeneration),
-    ]
+    [(T5Config, TFT5ForConditionalGeneration), (BartConfig, TFBartForConditionalGeneration)]
 )
 
 TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(

src/transformers/modeling_tf_bart.py

@@ -19,10 +19,9 @@ import random
 import warnings
 from typing import Dict, Optional, Tuple
 
-import numpy as np
 import tensorflow as tf
 from tensorflow import Tensor
-from tensorflow.keras.layers import Dense, Layer, LayerNormalization
+from tensorflow.keras.layers import Dense, LayerNormalization
 
 from .activations_tf import ACT2FN
 from .configuration_bart import BartConfig
@@ -44,6 +43,7 @@ from .utils import logging
 
 
 _CONFIG_FOR_DOC = "BartConfig"
+_TOKENIZER_FOR_DOC = "BartTokenizer"
 
 BART_START_DOCSTRING = r"""
 
@@ -218,21 +218,22 @@ PAST_KV_DEPRECATION_WARNING = (
 )
 
 
-class TFEncoderLayer(Layer):
+class TFEncoderLayer(tf.keras.layers.Layer):
     def __init__(self, config: BartConfig, **kwargs):
         super().__init__(**kwargs)
         self.embed_dim = config.d_model
         self.self_attn = TFAttention(
             self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout, name="self_attn"
         )
-        self.normalize_before = config.normalize_before
-        self.self_attn_layer_norm = LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
-        self.dropout = config.dropout
+
+        self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
+        self.dropout_wt = tf.keras.layers.Dropout(config.dropout)
         self.activation_fn = ACT2FN[config.activation_function]
-        self.activation_dropout = config.activation_dropout
+        self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout)
         self.fc1 = Dense(config.encoder_ffn_dim, name="fc1")
         self.fc2 = Dense(self.embed_dim, name="fc2")
         self.final_layer_norm = LayerNormalization(epsilon=1e-5, name="final_layer_norm")
+        self.normalize_before = config.normalize_before
 
     def call(self, x, encoder_padding_mask, training=False):
         """
@@ -251,7 +252,7 @@ class TFEncoderLayer(Layer):
             x = self.self_attn_layer_norm(x)
         x, self_attn_weights = self.self_attn(query=x, key=x, key_padding_mask=encoder_padding_mask)
         assert x.shape == residual.shape, f"Self attn modified the shape of query {residual.shape} to {x.shape}"
-        x = tf.nn.dropout(x, rate=self.dropout if training else 0)
+        x = self.dropout_wt(x, training=training)
         x = residual + x
         if not self.normalize_before:
             x = self.self_attn_layer_norm(x)
@@ -260,9 +261,9 @@ class TFEncoderLayer(Layer):
         if self.normalize_before:
             x = self.final_layer_norm(x)
         x = self.activation_fn(self.fc1(x))
-        x = tf.nn.dropout(x, rate=self.self.activation_dropout if training else 0)
+        x = self.activation_dropout(x, training=training)
         x = self.fc2(x)
-        x = tf.nn.dropout(x, rate=self.dropout if training else 0)
+        x = self.dropout_wt(x, training=training)
         x = residual + x
         if not self.normalize_before:
             x = self.final_layer_norm(x)
@@ -270,7 +271,7 @@ class TFEncoderLayer(Layer):
         return x, self_attn_weights
 
 
-class TFBartEncoder(Layer):
+class TFBartEncoder(tf.keras.layers.Layer):
     # config_class = BartConfig
     """
     Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
@@ -288,30 +289,26 @@ class TFBartEncoder(Layer):
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
 
-        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0
+        embed_dim = embed_tokens.vocab_size
+        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
         self.padding_idx = config.pad_token_id
         self.max_source_positions = config.max_position_embeddings
 
         self.embed_tokens = embed_tokens
-        if config.static_position_embeddings:
-            self.embed_positions = TFSinusoidalPositionalEmbedding(
-                config.max_position_embeddings,
-                config.d_model,
-                name="embed_positions",
-            )
-        else:
-            self.embed_positions = TFLearnedPositionalEmbedding(
-                config.max_position_embeddings,
-                config.d_model,
-                self.padding_idx,
-                config.extra_pos_embeddings,
-                name="embed_positions",
-            )
+        self.embed_positions = TFLearnedPositionalEmbedding(
+            config.max_position_embeddings,
+            embed_tokens.hidden_size,
+            self.padding_idx,
+            config.extra_pos_embeddings,
+            name="embed_positions",
+        )
         self.layers = [TFEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)]
-        self.layernorm_embedding = (
-            LayerNormalization(epsilon=1e-5, name="layernorm_embedding") if config.normalize_embedding else Layer()
+        self.layernorm_embedding = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
+        self.layer_norm = (
+            tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")
+            if config.add_final_layer_norm
+            else None
         )
-        self.layer_norm = LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None
         self.return_dict = config.return_dict
 
     def call(
@@ -350,7 +347,7 @@ class TFBartEncoder(Layer):
             ), f"expected attention_mask._rank() to be a 2D tensor got {attention_mask._rank()}"
             attention_mask = tf.cast(attention_mask, dtype=tf.float32)
             attention_mask = (1.0 - attention_mask) * LARGE_NEGATIVE
-        inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale
+        inputs_embeds = self.embed_tokens(input_ids)
         embed_pos = self.embed_positions(input_ids)
         x = inputs_embeds + embed_pos
         x = self.layernorm_embedding(x)
@@ -387,7 +384,7 @@ class TFBartEncoder(Layer):
         return TFBaseModelOutput(last_hidden_state=x, hidden_states=encoder_states, attentions=all_attentions)
 
 
-class TFDecoderLayer(Layer):
+class TFDecoderLayer(tf.keras.layers.Layer):
     def __init__(self, config: BartConfig, **kwargs):
         super().__init__(**kwargs)
         self.embed_dim = config.d_model
@@ -400,9 +397,8 @@ class TFDecoderLayer(Layer):
         self.dropout = config.dropout
         self.activation_fn = ACT2FN[config.activation_function]
         self.activation_dropout = config.activation_dropout
-        self.normalize_before = config.normalize_before
 
-        self.self_attn_layer_norm = LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
+        self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
         self.encoder_attn = TFAttention(
             self.embed_dim,
             config.decoder_attention_heads,
@@ -410,10 +406,10 @@ class TFDecoderLayer(Layer):
             encoder_decoder_attention=True,
             name="encoder_attn",
         )
-        self.encoder_attn_layer_norm = LayerNormalization(epsilon=1e-5, name="encoder_attn_layer_norm")
+        self.encoder_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="encoder_attn_layer_norm")
         self.fc1 = Dense(config.decoder_ffn_dim, name="fc1")
         self.fc2 = Dense(self.embed_dim, name="fc2")
-        self.final_layer_norm = LayerNormalization(epsilon=1e-5, name="final_layer_norm")
+        self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm")
 
     def call(
         self,
@@ -437,12 +433,10 @@ class TFDecoderLayer(Layer):
 
             Tuple containing, encoded output of shape `(seq_len, batch, embed_dim)`, self_attn_weights, layer_state
         """
-        residual = x  # Make a copy of the input tensor to add later.
         if layer_state is None:
             layer_state = {}
-        if self.normalize_before:
-            x = self.self_attn_layer_norm(x)
 
+        residual = x  # Make a copy of the input tensor to add later.
         # next line mutates layer state and we need a copy of it
         x, self_attn_weights = self.self_attn(
             query=x,
@@ -453,12 +447,9 @@ class TFDecoderLayer(Layer):
         )
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
-        if not self.normalize_before:
-            x = self.self_attn_layer_norm(x)
-        # Cross-Attention Block
+        x = self.self_attn_layer_norm(x)
         residual = x
-        if self.normalize_before:
-            x = self.encoder_attn_layer_norm(x)
+        # Cross-Attention
         x, _ = self.encoder_attn(
             query=x,
             key=encoder_hidden_states,
@@ -467,19 +458,16 @@ class TFDecoderLayer(Layer):
         )
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
-        if not self.normalize_before:
-            x = self.encoder_attn_layer_norm(x)
-        # Fully Connected
+
+        x = self.encoder_attn_layer_norm(x)
+
         residual = x
-        if self.normalize_before:
-            x = self.final_layer_norm(x)
         x = self.activation_fn(self.fc1(x))
         x = tf.nn.dropout(x, rate=self.activation_dropout if training else 0)
         x = self.fc2(x)
         x = tf.nn.dropout(x, rate=self.dropout if training else 0)
         x = residual + x
-        if not self.normalize_before:
-            x = self.final_layer_norm(x)
+        x = self.final_layer_norm(x)
         return (
             x,
             self_attn_weights,
@@ -487,7 +475,7 @@ class TFDecoderLayer(Layer):
         )  # just self_attn weights for now, following t5, layer_state = cache for decoding
 
 
-class TFBartDecoder(Layer):
+class TFBartDecoder(tf.keras.layers.Layer):
     """
     Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a :class:`TFDecoderLayer`
 
@@ -503,27 +491,26 @@ class TFBartDecoder(Layer):
         self.max_target_positions = config.max_position_embeddings
         self.embed_tokens = embed_tokens
         self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0
-        if config.static_position_embeddings:
-            self.embed_positions = TFSinusoidalPositionalEmbedding(
-                config.max_position_embeddings,
-                config.d_model,
-                name="embed_positions",
-            )
-        else:
-            self.embed_positions = TFLearnedPositionalEmbedding(
-                config.max_position_embeddings,
-                config.d_model,
-                self.padding_idx,
-                config.extra_pos_embeddings,
-                name="embed_positions",
-            )
+        self.embed_positions = TFLearnedPositionalEmbedding(
+            config.max_position_embeddings,
+            config.d_model,
+            self.padding_idx,
+            config.extra_pos_embeddings,
+            name="embed_positions",
+        )
         self.layers = [TFDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)]
         self.layernorm_embedding = (
-            LayerNormalization(epsilon=1e-5, name="layernorm_embedding") if config.normalize_embedding else Layer()
+            tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
+            if config.normalize_embedding
+            else tf.identity
+        )
+        self.layer_norm = (
+            tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")
+            if config.add_final_layer_norm
+            else None
         )
-        self.layer_norm = LayerNormalization(epsilon=1e-5, name="layer_norm") if config.add_final_layer_norm else None
 
-        self.dropout = config.dropout
+        self.dropout = tf.keras.layers.Dropout(config.dropout)
         self.output_hidden_states = config.output_hidden_states
         self.output_attentions = config.output_attentions
         self.use_cache = config.use_cache
@@ -566,7 +553,7 @@ class TFBartDecoder(Layer):
             x = self.layernorm_embedding(x) + positions
         else:
             x = self.layernorm_embedding(x + positions)
-        x = tf.nn.dropout(x, rate=self.dropout if training else 0)
+        x = self.dropout(x)
 
         # Convert to Bart output format: (seq_len, BS, model_dim) -> (BS, seq_len, model_dim)
         x = tf.transpose(x, perm=(1, 0, 2))
@@ -636,7 +623,7 @@ def _reorder_buffer(attn_cache, new_order):
     return attn_cache
 
 
-class TFAttention(Layer):
+class TFAttention(tf.keras.layers.Layer):
     """Multi-headed attention from "Attention Is All You Need"""
 
     def __init__(
@@ -793,57 +780,6 @@ class TFLearnedPositionalEmbedding(TFSharedEmbeddings):
         return super().call(positions + self.offset)  # super object is not callable for some reason
 
 
-class TFSinusoidalPositionalEmbedding(TFSharedEmbeddings):
-    """This module produces sinusoidal positional embeddings of any length."""
-
-    def __init__(self, num_positions, embedding_dim, **kwargs):
-
-        if embedding_dim % 2 != 0:
-            raise NotImplementedError(f"odd embedding_dim {embedding_dim} not supported")
-        super().__init__(
-            num_positions,
-            embedding_dim,
-            **kwargs,
-        )
-        # self.weight = self._init_weight(*self.weight.shape)
-
-    def build(self, input_shape):
-        """
-        Build shared token embedding layer Shared weights logic adapted from
-        https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
-        """
-        super().build(input_shape)  # Instantiates self.weight so it can be loaded
-        weight: np.ndarray = self._init_weight(self.vocab_size, self.hidden_size)
-        self.set_weights([weight])  # overwrite self.weight to correct value
-
-    @staticmethod
-    def _init_weight(n_pos, dim):
-        """
-        Identical to the XLM create_sinusoidal_embeddings except features are not interleaved. The cos features are in
-        the 2nd half of the vector. [dim // 2:]
-        """
-        position_enc = np.array(
-            [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
-        )
-        # index 0 is all zero
-        position_enc[:, 0 : dim // 2] = np.sin(position_enc[:, 0::2])
-        position_enc[:, dim // 2 :] = np.cos(position_enc[:, 1::2])
-        # convert to tensor
-        table = tf.convert_to_tensor(position_enc, dtype=tf.float32)
-        tf.stop_gradient(table)
-        return table
-
-    def call(self, input_ids, use_cache=False):
-        """Input is expected to be of size [bsz x seqlen]."""
-        bsz, seq_len = input_ids.shape[:2]
-        if use_cache:
-            positions = tf.fill((1, 1), seq_len - 1)
-        else:
-            # starts at 0, ends at 1-seq_len
-            positions = tf.range(0, seq_len, delta=1, dtype=tf.int32, name="range")
-        return super().call(positions)
-
-
 # Public API
 
 
@@ -1014,20 +950,16 @@ class TFBartForConditionalGeneration(TFPretrainedBartModel):
     base_model_prefix = "model"
     authorized_missing_keys = [
         r"final_logits_bias",
-    ]
-    authorized_unexpected_keys = [
-        r"model.encoder.embed_tokens.weight",
-        r"model.decoder.embed_tokens.weight",
+        r"encoder\.version",
+        r"decoder\.version",
+        "model.encoder.embed_tokens.weight",
+        "model.decoder.embed_tokens.weight",
     ]
 
     def __init__(self, config: BartConfig, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
         self.model = TFBartModel(config, name="model")
         self.use_cache = config.use_cache
-        # final_bias_logits is registered as a buffer in pytorch, so not trainable for the the sake of consistency.
-        self.final_logits_bias = self.add_weight(
-            name="/final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=False
-        )
 
     @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)
     @replace_return_docstrings(output_type=TFSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
@@ -1118,7 +1050,6 @@ class TFBartForConditionalGeneration(TFPretrainedBartModel):
             return_dict=True,  # TODO(SS): this may need to change to support compilation
         )
         logits = self.model.shared(outputs.last_hidden_state, mode="linear")
-        logits = logits + self.final_logits_bias
         loss = None if labels is None else self.compute_loss(labels, logits)
 
         past = outputs.past_key_values if cast_bool_to_primitive(use_cache, self.config.use_cache) else None
@@ -1165,7 +1096,7 @@ class TFBartForConditionalGeneration(TFPretrainedBartModel):
             ), f"decoder cached states must be truthy. got {decoder_cached_states} from the 2nd element of past"
         assert isinstance(
             encoder_outputs, TFBaseModelOutput
-        ), f"encoder_outputs should be a TFBaseModelOutput, Instead got {type(encoder_outputs)}."
+        ), "encoder_outputs should be a TFBaseModelOutput, Instead got "
         return {
             "inputs": None,  # encoder_outputs is defined. input_ids not needed
             "encoder_outputs": encoder_outputs,
@@ -1199,17 +1130,15 @@ class TFBartForConditionalGeneration(TFPretrainedBartModel):
         return logits
 
     @staticmethod
-    def _force_token_id_to_be_generated(scores, token_id, inverted=False) -> None:
+    def _force_token_id_to_be_generated(scores, token_id) -> None:
         """force one of token_ids to be generated by setting prob of all other tokens to 0 (logprob=-float("inf"))"""
-        # TODO: https://github.com/huggingface/transformers/issues/7954
         output_list = []
+
         # Is there a better way to do scores[:, [x for if x != token_id]] = -float("inf") in TF?
         bs, vocab_size = scores.shape
-        inf_tensor = tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype)
         for x in range(vocab_size):
-            do_inf = (x == token_id) if inverted else (x != token_id)
-            if do_inf:
-                output_list.append(inf_tensor)
+            if x != token_id:
+                output_list.append(tf.convert_to_tensor([-float("inf")] * bs, dtype=scores.dtype))
             else:
                 output_list.append(scores[:, x])
         scores = tf.stack(output_list, axis=1, name="scores")

src/transformers/modeling_tf_blenderbot.py

@@ -1,41 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""TF BlenderBot model, ported from the fairseq repo."""
-from .configuration_blenderbot import BlenderbotConfig
-from .file_utils import add_start_docstrings
-from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
-from .utils import logging
-
-
-_CONFIG_FOR_DOC = "BlenderbotConfig"
-
-START_DOCSTRING = BART_START_DOCSTRING.replace(
-    "inherits from :class:`~transformers.TFPreTrainedModel`",
-    "inherits from :class:`~transformers.TFBartForConditionalGeneration`",
-).replace("BartConfig", _CONFIG_FOR_DOC)
-
-
-logger = logging.get_logger(__name__)
-
-
-@add_start_docstrings("Blenderbot model for open domain dialogue", START_DOCSTRING)
-class TFBlenderbotForConditionalGeneration(TFBartForConditionalGeneration):
-    config_class = BlenderbotConfig
-
-    def adjust_logits_during_generation(self, logits, cur_len, max_length):
-        self._force_token_id_to_be_generated(logits, self.config.pad_token_id, inverted=True)
-        if cur_len == max_length - 1 and self.config.eos_token_id is not None:
-            self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
-        return logits

src/transformers/modeling_tf_marian.py

@@ -1,48 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""TF Marian model, ported from the fairseq repo."""
-from .configuration_marian import MarianConfig
-from .file_utils import add_start_docstrings
-from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
-from .utils import logging
-
-
-_CONFIG_FOR_DOC = "MarianConfig"
-
-START_DOCSTRING = BART_START_DOCSTRING.replace(
-    "inherits from :class:`~transformers.TFPreTrainedModel`",
-    "inherits from :class:`~transformers.TFBartForConditionalGeneration`",
-).replace("BartConfig", _CONFIG_FOR_DOC)
-
-
-logger = logging.get_logger(__name__)
-
-
-@add_start_docstrings("Marian model for machine translation", START_DOCSTRING)
-class TFMarianMTModel(TFBartForConditionalGeneration):
-    authorized_missing_keys = [
-        r"model.encoder.embed_positions.weight",
-        r"model.decoder.embed_positions.weight",
-    ]
-    config_class = MarianConfig
-
-    def adjust_logits_during_generation(self, logits, cur_len, max_length):
-        """Never predict pad_token_id. Predict </s> when max_length is reached."""
-        self._force_token_id_to_be_generated(logits, self.config.pad_token_id, inverted=True)
-        if cur_len == max_length - 1 and self.config.eos_token_id is not None:
-            logits = self._force_token_id_to_be_generated(logits, self.config.eos_token_id)
-        return logits
-
-    # All the code is in src/transformers/modeling_tf_bart.py

src/transformers/modeling_tf_mbart.py

@@ -1,36 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""TF mBART model, originally from fairseq."""
-from .configuration_mbart import MBartConfig
-from .file_utils import add_start_docstrings
-from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
-from .utils import logging
-
-
-_CONFIG_FOR_DOC = "MBartConfig"
-
-START_DOCSTRING = BART_START_DOCSTRING.replace(
-    "inherits from :class:`~transformers.TFPreTrainedModel`",
-    "inherits from :class:`~transformers.TFBartForConditionalGeneration`",
-).replace("BartConfig", _CONFIG_FOR_DOC)
-
-
-logger = logging.get_logger(__name__)
-
-
-@add_start_docstrings("mBART (multilingual BART) model for machine translation", START_DOCSTRING)
-class TFMBartForConditionalGeneration(TFBartForConditionalGeneration):
-    config_class = MBartConfig
-    # All the code is in src/transformers/modeling_tf_bart.py

src/transformers/modeling_tf_pegasus.py

@@ -1,41 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""TF Pegasus model, ported from the fairseq repo."""
-from .configuration_pegasus import PegasusConfig
-from .file_utils import add_start_docstrings
-from .modeling_tf_bart import BART_START_DOCSTRING, TFBartForConditionalGeneration
-from .utils import logging
-
-
-_CONFIG_FOR_DOC = "PegasusConfig"
-
-START_DOCSTRING = BART_START_DOCSTRING.replace(
-    "inherits from :class:`~transformers.TFPreTrainedModel`",
-    "inherits from :class:`~transformers.TFBartForConditionalGeneration`",
-).replace("BartConfig", _CONFIG_FOR_DOC)
-
-
-logger = logging.get_logger(__name__)
-
-
-@add_start_docstrings("Pegasus model for summarization", START_DOCSTRING)
-class TFPegasusForConditionalGeneration(TFBartForConditionalGeneration):
-    authorized_missing_keys = [
-        r"final_logits_bias",
-        r"model.encoder.embed_positions.weight",
-        r"model.decoder.embed_positions.weight",
-    ]
-    config_class = PegasusConfig
-    # All the code is in src/transformers/modeling_tf_bart.py

src/transformers/pipelines.py

@@ -177,6 +177,12 @@ class PipelineException(Exception):
         self.model = model
 
 
+class PipelineWarning(UserWarning):
+    """
+    Raised by a :class:`~transformers.Pipeline` when handling __call__.
+    """
+
+
 class ArgumentHandler(ABC):
     """
     Base interface for handling arguments for each :class:`~transformers.pipelines.Pipeline`.
@@ -683,6 +689,15 @@ class Pipeline(_ScikitCompat):
             return_tensors=self.framework,
             padding=padding,
         )
+        if "sequence-length-is-longer-than-the-specified-maximum" in self.tokenizer.deprecation_warnings:
+            max_length = self.tokenizer.model_max_length
+            length = inputs["input_ids"].shape[1]
+            warnings.warn(
+                "You input length was too long ({} > {}) for this model and was truncated.".format(length, max_length),
+                PipelineWarning,
+            )
+            for key in inputs:
+                inputs[key] = inputs[key][:, :max_length]
 
         return inputs
 
@@ -2830,7 +2845,7 @@ def pipeline(
                         '"translation" task was used, instead of "translation_XX_to_YY", defaulting to "{}"'.format(
                             task
                         ),
-                        UserWarning,
+                        PipelineWarning,
                     )
                     break
 

src/transformers/testing_utils.py

@@ -88,8 +88,8 @@ def is_pipeline_test(test_case):
     """
     Decorator marking a test as a pipeline test.
 
-    Pipeline tests are skipped by default and we can run only them by setting RUN_PIPELINE_TEST environment variable to
-    a truthy value and selecting the is_pipeline_test pytest mark.
+    Pipeline tests are skipped by default and we can run only them by setting RUN_PIPELINE_TESTS environment variable
+    to a truthy value and selecting the is_pipeline_test pytest mark.
 
     """
     if not _run_pipeline_tests:

src/transformers/tokenization_camembert_fast.py

@@ -66,30 +66,31 @@ class CamembertTokenizerFast(PreTrainedTokenizerFast):
     This tokenizer inherits from :class:`~transformers.PreTrainedTokenizerFast` which contains most of the main
     methods. Users should refer to this superclass for more information regarding those methods.
 
-    vocab_file (:obj:`str`): `SentencePiece <https://github.com/google/sentencepiece>`__ file (generally has a `.spm`
-    extension) that contains the vocabulary necessary to instantiate a tokenizer. bos_token (:obj:`str`, `optional`,
-    defaults to :obj:`"<s>"`): The beginning of sequence token that was used during pretraining. Can be used a sequence
-    classifier token.
+    Args:
+        vocab_file (:obj:`str`):
+            `SentencePiece <https://github.com/google/sentencepiece>`__ file (generally has a `.spm` extension) that
+            contains the vocabulary necessary to instantiate a tokenizer.
+        bos_token (:obj:`str`, `optional`, defaults to :obj:`"<s>"`):
+            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
 
             .. note::
 
-                When building a sequence using special tokens, this is not the token that is used for the beginning
-                of sequence. The token used is the :obj:`cls_token`.
+                When building a sequence using special tokens, this is not the token that is used for the beginning of
+                sequence. The token used is the :obj:`cls_token`.
         eos_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
             The end of sequence token.
 
             .. note::
 
-                When building a sequence using special tokens, this is not the token that is used for the end
-                of sequence. The token used is the :obj:`sep_token`.
+                When building a sequence using special tokens, this is not the token that is used for the end of
+                sequence. The token used is the :obj:`sep_token`.
         sep_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
-            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences
-            for sequence classification or for a text and a question for question answering.
-            It is also used as the last token of a sequence built with special tokens.
+            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
+            sequence classification or for a text and a question for question answering. It is also used as the last
+            token of a sequence built with special tokens.
         cls_token (:obj:`str`, `optional`, defaults to :obj:`"<s>"`):
-            The classifier token which is used when doing sequence classification (classification of the whole
-            sequence instead of per-token classification). It is the first token of the sequence when built with
-            special tokens.
+            The classifier token which is used when doing sequence classification (classification of the whole sequence
+            instead of per-token classification). It is the first token of the sequence when built with special tokens.
         unk_token (:obj:`str`, `optional`, defaults to :obj:`"<unk>"`):
             The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
             token instead.

src/transformers/tokenization_dpr.py

@@ -129,10 +129,10 @@ DPRReaderOutput = collections.namedtuple("DPRReaderOutput", ["start_logits", "en
 
 
 CUSTOM_DPR_READER_DOCSTRING = r"""
-        Return a dictionary with the token ids of the input strings and other information to give to
-        :obj:`.decode_best_spans`. It converts the strings of a question and different passages (title and text) in a
-        sequence of IDs (integers), using the tokenizer and vocabulary. The resulting :obj:`input_ids` is a matrix of
-        size :obj:`(n_passages, sequence_length)` with the format:
+    Return a dictionary with the token ids of the input strings and other information to give to
+    :obj:`.decode_best_spans`. It converts the strings of a question and different passages (title and text) in a
+    sequence of IDs (integers), using the tokenizer and vocabulary. The resulting :obj:`input_ids` is a matrix of size
+    :obj:`(n_passages, sequence_length)` with the format:
 
     ::
 
@@ -189,12 +189,12 @@ CUSTOM_DPR_READER_DOCSTRING = r"""
 
             `What are attention masks? <../glossary.html#attention-mask>`__
 
-    Return:
+    Returns:
         :obj:`Dict[str, List[List[int]]]`: A dictionary with the following keys:
 
         - ``input_ids``: List of token ids to be fed to a model.
         - ``attention_mask``: List of indices specifying which tokens should be attended to by the model.
-        """
+    """
 
 
 @add_start_docstrings(CUSTOM_DPR_READER_DOCSTRING)

src/transformers/tokenization_dpr_fast.py

@@ -132,12 +132,12 @@ DPRReaderOutput = collections.namedtuple("DPRReaderOutput", ["start_logits", "en
 
 
 CUSTOM_DPR_READER_DOCSTRING = r"""
-        Return a dictionary with the token ids of the input strings and other information to give to
-        :obj:`.decode_best_spans`. It converts the strings of a question and different passages (title and text) in a
-        sequence of IDs (integers), using the tokenizer and vocabulary. The resulting :obj:`input_ids` is a matrix of
-        size :obj:`(n_passages, sequence_length)` with the format:
+    Return a dictionary with the token ids of the input strings and other information to give to
+    :obj:`.decode_best_spans`. It converts the strings of a question and different passages (title and text) in a
+    sequence of IDs (integers), using the tokenizer and vocabulary. The resulting :obj:`input_ids` is a matrix of size
+    :obj:`(n_passages, sequence_length)` with the format:
 
-        [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>
+    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>
 
     Args:
         questions (:obj:`str` or :obj:`List[str]`):
@@ -195,7 +195,7 @@ CUSTOM_DPR_READER_DOCSTRING = r"""
 
         - ``input_ids``: List of token ids to be fed to a model.
         - ``attention_mask``: List of indices specifying which tokens should be attended to by the model.
-        """
+    """
 
 
 @add_start_docstrings(CUSTOM_DPR_READER_DOCSTRING)

src/transformers/trainer.py

@@ -26,18 +26,9 @@ import warnings
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
-import numpy as np
-import torch
-from packaging import version
-from torch import nn
-from torch.utils.data.dataloader import DataLoader
-from torch.utils.data.dataset import Dataset
-from torch.utils.data.distributed import DistributedSampler
-from torch.utils.data.sampler import RandomSampler, SequentialSampler
 
-from .data.data_collator import DataCollator, DataCollatorWithPadding, default_data_collator
-from .file_utils import WEIGHTS_NAME, is_datasets_available, is_in_notebook, is_torch_tpu_available
-from .integrations import (
+# Integrations must be imported before ML frameworks:
+from .integrations import (  # isort: split
     default_hp_search_backend,
     hp_params,
     is_comet_available,
@@ -49,6 +40,18 @@ from .integrations import (
     run_hp_search_optuna,
     run_hp_search_ray,
 )
+
+import numpy as np
+import torch
+from packaging import version
+from torch import nn
+from torch.utils.data.dataloader import DataLoader
+from torch.utils.data.dataset import Dataset
+from torch.utils.data.distributed import DistributedSampler
+from torch.utils.data.sampler import RandomSampler, SequentialSampler
+
+from .data.data_collator import DataCollator, DataCollatorWithPadding, default_data_collator
+from .file_utils import WEIGHTS_NAME, is_datasets_available, is_in_notebook, is_torch_tpu_available
 from .modeling_auto import MODEL_FOR_QUESTION_ANSWERING_MAPPING
 from .modeling_utils import PreTrainedModel
 from .optimization import AdamW, get_linear_schedule_with_warmup
@@ -191,18 +194,21 @@ class Trainer:
 
             The function may have zero argument, or a single one containing the optuna/Ray Tune trial object, to be
             able to choose different architectures according to hyper parameters (such as layer count, sizes of inner
-            layers, dropout probabilities etc). compute_metrics (:obj:`Callable[[EvalPrediction], Dict]`, `optional`):
+            layers, dropout probabilities etc).
+        compute_metrics (:obj:`Callable[[EvalPrediction], Dict]`, `optional`):
             The function that will be used to compute metrics at evaluation. Must take a
-            :class:`~transformers.EvalPrediction` and return a dictionary string to metric values. callbacks (List of
-            :obj:`~transformers.TrainerCallback`, `optional`): A list of callbacks to customize the training loop. Will
-            add those to the list of default callbacks detailed in :doc:`here <callback>`.
+            :class:`~transformers.EvalPrediction` and return a dictionary string to metric values.
+        callbacks (List of :obj:`~transformers.TrainerCallback`, `optional`):
+            A list of callbacks to customize the training loop. Will add those to the list of default callbacks
+            detailed in :doc:`here <callback>`.
 
             If you want to remove one of the default callbacks used, use the :meth:`Trainer.remove_callback` method.
-            optimizers (:obj:`Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR`, `optional`): A tuple
+        optimizers (:obj:`Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR`, `optional`): A tuple
             containing the optimizer and the scheduler to use. Will default to an instance of
             :class:`~transformers.AdamW` on your model and a scheduler given by
-            :func:`~transformers.get_linear_schedule_with_warmup` controlled by :obj:`args`. kwargs: Deprecated keyword
-            arguments.
+            :func:`~transformers.get_linear_schedule_with_warmup` controlled by :obj:`args`.
+        kwargs:
+            Deprecated keyword arguments.
     """
 
     def __init__(

src/transformers/trainer_tf.py

@@ -6,12 +6,18 @@ import os
 import warnings
 from typing import Callable, Dict, Optional, Tuple
 
+
+# Integrations must be imported before ML frameworks:
+from .integrations import (  # isort: split
+    is_comet_available,
+    is_wandb_available,
+)
+
 import numpy as np
 import tensorflow as tf
 from packaging.version import parse
 from tensorflow.python.distribute.values import PerReplica
 
-from .integrations import is_comet_available, is_wandb_available
 from .modeling_tf_utils import TFPreTrainedModel
 from .optimization_tf import GradientAccumulator, create_optimizer
 from .trainer_utils import PREFIX_CHECKPOINT_DIR, EvalPrediction, PredictionOutput, set_seed

src/transformers/training_args.py

@@ -144,29 +144,31 @@ class TrainingArguments:
             If using `nlp.Dataset` datasets, whether or not to automatically remove the columns unused by the model
             forward method.
 
-            (Note that this behavior is not implemented for :class:`~transformers.TFTrainer` yet.) label_names
-            (:obj:`List[str]`, `optional`): The list of keys in your dictionary of inputs that correspond to the
-            labels.
+            (Note that this behavior is not implemented for :class:`~transformers.TFTrainer` yet.)
+        label_names (:obj:`List[str]`, `optional`):
+            The list of keys in your dictionary of inputs that correspond to the labels.
 
             Will eventually default to :obj:`["labels"]` except if the model used is one of the
             :obj:`XxxForQuestionAnswering` in which case it will default to :obj:`["start_positions",
-            "end_positions"]`. load_best_model_at_end (:obj:`bool`, `optional`, defaults to :obj:`False`): Whether or
-            not to load the best model found during training at the end of training.
+            "end_positions"]`.
+        load_best_model_at_end (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether or not to load the best model found during training at the end of training.
 
             .. note::
 
                 When set to :obj:`True`, the parameters :obj:`save_steps` will be ignored and the model will be saved
                 after each evaluation.
-        metric_for_best_model (:obj:`str`, `optional`)
+        metric_for_best_model (:obj:`str`, `optional`):
             Use in conjunction with :obj:`load_best_model_at_end` to specify the metric to use to compare two different
             models. Must be the name of a metric returned by the evaluation with or without the prefix :obj:`"eval_"`.
             Will default to :obj:`"loss"` if unspecified and :obj:`load_best_model_at_end=True` (to use the evaluation
             loss).
 
             If you set this value, :obj:`greater_is_better` will default to :obj:`True`. Don't forget to set it to
-            :obj:`False` if your metric is better when lower. greater_is_better (:obj:`bool`, `optional`) Use in
-            conjunction with :obj:`load_best_model_at_end` and :obj:`metric_for_best_model` to specify if better models
-            should have a greater metric or not. Will default to:
+            :obj:`False` if your metric is better when lower.
+        greater_is_better (:obj:`bool`, `optional`):
+            Use in conjunction with :obj:`load_best_model_at_end` and :obj:`metric_for_best_model` to specify if better
+            models should have a greater metric or not. Will default to:
 
             - :obj:`True` if :obj:`metric_for_best_model` is set to a value that isn't :obj:`"loss"` or
               :obj:`"eval_loss"`.

src/transformers/utils/dummy_tf_objects.py

@@ -325,15 +325,6 @@ class TFBertPreTrainedModel:
         requires_tf(self)
 
 
-class TFBlenderbotForConditionalGeneration:
-    def __init__(self, *args, **kwargs):
-        requires_tf(self)
-
-    @classmethod
-    def from_pretrained(self, *args, **kwargs):
-        requires_tf(self)
-
-
 TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None
 
 
@@ -806,24 +797,6 @@ class TFLxmertVisualFeatureEncoder:
         requires_tf(self)
 
 
-class TFMarianMTModel:
-    def __init__(self, *args, **kwargs):
-        requires_tf(self)
-
-    @classmethod
-    def from_pretrained(self, *args, **kwargs):
-        requires_tf(self)
-
-
-class TFMBartForConditionalGeneration:
-    def __init__(self, *args, **kwargs):
-        requires_tf(self)
-
-    @classmethod
-    def from_pretrained(self, *args, **kwargs):
-        requires_tf(self)
-
-
 TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None
 
 
@@ -949,15 +922,6 @@ class TFOpenAIGPTPreTrainedModel:
         requires_tf(self)
 
 
-class TFPegasusForConditionalGeneration:
-    def __init__(self, *args, **kwargs):
-        requires_tf(self)
-
-    @classmethod
-    def from_pretrained(self, *args, **kwargs):
-        requires_tf(self)
-
-
 TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = None
 
 

tests/test_modeling_tf_bart.py

@@ -17,9 +17,7 @@
 import tempfile
 import unittest
 
-import numpy as np
-
-from transformers import BartConfig, BartTokenizer, is_tf_available
+from transformers import is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import is_pt_tf_cross_test, require_tf, slow
 
@@ -30,16 +28,12 @@ from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers import TFBartForConditionalGeneration, TFBartModel
-    from transformers.modeling_tf_bart import TFSinusoidalPositionalEmbedding
+    from transformers import BartConfig, TFBartForConditionalGeneration, TFBartModel
+    from transformers.tokenization_bart import BartTokenizer
 
 
 @require_tf
-class TFBartModelTester:
-    config_cls = BartConfig
-    config_updates = {}
-    hidden_act = "gelu"
-
+class ModelTester:
     def __init__(self, parent):
         self.parent = parent
         self.batch_size = 13
@@ -51,13 +45,14 @@ class TFBartModelTester:
         self.num_hidden_layers = 5
         self.num_attention_heads = 4
         self.intermediate_size = 37
-
+        self.hidden_act = "gelu"
         self.hidden_dropout_prob = 0.1
         self.attention_probs_dropout_prob = 0.1
         self.max_position_embeddings = 20
         self.eos_token_ids = [2]
         self.pad_token_id = 1
         self.bos_token_id = 0
+        # torch.manual_seed(0)
 
     def prepare_config_and_inputs_for_common(self):
         input_ids = ids_tensor([self.batch_size, self.seq_length - 1], self.vocab_size)
@@ -65,7 +60,7 @@ class TFBartModelTester:
         input_ids = tf.concat([input_ids, eos_tensor], axis=1)
         input_ids = tf.clip_by_value(input_ids, 3, self.vocab_size + 1)
 
-        config = self.config_cls(
+        config = BartConfig(
             vocab_size=self.vocab_size,
             d_model=self.hidden_size,
             encoder_layers=self.num_hidden_layers,
@@ -81,7 +76,6 @@ class TFBartModelTester:
             bos_token_id=self.bos_token_id,
             pad_token_id=self.pad_token_id,
             decoder_start_token_id=self.pad_token_id,
-            **self.config_updates,
         )
         inputs_dict = prepare_bart_inputs_dict(config, input_ids)
         return config, inputs_dict
@@ -107,10 +101,9 @@ class TestTFBart(TFModelTesterMixin, unittest.TestCase):
     all_generative_model_classes = (TFBartForConditionalGeneration,) if is_tf_available() else ()
     is_encoder_decoder = True
     test_pruning = False
-    model_tester_cls = TFBartModelTester
 
     def setUp(self):
-        self.model_tester = self.model_tester_cls(self)
+        self.model_tester = ModelTester(self)
         self.config_tester = ConfigTester(self, config_class=BartConfig)
 
     def test_config(self):
@@ -127,7 +120,7 @@ class TestTFBart(TFModelTesterMixin, unittest.TestCase):
         loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
         metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
 
-        model_class = self.all_generative_model_classes[0]
+        model_class = TFBartForConditionalGeneration
         input_ids = {
             "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
             "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
@@ -361,29 +354,3 @@ class FasterTFBartModelIntegrationTests(unittest.TestCase):
 
         expected = np.array([[-0.0828, -0.0251, -0.0674], [0.1277, 0.3311, -0.0255], [0.2613, -0.0840, -0.2763]])
         assert np.allclose(features[0, :3, :3].numpy(), expected, atol=1e-3)
-
-
-@require_tf
-class TestTFSinusoidalPositionalEmbeddings(unittest.TestCase):
-    desired_weights = [
-        [0, 0, 0, 0, 0],
-        [0.84147096, 0.82177866, 0.80180490, 0.78165019, 0.76140374],
-        [0.90929741, 0.93651021, 0.95829457, 0.97505713, 0.98720258],
-    ]
-
-    def test_positional_emb_cache_logic(self):
-        input_ids = _long_tensor([[4, 10]])
-        emb1 = TFSinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6)
-        no_cache = emb1(input_ids, use_cache=False)
-        yes_cache = emb1(input_ids, use_cache=True)
-        self.assertEqual((1, 1, 6), yes_cache.shape)  # extra dim to allow broadcasting, feel free to delete!
-
-        np.testing.assert_almost_equal(no_cache[-1].numpy(), yes_cache[0][0].numpy())
-
-    def test_positional_emb_weights_against_marian(self):
-        emb1 = TFSinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512)
-        emb1.build(None)
-        weights = emb1.weight.numpy()
-        for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
-            for j in range(5):
-                self.assertAlmostEqual(expected_weight[j], actual_weight[j], places=3)

tests/test_modeling_tf_blenderbot.py

@@ -1,132 +0,0 @@
-# coding=utf-8
-# Copyright 2020 HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import tempfile
-import unittest
-
-from tests.test_configuration_common import ConfigTester
-from tests.test_modeling_tf_bart import TFBartModelTester
-from tests.test_modeling_tf_common import TFModelTesterMixin
-from transformers import BlenderbotConfig, BlenderbotSmallTokenizer, is_tf_available
-from transformers.file_utils import cached_property
-from transformers.testing_utils import is_pt_tf_cross_test, require_tf, require_tokenizers, slow
-
-
-if is_tf_available():
-    import tensorflow as tf
-
-    from transformers import TFAutoModelForSeq2SeqLM, TFBlenderbotForConditionalGeneration
-
-
-class ModelTester(TFBartModelTester):
-    config_updates = dict(
-        normalize_before=True,
-        static_position_embeddings=True,
-        do_blenderbot_90_layernorm=True,
-        normalize_embeddings=True,
-    )
-    config_cls = BlenderbotConfig
-
-
-@require_tf
-class TestTFBlenderbotCommon(TFModelTesterMixin, unittest.TestCase):
-    all_model_classes = (TFBlenderbotForConditionalGeneration,) if is_tf_available() else ()
-    all_generative_model_classes = (TFBlenderbotForConditionalGeneration,) if is_tf_available() else ()
-    model_tester_cls = ModelTester
-    is_encoder_decoder = True
-    test_pruning = False
-
-    def setUp(self):
-        self.model_tester = self.model_tester_cls(self)
-        self.config_tester = ConfigTester(self, config_class=BlenderbotConfig)
-
-    def test_config(self):
-        self.config_tester.run_common_tests()
-
-    def test_inputs_embeds(self):
-        # inputs_embeds not supported
-        pass
-
-    def test_saved_model_with_hidden_states_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
-    def test_saved_model_with_attentions_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
-    def test_compile_tf_model(self):
-        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
-        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
-        metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
-
-        model_class = self.all_generative_model_classes[0]
-        input_ids = {
-            "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
-            "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
-        }
-
-        # Prepare our model
-        model = model_class(config)
-        model(self._prepare_for_class(inputs_dict, model_class))  # Model must be called before saving.
-        # Let's load it from the disk to be sure we can use pretrained weights
-        with tempfile.TemporaryDirectory() as tmpdirname:
-            model.save_pretrained(tmpdirname)
-            model = model_class.from_pretrained(tmpdirname)
-
-        outputs_dict = model(input_ids)
-        hidden_states = outputs_dict[0]
-
-        # Add a dense layer on top to test integration with other keras modules
-        outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
-
-        # Compile extended model
-        extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
-        extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
-
-
-@is_pt_tf_cross_test
-@require_tokenizers
-class TFBlenderbot90MIntegrationTests(unittest.TestCase):
-    src_text = [
-        "Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like   i'm going to throw up.\nand why is that?"
-    ]
-    model_name = "facebook/blenderbot-90M"
-
-    @cached_property
-    def tokenizer(self):
-        return BlenderbotSmallTokenizer.from_pretrained(self.model_name)
-
-    @cached_property
-    def model(self):
-        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
-        return model
-
-    @slow
-    def test_90_generation_from_long_input(self):
-        model_inputs = self.tokenizer(self.src_text, return_tensors="tf")
-        generated_ids = self.model.generate(
-            model_inputs.input_ids,
-            attention_mask=model_inputs.attention_mask,
-            num_beams=2,
-            use_cache=True,
-        )
-        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)[0]
-        assert generated_words in (
-            "i don't know. i just feel like i'm going to throw up. it's not fun.",
-            "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
-            "i'm not sure. i just feel like i've been in a bad situation.",
-        )

tests/test_modeling_tf_marian.py

@@ -1,179 +0,0 @@
-# coding=utf-8
-# Copyright 2020 HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import tempfile
-import unittest
-import warnings
-
-from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
-from transformers.file_utils import cached_property
-from transformers.testing_utils import is_pt_tf_cross_test, require_sentencepiece, require_tf, require_tokenizers, slow
-
-from .test_configuration_common import ConfigTester
-from .test_modeling_tf_bart import TFBartModelTester
-from .test_modeling_tf_common import TFModelTesterMixin
-
-
-if is_tf_available():
-    import tensorflow as tf
-
-    from transformers import TFAutoModelForSeq2SeqLM, TFMarianMTModel
-
-
-class ModelTester(TFBartModelTester):
-    config_updates = dict(static_position_embeddings=True, add_bias_logits=True)
-    config_cls = MarianConfig
-
-
-@require_tf
-class TestTFMarianCommon(TFModelTesterMixin, unittest.TestCase):
-    all_model_classes = (TFMarianMTModel,) if is_tf_available() else ()
-    all_generative_model_classes = (TFMarianMTModel,) if is_tf_available() else ()
-    model_tester_cls = ModelTester
-    is_encoder_decoder = True
-    test_pruning = False
-
-    def setUp(self):
-        self.model_tester = self.model_tester_cls(self)
-        self.config_tester = ConfigTester(self, config_class=MarianConfig)
-
-    def test_config(self):
-        self.config_tester.run_common_tests()
-
-    def test_inputs_embeds(self):
-        # inputs_embeds not supported
-        pass
-
-    def test_saved_model_with_hidden_states_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
-    def test_saved_model_with_attentions_output(self):
-        pass
-
-    def test_compile_tf_model(self):
-        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
-        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
-        metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
-
-        model_class = self.all_generative_model_classes[0]
-        input_ids = {
-            "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
-            "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
-        }
-
-        # Prepare our model
-        model = model_class(config)
-        model(self._prepare_for_class(inputs_dict, model_class))  # Model must be called before saving.
-        # Let's load it from the disk to be sure we can use pre-trained weights
-        with tempfile.TemporaryDirectory() as tmpdirname:
-            model.save_pretrained(tmpdirname)
-            model = model_class.from_pretrained(tmpdirname)
-
-        outputs_dict = model(input_ids)
-        hidden_states = outputs_dict[0]
-
-        # Add a dense layer on top to test integration with other keras modules
-        outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
-
-        # Compile extended model
-        extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
-        extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
-
-
-class AbstractMarianIntegrationTest(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls) -> None:
-        cls.model_name = f"Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"
-        return cls
-
-    @cached_property
-    def tokenizer(self) -> MarianTokenizer:
-        return AutoTokenizer.from_pretrained(self.model_name)
-
-    @property
-    def eos_token_id(self) -> int:
-        return self.tokenizer.eos_token_id
-
-    @cached_property
-    def model(self):
-        warnings.simplefilter("error")
-        model: TFMarianMTModel = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
-        assert isinstance(model, TFMarianMTModel)
-        c = model.config
-        self.assertListEqual(c.bad_words_ids, [[c.pad_token_id]])
-        self.assertEqual(c.max_length, 512)
-        self.assertEqual(c.decoder_start_token_id, c.pad_token_id)
-        return model
-
-    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
-        generated_words = self.translate_src_text(**tokenizer_kwargs)
-        self.assertListEqual(self.expected_text, generated_words)
-
-    def translate_src_text(self, **tokenizer_kwargs):
-        model_inputs = self.tokenizer.prepare_seq2seq_batch(
-            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
-        )
-        generated_ids = self.model.generate(
-            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
-        )
-        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)
-        return generated_words
-
-
-@is_pt_tf_cross_test
-@require_sentencepiece
-@require_tokenizers
-class TestMarian_en_zh(AbstractMarianIntegrationTest):
-    src = "en"
-    tgt = "zh"
-    src_text = ["My name is Wolfgang and I live in Berlin"]
-    expected_text = ["我叫沃尔夫冈 我住在柏林"]
-
-    @slow
-    def test_batch_generation_en_zh(self):
-        self._assert_generated_batch_equal_expected()
-
-
-@is_pt_tf_cross_test
-@require_sentencepiece
-@require_tokenizers
-class TestMarian_en_ROMANCE(AbstractMarianIntegrationTest):
-    """Multilingual on target side."""
-
-    src = "en"
-    tgt = "ROMANCE"
-    src_text = [
-        ">>fr<< Don't spend so much time watching TV.",
-        ">>pt<< Your message has been sent.",
-        ">>es<< He's two years older than me.",
-    ]
-    expected_text = [
-        "Ne passez pas autant de temps à regarder la télé.",
-        "A sua mensagem foi enviada.",
-        "Es dos años más viejo que yo.",
-    ]
-
-    @slow
-    def test_batch_generation_en_ROMANCE_multi(self):
-        self._assert_generated_batch_equal_expected()
-
-    @slow
-    def test_pipeline(self):
-        pipeline = TranslationPipeline(self.model, self.tokenizer, framework="tf")
-        output = pipeline(self.src_text)
-        self.assertEqual(self.expected_text, [x["translation_text"] for x in output])

tests/test_modeling_tf_mbart.py

@@ -1,134 +0,0 @@
-# coding=utf-8
-# Copyright 2020 HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import tempfile
-import unittest
-
-from tests.test_configuration_common import ConfigTester
-from tests.test_modeling_tf_bart import TFBartModelTester
-from tests.test_modeling_tf_common import TFModelTesterMixin
-from transformers import AutoTokenizer, MBartConfig, is_tf_available
-from transformers.file_utils import cached_property
-from transformers.testing_utils import is_pt_tf_cross_test, require_sentencepiece, require_tf, require_tokenizers, slow
-
-
-if is_tf_available():
-
-    import tensorflow as tf
-
-    from transformers import TFAutoModelForSeq2SeqLM, TFMBartForConditionalGeneration
-
-
-class ModelTester(TFBartModelTester):
-    config_updates = dict(normalize_before=True, add_final_layer_norm=True)
-    config_cls = MBartConfig
-
-
-@require_tf
-class TestTFMBartCommon(TFModelTesterMixin, unittest.TestCase):
-    all_model_classes = (TFMBartForConditionalGeneration,) if is_tf_available() else ()
-    all_generative_model_classes = (TFMBartForConditionalGeneration,) if is_tf_available() else ()
-    model_tester_cls = ModelTester
-    is_encoder_decoder = True
-    test_pruning = False
-
-    def setUp(self):
-        self.model_tester = self.model_tester_cls(self)
-        self.config_tester = ConfigTester(self, config_class=MBartConfig)
-
-    def test_config(self):
-        self.config_tester.run_common_tests()
-
-    def test_inputs_embeds(self):
-        # inputs_embeds not supported
-        pass
-
-    def test_saved_model_with_hidden_states_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
-    def test_saved_model_with_attentions_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
-    def test_compile_tf_model(self):
-        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
-        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
-        metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
-
-        model_class = self.all_generative_model_classes[0]
-        input_ids = {
-            "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
-            "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
-        }
-
-        # Prepare our model
-        model = model_class(config)
-        model(self._prepare_for_class(inputs_dict, model_class))  # Model must be called before saving.
-        # Let's load it from the disk to be sure we can use pretrained weights
-        with tempfile.TemporaryDirectory() as tmpdirname:
-            model.save_pretrained(tmpdirname)
-            model = model_class.from_pretrained(tmpdirname)
-
-        outputs_dict = model(input_ids)
-        hidden_states = outputs_dict[0]
-
-        # Add a dense layer on top to test integration with other keras modules
-        outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
-
-        # Compile extended model
-        extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
-        extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
-
-
-@is_pt_tf_cross_test
-@require_sentencepiece
-@require_tokenizers
-class TestMBartEnRO(unittest.TestCase):
-    src_text = [
-        " UN Chief Says There Is No Military Solution in Syria",
-    ]
-    expected_text = [
-        "Şeful ONU declară că nu există o soluţie militară în Siria",
-    ]
-    model_name = "facebook/mbart-large-en-ro"
-
-    @cached_property
-    def tokenizer(self):
-        return AutoTokenizer.from_pretrained(self.model_name)
-
-    @cached_property
-    def model(self):
-        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
-        return model
-
-    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
-        generated_words = self.translate_src_text(**tokenizer_kwargs)
-        self.assertListEqual(self.expected_text, generated_words)
-
-    def translate_src_text(self, **tokenizer_kwargs):
-        model_inputs = self.tokenizer.prepare_seq2seq_batch(
-            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
-        )
-        generated_ids = self.model.generate(
-            model_inputs.input_ids, attention_mask=model_inputs.attention_mask, num_beams=2
-        )
-        generated_words = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
-        return generated_words
-
-    @slow
-    def test_batch_generation_en_ro(self):
-        self._assert_generated_batch_equal_expected()

tests/test_modeling_tf_pegasus.py

@@ -1,141 +0,0 @@
-# coding=utf-8
-# Copyright 2020 HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import tempfile
-import unittest
-
-from transformers import AutoTokenizer, PegasusConfig, is_tf_available
-from transformers.file_utils import cached_property
-from transformers.testing_utils import is_pt_tf_cross_test, require_sentencepiece, require_tf, require_tokenizers, slow
-
-from .test_configuration_common import ConfigTester
-from .test_modeling_pegasus import PGE_ARTICLE, XSUM_ENTRY_LONGER
-from .test_modeling_tf_bart import TFBartModelTester
-from .test_modeling_tf_common import TFModelTesterMixin
-
-
-if is_tf_available():
-    import tensorflow as tf
-
-    from transformers import TFAutoModelForSeq2SeqLM, TFPegasusForConditionalGeneration
-
-
-class ModelTester(TFBartModelTester):
-    config_updates = dict(
-        normalize_before=True,
-        static_position_embeddings=True,
-    )
-    hidden_act = "relu"
-    config_cls = PegasusConfig
-
-
-@require_tf
-class TestTFPegasusCommon(TFModelTesterMixin, unittest.TestCase):
-    all_model_classes = (TFPegasusForConditionalGeneration,) if is_tf_available() else ()
-    all_generative_model_classes = (TFPegasusForConditionalGeneration,) if is_tf_available() else ()
-    model_tester_cls = ModelTester
-    is_encoder_decoder = True
-    test_pruning = False
-
-    def setUp(self):
-        self.model_tester = self.model_tester_cls(self)
-        self.config_tester = ConfigTester(self, config_class=PegasusConfig)
-
-    def test_config(self):
-        self.config_tester.run_common_tests()
-
-    def test_inputs_embeds(self):
-        # inputs_embeds not supported
-        pass
-
-    def test_saved_model_with_hidden_states_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
-    def test_saved_model_with_attentions_output(self):
-        # Should be uncommented during patrick TF refactor
-        pass
-
-    def test_compile_tf_model(self):
-        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
-        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
-        metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")
-
-        model_class = self.all_generative_model_classes[0]
-        input_ids = {
-            "decoder_input_ids": tf.keras.Input(batch_shape=(2, 2000), name="decoder_input_ids", dtype="int32"),
-            "input_ids": tf.keras.Input(batch_shape=(2, 2000), name="input_ids", dtype="int32"),
-        }
-
-        # Prepare our model
-        model = model_class(config)
-        model(self._prepare_for_class(inputs_dict, model_class))  # Model must be called before saving.
-        # Let's load it from the disk to be sure we can use pretrained weights
-        with tempfile.TemporaryDirectory() as tmpdirname:
-            model.save_pretrained(tmpdirname)
-            model = model_class.from_pretrained(tmpdirname)
-
-        outputs_dict = model(input_ids)
-        hidden_states = outputs_dict[0]
-
-        # Add a dense layer on top to test integration with other keras modules
-        outputs = tf.keras.layers.Dense(2, activation="softmax", name="outputs")(hidden_states)
-
-        # Compile extended model
-        extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs])
-        extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric])
-
-
-@is_pt_tf_cross_test
-@require_sentencepiece
-@require_tokenizers
-class TFPegasusIntegrationTests(unittest.TestCase):
-    src_text = [PGE_ARTICLE, XSUM_ENTRY_LONGER]
-    expected_text = [
-        "California's largest electricity provider has cut power to hundreds of thousands of customers in an effort to reduce the risk of wildfires.",
-        'N-Dubz have revealed they\'re "grateful" to have been nominated for four Mobo Awards.',
-    ]  # differs slightly from pytorch, likely due to numerical differences in linear layers
-    model_name = "google/pegasus-xsum"
-
-    @cached_property
-    def tokenizer(self):
-        return AutoTokenizer.from_pretrained(self.model_name)
-
-    @cached_property
-    def model(self):
-        model = TFAutoModelForSeq2SeqLM.from_pretrained(self.model_name, from_pt=True)
-        return model
-
-    def _assert_generated_batch_equal_expected(self, **tokenizer_kwargs):
-        generated_words = self.translate_src_text(**tokenizer_kwargs)
-        assert self.expected_text == generated_words
-
-    def translate_src_text(self, **tokenizer_kwargs):
-        model_inputs = self.tokenizer.prepare_seq2seq_batch(
-            src_texts=self.src_text, **tokenizer_kwargs, return_tensors="tf"
-        )
-        generated_ids = self.model.generate(
-            model_inputs.input_ids,
-            attention_mask=model_inputs.attention_mask,
-            num_beams=2,
-            use_cache=True,
-        )
-        generated_words = self.tokenizer.batch_decode(generated_ids.numpy(), skip_special_tokens=True)
-        return generated_words
-
-    @slow
-    def test_batch_generation(self):
-        self._assert_generated_batch_equal_expected()

tests/test_pipelines_sentiment_analysis.py

@@ -1,5 +1,11 @@
 import unittest
 
+import pytest
+
+from transformers import pipeline
+from transformers.pipelines import PipelineWarning
+from transformers.testing_utils import require_torch, slow
+
 from .test_pipelines_common import MonoInputPipelineCommonMixin
 
 
@@ -10,3 +16,44 @@ class SentimentAnalysisPipelineTests(MonoInputPipelineCommonMixin, unittest.Test
     ]  # Default model - Models tested without the @slow decorator
     large_models = [None]  # Models tested with the @slow decorator
     mandatory_keys = {"label", "score"}  # Keys which should be in the output
+
+    @require_torch
+    def test_input_too_long(self):
+        model = self.small_models[0]
+        pipe = pipeline(self.pipeline_task, model=model)
+
+        with pytest.warns(PipelineWarning, match=r".*truncated.*"):
+            pipe(
+                """
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. At ultrices mi tempus imperdiet nulla malesuada. Nam libero justo laoreet sit amet cursus. Libero volutpat sed cras ornare arcu dui. Nunc aliquet bibendum enim facilisis gravida neque convallis. Odio pellentesque diam volutpat commodo sed egestas. Malesuada nunc vel risus commodo viverra maecenas accumsan. Id semper risus in hendrerit gravida. Habitant morbi tristique senectus et netus. Habitant morbi tristique senectus et netus. Id semper risus in hendrerit gravida rutrum quisque non. Faucibus vitae aliquet nec ullamcorper sit amet risus nullam eget. Non pulvinar neque laoreet suspendisse interdum consectetur libero id. Quis commodo odio aenean sed adipiscing diam. Ut diam quam nulla porttitor massa id. Posuere lorem ipsum dolor sit amet consectetur. Sollicitudin nibh sit amet commodo nulla facilisi nullam vehicula. Mattis rhoncus urna neque viverra justo nec. Odio ut sem nulla pharetra diam sit amet.
+
+Magna fringilla urna porttitor rhoncus dolor. Lorem ipsum dolor sit amet consectetur adipiscing elit duis tristique. Sit amet consectetur adipiscing elit. Auctor elit sed vulputate mi sit. Ac turpis egestas sed tempus. Ut aliquam purus sit amet. Id semper risus in hendrerit gravida rutrum. A diam sollicitudin tempor id eu. Lorem ipsum dolor sit amet. Enim neque volutpat ac tincidunt. Dictum sit amet justo donec enim diam. Sapien faucibus et molestie ac. Dictum sit amet justo donec enim diam vulputate ut pharetra. Porttitor eget dolor morbi non arcu risus. Viverra nam libero justo laoreet. Consectetur purus ut faucibus pulvinar. Nunc mi ipsum faucibus vitae aliquet nec ullamcorper sit amet. Auctor eu augue ut lectus arcu. Ultricies mi eget mauris pharetra et ultrices. Volutpat diam ut venenatis tellus.
+
+Tortor at auctor urna nunc id cursus. Massa eget egestas purus viverra accumsan in nisl. Sed lectus vestibulum mattis ullamcorper velit sed ullamcorper. Morbi tincidunt ornare massa eget egestas. Tincidunt vitae semper quis lectus nulla at. Viverra nam libero justo laoreet sit amet cursus sit. A iaculis at erat pellentesque. A pellentesque sit amet porttitor eget dolor morbi non. Massa sed elementum tempus egestas sed sed risus pretium quam. Ac turpis egestas maecenas pharetra convallis. Nisi quis eleifend quam adipiscing vitae proin.
+
+Morbi enim nunc faucibus a. Vel quam elementum pulvinar etiam non quam. Egestas dui id ornare arcu odio ut. Ut ornare lectus sit amet est placerat in. Ut pharetra sit amet aliquam id diam. Arcu ac tortor dignissim convallis aenean et tortor at risus. Phasellus faucibus scelerisque eleifend donec pretium vulputate sapien nec sagittis. Vestibulum rhoncus est pellentesque elit ullamcorper dignissim. Eros in cursus turpis massa tincidunt. Non sodales neque sodales ut etiam sit. Ultricies lacus sed turpis tincidunt id aliquet. In nisl nisi scelerisque eu ultrices vitae auctor. Eget mi proin sed libero enim sed faucibus. Orci dapibus ultrices in iaculis nunc sed augue lacus. Commodo elit at imperdiet dui accumsan sit amet. Ac odio tempor orci dapibus. Ullamcorper morbi tincidunt ornare massa eget. Sed euismod nisi porta lorem mollis aliquam.
+
+Dictumst vestibulum rhoncus est pellentesque elit ullamcorper dignissim cras tincidunt. Viverra suspendisse potenti nullam ac tortor vitae purus. Ornare massa eget egestas purus. Parturient montes nascetur ridiculus mus mauris vitae ultricies. Maecenas accumsan lacus vel facilisis. Consectetur lorem donec massa sapien faucibus et molestie. Elit ut aliquam purus sit amet luctus. Quis auctor elit sed vulputate mi sit amet mauris. Id leo in vitae turpis massa sed. Consequat ac felis donec et odio pellentesque diam volutpat commodo. Platea dictumst vestibulum rhoncus est pellentesque elit ullamcorper dignissim. Nec ultrices dui sapien eget mi proin sed libero. Quisque non tellus orci ac auctor augue.
+"""
+            )
+
+    @require_torch
+    @slow
+    def test_input_too_long_roberta_openai_detector(self):
+        model = "roberta-base-openai-detector"
+        pipe = pipeline(self.pipeline_task, model=model)
+
+        with pytest.warns(PipelineWarning, match=r".*truncated.*"):
+            pipe(
+                """
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. At ultrices mi tempus imperdiet nulla malesuada. Nam libero justo laoreet sit amet cursus. Libero volutpat sed cras ornare arcu dui. Nunc aliquet bibendum enim facilisis gravida neque convallis. Odio pellentesque diam volutpat commodo sed egestas. Malesuada nunc vel risus commodo viverra maecenas accumsan. Id semper risus in hendrerit gravida. Habitant morbi tristique senectus et netus. Habitant morbi tristique senectus et netus. Id semper risus in hendrerit gravida rutrum quisque non. Faucibus vitae aliquet nec ullamcorper sit amet risus nullam eget. Non pulvinar neque laoreet suspendisse interdum consectetur libero id. Quis commodo odio aenean sed adipiscing diam. Ut diam quam nulla porttitor massa id. Posuere lorem ipsum dolor sit amet consectetur. Sollicitudin nibh sit amet commodo nulla facilisi nullam vehicula. Mattis rhoncus urna neque viverra justo nec. Odio ut sem nulla pharetra diam sit amet.
+
+Magna fringilla urna porttitor rhoncus dolor. Lorem ipsum dolor sit amet consectetur adipiscing elit duis tristique. Sit amet consectetur adipiscing elit. Auctor elit sed vulputate mi sit. Ac turpis egestas sed tempus. Ut aliquam purus sit amet. Id semper risus in hendrerit gravida rutrum. A diam sollicitudin tempor id eu. Lorem ipsum dolor sit amet. Enim neque volutpat ac tincidunt. Dictum sit amet justo donec enim diam. Sapien faucibus et molestie ac. Dictum sit amet justo donec enim diam vulputate ut pharetra. Porttitor eget dolor morbi non arcu risus. Viverra nam libero justo laoreet. Consectetur purus ut faucibus pulvinar. Nunc mi ipsum faucibus vitae aliquet nec ullamcorper sit amet. Auctor eu augue ut lectus arcu. Ultricies mi eget mauris pharetra et ultrices. Volutpat diam ut venenatis tellus.
+
+Tortor at auctor urna nunc id cursus. Massa eget egestas purus viverra accumsan in nisl. Sed lectus vestibulum mattis ullamcorper velit sed ullamcorper. Morbi tincidunt ornare massa eget egestas. Tincidunt vitae semper quis lectus nulla at. Viverra nam libero justo laoreet sit amet cursus sit. A iaculis at erat pellentesque. A pellentesque sit amet porttitor eget dolor morbi non. Massa sed elementum tempus egestas sed sed risus pretium quam. Ac turpis egestas maecenas pharetra convallis. Nisi quis eleifend quam adipiscing vitae proin.
+
+Morbi enim nunc faucibus a. Vel quam elementum pulvinar etiam non quam. Egestas dui id ornare arcu odio ut. Ut ornare lectus sit amet est placerat in. Ut pharetra sit amet aliquam id diam. Arcu ac tortor dignissim convallis aenean et tortor at risus. Phasellus faucibus scelerisque eleifend donec pretium vulputate sapien nec sagittis. Vestibulum rhoncus est pellentesque elit ullamcorper dignissim. Eros in cursus turpis massa tincidunt. Non sodales neque sodales ut etiam sit. Ultricies lacus sed turpis tincidunt id aliquet. In nisl nisi scelerisque eu ultrices vitae auctor. Eget mi proin sed libero enim sed faucibus. Orci dapibus ultrices in iaculis nunc sed augue lacus. Commodo elit at imperdiet dui accumsan sit amet. Ac odio tempor orci dapibus. Ullamcorper morbi tincidunt ornare massa eget. Sed euismod nisi porta lorem mollis aliquam.
+
+Dictumst vestibulum rhoncus est pellentesque elit ullamcorper dignissim cras tincidunt. Viverra suspendisse potenti nullam ac tortor vitae purus. Ornare massa eget egestas purus. Parturient montes nascetur ridiculus mus mauris vitae ultricies. Maecenas accumsan lacus vel facilisis. Consectetur lorem donec massa sapien faucibus et molestie. Elit ut aliquam purus sit amet luctus. Quis auctor elit sed vulputate mi sit amet mauris. Id leo in vitae turpis massa sed. Consequat ac felis donec et odio pellentesque diam volutpat commodo. Platea dictumst vestibulum rhoncus est pellentesque elit ullamcorper dignissim. Nec ultrices dui sapien eget mi proin sed libero. Quisque non tellus orci ac auctor augue.
+"""
+            )

tests/test_pipelines_translation.py

@@ -3,6 +3,7 @@ import unittest
 import pytest
 
 from transformers import pipeline
+from transformers.pipelines import PipelineWarning
 from transformers.testing_utils import is_pipeline_test, require_torch, slow
 
 from .test_pipelines_common import MonoInputPipelineCommonMixin
@@ -44,7 +45,7 @@ class TranslationNewFormatPipelineTests(unittest.TestCase):
     @require_torch
     def test_translation_default_language_selection(self):
         model = "patrickvonplaten/t5-tiny-random"
-        with pytest.warns(UserWarning, match=r".*translation_en_to_de.*"):
+        with pytest.warns(PipelineWarning, match=r".*translation_en_to_de.*"):
             nlp = pipeline(task="translation", model=model)
         self.assertEqual(nlp.task, "translation_en_to_de")
 

utils/check_repo.py

@@ -67,7 +67,6 @@ MODEL_NAME_TO_DOC_FILE = {
     "xlm_prophetnet": "xlmprophetnet.rst",
     "xlm_roberta": "xlmroberta.rst",
     "bert_generation": "bertgeneration.rst",
-    "marian": "marian.rst",
 }
 
 # This is to make sure the transformers module imported is the one in the repo.
@@ -149,6 +148,7 @@ def get_model_doc_files():
     _ignore_modules = [
         "auto",
         "dialogpt",
+        "marian",
         "retribert",
     ]
     doc_files = []
@@ -245,7 +245,6 @@ def check_models_are_documented(module, doc_file):
 def _get_model_name(module):
     """ Get the model name for the module defining it."""
     splits = module.__name__.split("_")
-
     # Secial case for transfo_xl
     if splits[-1] == "xl":
         return "_".join(splits[-2:])

utils/style_doc.py

@@ -312,10 +312,11 @@ class DocstringStyler(CodeStyler):
     """Class to style docstrings that take the main method from `CodeStyler`."""
 
     def is_no_style_block(self, line):
+        if _re_textual_blocks.search(line) is not None:
+            return False
         if _re_example.search(line) is not None:
             return True
         return _re_code_block.search(line) is not None
-        # return super().is_no_style_block(line) is not None
 
     def is_comment_or_textual_block(self, line):
         if _re_return.search(line) is not None: