code formating

.circleci/config.yml

@@ -101,7 +101,7 @@ jobs:
             # we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
             - run: sudo pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
             - run: sudo pip install .[tf,torch,quality]
-            - run: black --check --line-length 119 --target-version py35 examples templates tests src utils
+            - run: black --check --line-length 119 examples templates tests src utils
             - run: isort --check-only --recursive examples templates tests src utils
             - run: flake8 examples templates tests src utils
     check_repository_consistency:

Makefile

@@ -3,14 +3,14 @@
 # Check that source code meets quality standards
 
 quality:
-	black --check --line-length 119 --target-version py35 examples templates tests src utils
+	black --check --line-length 119 examples templates tests src utils
 	isort --check-only --recursive examples templates tests src utils
 	flake8 examples templates tests src utils
 
 # Format source code automatically
 
 style:
-	black --line-length 119 --target-version py35 examples templates tests src utils
+	black --line-length 119 examples templates tests src utils
 	isort --recursive examples templates tests src utils
 
 # Run tests for the library

README.md

@@ -160,8 +160,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training
 12. **[T5](https://github.com/google-research/text-to-text-transfer-transformer)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
 13. **[XLM-RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/xlmr)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov.
 14. **[MMBT](https://github.com/facebookresearch/mmbt/)** (from Facebook), released together with the paper a [Supervised Multimodal Bitransformers for Classifying Images and Text](https://arxiv.org/pdf/1909.02950.pdf) by Douwe Kiela, Suvrat Bhooshan, Hamed Firooz, Davide Testuggine.
-15. **[Other community models](https://huggingface.co/models)**, contributed by the [community](https://huggingface.co/users).
-16. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
+15. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR.
 
 These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html).
 

docs/source/converting_tensorflow_models.rst

@@ -3,12 +3,6 @@ Converting Tensorflow Checkpoints
 
 A command-line interface is provided to convert original Bert/GPT/GPT-2/Transformer-XL/XLNet/XLM checkpoints in models than be loaded using the ``from_pretrained`` methods of the library.
 
-.. note::
-    Since 2.3.0 the conversion script is now part of the transformers CLI (**transformers-cli**)
-    available in any transformers >= 2.3.0 installation.
-
-    The documentation below reflects the **transformers-cli convert** command format.
-
 BERT
 ^^^^
 
@@ -26,10 +20,10 @@ Here is an example of the conversion process for a pre-trained ``BERT-Base Uncas
 
    export BERT_BASE_DIR=/path/to/bert/uncased_L-12_H-768_A-12
 
-   transformers-cli convert --model_type bert \
-     --tf_checkpoint $BERT_BASE_DIR/bert_model.ckpt \
-     --config $BERT_BASE_DIR/bert_config.json \
-     --pytorch_dump_output $BERT_BASE_DIR/pytorch_model.bin
+   transformers bert \
+     $BERT_BASE_DIR/bert_model.ckpt \
+     $BERT_BASE_DIR/bert_config.json \
+     $BERT_BASE_DIR/pytorch_model.bin
 
 You can download Google's pre-trained models for the conversion `here <https://github.com/google-research/bert#pre-trained-models>`__.
 
@@ -42,12 +36,10 @@ Here is an example of the conversion process for a pre-trained OpenAI GPT model,
 
    export OPENAI_GPT_CHECKPOINT_FOLDER_PATH=/path/to/openai/pretrained/numpy/weights
 
-   transformers-cli convert --model_type gpt \
-     --tf_checkpoint $OPENAI_GPT_CHECKPOINT_FOLDER_PATH \
-     --pytorch_dump_output $PYTORCH_DUMP_OUTPUT \
-     [--config OPENAI_GPT_CONFIG] \
-     [--finetuning_task_name OPENAI_GPT_FINETUNED_TASK] \
-
+   transformers gpt \
+     $OPENAI_GPT_CHECKPOINT_FOLDER_PATH \
+     $PYTORCH_DUMP_OUTPUT \
+     [OPENAI_GPT_CONFIG]
 
 OpenAI GPT-2
 ^^^^^^^^^^^^
@@ -58,11 +50,10 @@ Here is an example of the conversion process for a pre-trained OpenAI GPT-2 mode
 
    export OPENAI_GPT2_CHECKPOINT_PATH=/path/to/gpt2/pretrained/weights
 
-   transformers-cli convert --model_type gpt2 \
-     --tf_checkpoint $OPENAI_GPT2_CHECKPOINT_PATH \
-     --pytorch_dump_output $PYTORCH_DUMP_OUTPUT \
-     [--config OPENAI_GPT2_CONFIG] \
-     [--finetuning_task_name OPENAI_GPT2_FINETUNED_TASK]
+   transformers gpt2 \
+     $OPENAI_GPT2_CHECKPOINT_PATH \
+     $PYTORCH_DUMP_OUTPUT \
+     [OPENAI_GPT2_CONFIG]
 
 Transformer-XL
 ^^^^^^^^^^^^^^
@@ -73,28 +64,27 @@ Here is an example of the conversion process for a pre-trained Transformer-XL mo
 
    export TRANSFO_XL_CHECKPOINT_FOLDER_PATH=/path/to/transfo/xl/checkpoint
 
-   transformers-cli convert --model_type transfo_xl \
-     --tf_checkpoint $TRANSFO_XL_CHECKPOINT_FOLDER_PATH \
-     --pytorch_dump_output $PYTORCH_DUMP_OUTPUT \
-     [--config TRANSFO_XL_CONFIG] \
-     [--finetuning_task_name TRANSFO_XL_FINETUNED_TASK]
+   transformers transfo_xl \
+     $TRANSFO_XL_CHECKPOINT_FOLDER_PATH \
+     $PYTORCH_DUMP_OUTPUT \
+     [TRANSFO_XL_CONFIG]
 
 
 XLNet
 ^^^^^
 
-Here is an example of the conversion process for a pre-trained XLNet model:
+Here is an example of the conversion process for a pre-trained XLNet model, fine-tuned on STS-B using the TensorFlow script:
 
 .. code-block:: shell
 
    export TRANSFO_XL_CHECKPOINT_PATH=/path/to/xlnet/checkpoint
    export TRANSFO_XL_CONFIG_PATH=/path/to/xlnet/config
 
-   transformers-cli convert --model_type xlnet \
-     --tf_checkpoint $TRANSFO_XL_CHECKPOINT_PATH \
-     --config $TRANSFO_XL_CONFIG_PATH \
-     --pytorch_dump_output $PYTORCH_DUMP_OUTPUT \
-     [--finetuning_task_name XLNET_FINETUNED_TASK] \
+   transformers xlnet \
+     $TRANSFO_XL_CHECKPOINT_PATH \
+     $TRANSFO_XL_CONFIG_PATH \
+     $PYTORCH_DUMP_OUTPUT \
+     STS-B \
 
 
 XLM
@@ -106,8 +96,6 @@ Here is an example of the conversion process for a pre-trained XLM model:
 
    export XLM_CHECKPOINT_PATH=/path/to/xlm/checkpoint
 
-   transformers-cli convert --model_type xlm \
-     --tf_checkpoint $XLM_CHECKPOINT_PATH \
-     --pytorch_dump_output $PYTORCH_DUMP_OUTPUT
-    [--config XML_CONFIG] \
-    [--finetuning_task_name XML_FINETUNED_TASK]
+   transformers xlm \
+     $XLM_CHECKPOINT_PATH \
+     $PYTORCH_DUMP_OUTPUT \

examples/distillation/run_squad_w_distillation.py

@@ -56,7 +56,6 @@ from ..utils_squad import (
     write_predictions,
     write_predictions_extended,
 )
-
 # The follwing import is the official SQuAD evaluation script (2.0).
 # You can remove it from the dependencies if you are using this script outside of the library
 # We've added it here for automated tests (see examples/test_examples.py file)

examples/run_lm_finetuning.py

@@ -28,7 +28,6 @@ import pickle
 import random
 import re
 import shutil
-from typing import Tuple
 
 import numpy as np
 import torch
@@ -54,7 +53,6 @@ from transformers import (
     OpenAIGPTConfig,
     OpenAIGPTLMHeadModel,
     OpenAIGPTTokenizer,
-    PreTrainedTokenizer,
     RobertaConfig,
     RobertaForMaskedLM,
     RobertaTokenizer,
@@ -86,7 +84,7 @@ class TextDataset(Dataset):
         assert os.path.isfile(file_path)
         directory, filename = os.path.split(file_path)
         cached_features_file = os.path.join(
-            directory, args.model_name_or_path + "_cached_lm_" + str(block_size) + "_" + filename
+            directory, args.model_name_or_path + "_cached_lm_" + str(block_size) + "_" + filename + ".bin"
         )
 
         if os.path.exists(cached_features_file) and not args.overwrite_cache:
@@ -166,7 +164,7 @@ def _rotate_checkpoints(args, checkpoint_prefix, use_mtime=False):
         shutil.rmtree(checkpoint)
 
 
-def mask_tokens(inputs: torch.Tensor, tokenizer: PreTrainedTokenizer, args) -> Tuple[torch.Tensor, torch.Tensor]:
+def mask_tokens(inputs, tokenizer, args):
     """ Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. """
     labels = inputs.clone()
     # We sample a few tokens in each sequence for masked-LM training (with probability args.mlm_probability defaults to 0.15 in Bert/RoBERTa)
@@ -627,6 +625,12 @@ def main():
         args.config_name if args.config_name else args.model_name_or_path,
         cache_dir=args.cache_dir if args.cache_dir else None,
     )
+
+    # Desactivate past output for now (reduce memory)
+    # we use it only for GPT/GPT2 generation or specific XLNet/Transformer-XL trainings (not implemented currently)
+    if hasattr(config, "output_past"):
+        config.output_past = False
+
     tokenizer = tokenizer_class.from_pretrained(
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,

examples/run_tf_ner.py

@@ -9,6 +9,7 @@ import re
 import numpy as np
 import tensorflow as tf
 from absl import app, flags, logging
+from fastprogress import master_bar, progress_bar
 from seqeval import metrics
 
 from transformers import (
@@ -28,12 +29,6 @@ from transformers import (
 from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file
 
 
-try:
-    from fastprogress import master_bar, progress_bar
-except ImportError:
-    from fastprogress.fastprogress import master_bar, progress_bar
-
-
 ALL_MODELS = sum(
     (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)), ()
 )

src/transformers/__init__.py

@@ -31,7 +31,6 @@ from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, Open
 from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig
 from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config
 from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
-
 # Configurations
 from .configuration_utils import PretrainedConfig
 from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig
@@ -56,7 +55,6 @@ from .data import (
     xnli_processors,
     xnli_tasks_num_labels,
 )
-
 # Files and general utilities
 from .file_utils import (
     CONFIG_NAME,
@@ -73,10 +71,8 @@ from .file_utils import (
     is_tf_available,
     is_torch_available,
 )
-
 # Model Cards
 from .modelcard import ModelCard
-
 # TF 2.0 <=> PyTorch conversion utilities
 from .modeling_tf_pytorch_utils import (
     convert_tf_weight_name_to_pt_weight_name,
@@ -87,7 +83,6 @@ from .modeling_tf_pytorch_utils import (
     load_tf2_model_in_pytorch_model,
     load_tf2_weights_in_pytorch_model,
 )
-
 # Pipelines
 from .pipelines import (
     CsvPipelineDataFormat,
@@ -113,7 +108,6 @@ from .tokenization_openai import OpenAIGPTTokenizer
 from .tokenization_roberta import RobertaTokenizer
 from .tokenization_t5 import T5Tokenizer
 from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer
-
 # Tokenizers
 from .tokenization_utils import PreTrainedTokenizer
 from .tokenization_xlm import XLMTokenizer

src/transformers/data/processors/glue.py

@@ -81,7 +81,7 @@ def glue_convert_examples_to_features(
     features = []
     for (ex_index, example) in enumerate(examples):
         if ex_index % 10000 == 0:
-            logger.info("Writing example %d/%d" % (ex_index, len(examples)))
+            logger.info("Writing example %d" % (ex_index))
         if is_tf_dataset:
             example = processor.get_example_from_tensor_dict(example)
             example = processor.tfds_map(example)

src/transformers/data/processors/utils.py

@@ -93,33 +93,6 @@ class InputFeatures(object):
 class DataProcessor(object):
     """Base class for data converters for sequence classification data sets."""
 
-    def get_example_from_tensor_dict(self, tensor_dict):
-        """Gets an example from a dict with tensorflow tensors
-        Args:
-            tensor_dict: Keys and values should match the corresponding Glue
-                tensorflow_dataset examples.
-        """
-        raise NotImplementedError()
-
-    def get_train_examples(self, data_dir):
-        """Gets a collection of `InputExample`s for the train set."""
-        raise NotImplementedError()
-
-    def get_dev_examples(self, data_dir):
-        """Gets a collection of `InputExample`s for the dev set."""
-        raise NotImplementedError()
-
-    def get_labels(self):
-        """Gets the list of labels for this data set."""
-        raise NotImplementedError()
-
-    def tfds_map(self, example):
-        """Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are.
-        This method converts examples to the correct format."""
-        if len(self.get_labels()) > 1:
-            example.label = self.get_labels()[int(example.label)]
-        return example
-
     @classmethod
     def _read_tsv(cls, input_file, quotechar=None):
         """Reads a tab separated value file."""
@@ -280,7 +253,7 @@ class SingleSentenceClassificationProcessor(DataProcessor):
         features = []
         for (ex_index, (input_ids, example)) in enumerate(zip(all_input_ids, self.examples)):
             if ex_index % 10000 == 0:
-                logger.info("Writing example %d/%d" % (ex_index, len(self.examples)))
+                logger.info("Writing example %d", ex_index)
             # The mask has 1 for real tokens and 0 for padding tokens. Only real
             # tokens are attended to.
             attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)

src/transformers/file_utils.py

@@ -98,6 +98,7 @@ def is_torch_available():
 
 
 def is_tf_available():
+
     return _tf_available
 
 
@@ -273,10 +274,6 @@ def s3_get(url, temp_file, proxies=None):
 
 def http_get(url, temp_file, proxies=None, resume_size=0, user_agent=None):
     ua = "transformers/{}; python/{}".format(__version__, sys.version.split()[0])
-    if is_torch_available():
-        ua += "; torch/{}".format(torch.__version__)
-    if is_tf_available():
-        ua += "; tensorflow/{}".format(tf.__version__)
     if isinstance(user_agent, dict):
         ua += "; " + "; ".join("{}/{}".format(k, v) for k, v in user_agent.items())
     elif isinstance(user_agent, str):

src/transformers/modeling_bert.py

@@ -1392,7 +1392,8 @@ class BertForQuestionAnswering(BertPreTrainedModel):
         tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
         model = BertForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')
         question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet"
-        input_ids = tokenizer.encode(question, text)
+        input_text = "[CLS] " + question + " [SEP] " + text + " [SEP]"
+        input_ids = tokenizer.encode(input_text)
         token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))]
         start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids]))
         all_tokens = tokenizer.convert_ids_to_tokens(input_ids)

src/transformers/modeling_encoder_decoder.py

@@ -325,7 +325,7 @@ class Model2Model(PreTrainedEncoderDecoder):
             encoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
             decoder_pretrained_model_name_or_path=pretrained_model_name_or_path,
             *args,
-            **kwargs,
+            **kwargs
         )
 
         return model

src/transformers/modeling_gpt2.py

@@ -103,6 +103,7 @@ class Attention(nn.Module):
     def __init__(self, nx, n_ctx, config, scale=False):
         super(Attention, self).__init__()
         self.output_attentions = config.output_attentions
+        self.output_past = config.output_past
 
         n_state = nx  # in Attention: n_state=768 (nx=n_embd)
         # [switch nx => n_state from Block to Attention to keep identical to TF implem]
@@ -159,10 +160,11 @@ class Attention(nn.Module):
         if head_mask is not None:
             w = w * head_mask
 
-        outputs = [torch.matmul(w, v)]
+        x = torch.matmul(w, v)
         if self.output_attentions:
-            outputs.append(w)
-        return outputs
+            return (x, w)
+        else:
+            return (x,)
 
     def merge_heads(self, x):
         x = x.permute(0, 2, 1, 3).contiguous()
@@ -187,7 +189,8 @@ class Attention(nn.Module):
             past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1]  # transpose back cf below
             key = torch.cat((past_key, key), dim=-1)
             value = torch.cat((past_value, value), dim=-2)
-        present = torch.stack((key.transpose(-2, -1), value))  # transpose to have same shapes for stacking
+        if self.output_past:
+            present = torch.stack((key.transpose(-2, -1), value))  # transpose to have same shapes for stacking
 
         attn_outputs = self._attn(query, key, value, attention_mask, head_mask)
         a = attn_outputs[0]
@@ -196,8 +199,11 @@ class Attention(nn.Module):
         a = self.c_proj(a)
         a = self.resid_dropout(a)
 
-        outputs = [a, present] + attn_outputs[1:]
-        return outputs  # a, present, (attentions)
+        if self.output_past:
+            outputs = (a, present) + attn_outputs[1:]
+        else:
+            outputs = (a,) + attn_outputs[1:]
+        return outputs  # a, (present), (attentions)
 
 
 class MLP(nn.Module):
@@ -228,14 +234,14 @@ class Block(nn.Module):
         output_attn = self.attn(
             self.ln_1(x), layer_past=layer_past, attention_mask=attention_mask, head_mask=head_mask
         )
-        a = output_attn[0]  # output_attn: a, present, (attentions)
+        a = output_attn[0]  # output_attn: a, (present), (attentions)
 
         x = x + a
         m = self.mlp(self.ln_2(x))
         x = x + m
 
-        outputs = [x] + output_attn[1:]
-        return outputs  # x, present, (attentions)
+        outputs = (x,) + output_attn[1:]
+        return outputs  # x, (present), (attentions)
 
 
 class GPT2PreTrainedModel(PreTrainedModel):
@@ -465,7 +471,7 @@ class GPT2Model(GPT2PreTrainedModel):
         output_shape = input_shape + (hidden_states.size(-1),)
 
         presents = ()
-        all_attentions = []
+        all_attentions = ()
         all_hidden_states = ()
         for i, (block, layer_past) in enumerate(zip(self.h, past)):
             if self.output_hidden_states:
@@ -475,12 +481,12 @@ class GPT2Model(GPT2PreTrainedModel):
                 hidden_states, layer_past=layer_past, attention_mask=attention_mask, head_mask=head_mask[i]
             )
 
-            hidden_states, present = outputs[:2]
+            hidden_states = outputs[0]
             if self.output_past:
-                presents = presents + (present,)
+                presents = presents + (outputs[1],)
 
             if self.output_attentions:
-                all_attentions.append(outputs[2])
+                all_attentions = all_attentions + (outputs[2 if self.output_past else 1],)
 
         hidden_states = self.ln_f(hidden_states)
 

src/transformers/modeling_tf_utils.py

@@ -250,7 +250,7 @@ class TFPreTrainedModel(tf.keras.Model):
                 return_unused_kwargs=True,
                 force_download=force_download,
                 resume_download=resume_download,
-                **kwargs,
+                **kwargs
             )
         else:
             model_kwargs = kwargs

src/transformers/modeling_utils.py

@@ -355,7 +355,7 @@ class PreTrainedModel(nn.Module):
                 force_download=force_download,
                 resume_download=resume_download,
                 proxies=proxies,
-                **kwargs,
+                **kwargs
             )
         else:
             model_kwargs = kwargs

src/transformers/pipelines.py

@@ -335,13 +335,13 @@ class Pipeline(_ScikitCompat):
         self.tokenizer = tokenizer
         self.modelcard = modelcard
         self.framework = framework
-        self.device = device if framework == "tf" else torch.device("cpu" if device < 0 else f"cuda:{device}")
+        self.device = device
         self.binary_output = binary_output
         self._args_parser = args_parser or DefaultArgumentHandler()
 
         # Special handling
-        if self.framework == "pt" and self.device.index >= 0:
-            self.model = self.model.to(self.device)
+        if self.device >= 0 and self.framework == "pt":
+            self.model = self.model.to("cuda:{}".format(self.device))
 
     def save_pretrained(self, save_directory):
         """
@@ -385,19 +385,11 @@ class Pipeline(_ScikitCompat):
             with tf.device("/CPU:0" if self.device == -1 else "/device:GPU:{}".format(self.device)):
                 yield
         else:
-            if self.device.index >= 0:
+            if self.device >= 0:
                 torch.cuda.set_device(self.device)
 
             yield
 
-    def ensure_tensor_on_device(self, **inputs):
-        """
-        Ensure PyTorch tensors are on the specified device.
-        :param inputs:
-        :return:
-        """
-        return {name: tensor.to(self.device) for name, tensor in inputs.items()}
-
     def inputs_for_model(self, features: Union[dict, List[dict]]) -> Dict:
         """
         Generates the input dictionary with model-specific parameters.
@@ -423,13 +415,16 @@ class Pipeline(_ScikitCompat):
     def __call__(self, *texts, **kwargs):
         # Parse arguments
         inputs = self._args_parser(*texts, **kwargs)
-        inputs = self.tokenizer.batch_encode_plus(
-            inputs, add_special_tokens=True, return_tensors=self.framework, max_length=self.tokenizer.max_len
-        )
 
-        # Filter out features not available on specific models
-        inputs = self.inputs_for_model(inputs)
-        return self._forward(inputs)
+        # Encode for forward
+        with self.device_placement():
+            inputs = self.tokenizer.batch_encode_plus(
+                inputs, add_special_tokens=True, return_tensors=self.framework, max_length=self.tokenizer.max_len
+            )
+
+            # Filter out features not available on specific models
+            inputs = self.inputs_for_model(inputs)
+            return self._forward(inputs)
 
     def _forward(self, inputs):
         """
@@ -439,15 +434,12 @@ class Pipeline(_ScikitCompat):
         Returns:
             Numpy array
         """
-        # Encode for forward
-        with self.device_placement():
-            if self.framework == "tf":
-                # TODO trace model
-                predictions = self.model(inputs, training=False)[0]
-            else:
-                with torch.no_grad():
-                    inputs = self.ensure_tensor_on_device(**inputs)
-                    predictions = self.model(**inputs)[0].cpu()
+        if self.framework == "tf":
+            # TODO trace model
+            predictions = self.model(inputs, training=False)[0]
+        else:
+            with torch.no_grad():
+                predictions = self.model(**inputs)[0].cpu()
 
         return predictions.numpy()
 
@@ -542,7 +534,6 @@ class NerPipeline(Pipeline):
                     input_ids = tokens["input_ids"].numpy()[0]
                 else:
                     with torch.no_grad():
-                        tokens = self.ensure_tensor_on_device(**tokens)
                         entities = self.model(**tokens)[0][0].cpu().numpy()
                         input_ids = tokens["input_ids"].cpu().numpy()[0]
 
@@ -652,7 +643,7 @@ class QuestionAnsweringPipeline(Pipeline):
             framework=framework,
             args_parser=QuestionAnsweringArgumentHandler(),
             device=device,
-            **kwargs,
+            **kwargs
         )
 
     @staticmethod
@@ -719,7 +710,7 @@ class QuestionAnsweringPipeline(Pipeline):
             else:
                 with torch.no_grad():
                     # Retrieve the score for the context tokens only (removing question tokens)
-                    fw_args = {k: torch.tensor(v, device=self.device) for (k, v) in fw_args.items()}
+                    fw_args = {k: torch.tensor(v) for (k, v) in fw_args.items()}
                     start, end = self.model(**fw_args)
                     start, end = start.cpu().numpy(), end.cpu().numpy()
 

src/transformers/tokenization_albert.py

@@ -87,7 +87,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs,
+            **kwargs
         )
 
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens

src/transformers/tokenization_bert.py

@@ -169,7 +169,7 @@ class BertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs,
+            **kwargs
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens
@@ -560,7 +560,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs,
+            **kwargs
         )
 
         self._tokenizer = tk.Tokenizer(tk.models.WordPiece.from_files(vocab_file, unk_token=unk_token))

src/transformers/tokenization_bert_japanese.py

@@ -113,7 +113,7 @@ class BertJapaneseTokenizer(BertTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs,
+            **kwargs
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens

src/transformers/tokenization_camembert.py

@@ -76,7 +76,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs,
+            **kwargs
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens

src/transformers/tokenization_roberta.py

@@ -95,7 +95,7 @@ class RobertaTokenizer(GPT2Tokenizer):
             cls_token=cls_token,
             pad_token=pad_token,
             mask_token=mask_token,
-            **kwargs,
+            **kwargs
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens

src/transformers/tokenization_t5.py

@@ -96,7 +96,7 @@ class T5Tokenizer(PreTrainedTokenizer):
             unk_token=unk_token,
             pad_token=pad_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs,
+            **kwargs
         )
 
         try:

src/transformers/tokenization_utils.py

@@ -817,7 +817,7 @@ class PreTrainedTokenizer(object):
             truncation_strategy=truncation_strategy,
             pad_to_max_length=pad_to_max_length,
             return_tensors=return_tensors,
-            **kwargs,
+            **kwargs
         )
 
         return encoded_inputs["input_ids"]

src/transformers/tokenization_xlm.py

@@ -586,7 +586,7 @@ class XLMTokenizer(PreTrainedTokenizer):
             cls_token=cls_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs,
+            **kwargs
         )
 
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens

src/transformers/tokenization_xlm_roberta.py

@@ -83,7 +83,7 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
             cls_token=cls_token,
             pad_token=pad_token,
             mask_token=mask_token,
-            **kwargs,
+            **kwargs
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 4  # take into account special tokens

src/transformers/tokenization_xlnet.py

@@ -86,7 +86,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
             cls_token=cls_token,
             mask_token=mask_token,
             additional_special_tokens=additional_special_tokens,
-            **kwargs,
+            **kwargs
         )
 
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens

templates/adding_a_new_example_script/run_xxx.py

@@ -52,7 +52,6 @@ from utils_squad import (
     write_predictions,
     write_predictions_extended,
 )
-
 # The follwing import is the official SQuAD evaluation script (2.0).
 # You can remove it from the dependencies if you are using this script outside of the library
 # We've added it here for automated tests (see examples/test_examples.py file)

templates/adding_a_new_example_script/utils_xxx.py

@@ -21,7 +21,6 @@ import logging
 import math
 
 from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
-
 # Required by XLNet evaluation method to compute optimal threshold (see write_predictions_extended() method)
 from utils_squad_evaluate import find_all_best_thresh_v2, get_raw_scores, make_qid_to_has_ans
 

templates/adding_a_new_model/tokenization_xxx.py

@@ -115,7 +115,7 @@ class XxxTokenizer(PreTrainedTokenizer):
             pad_token=pad_token,
             cls_token=cls_token,
             mask_token=mask_token,
-            **kwargs,
+            **kwargs
         )
         self.max_len_single_sentence = self.max_len - 2  # take into account special tokens
         self.max_len_sentences_pair = self.max_len - 3  # take into account special tokens

tests/test_modeling_tf_common.py

@@ -115,13 +115,6 @@ class TFModelTesterMixin:
             tf_hidden_states[np.isnan(tf_hidden_states)] = 0
             pt_hidden_states[np.isnan(pt_hidden_states)] = 0
             max_diff = np.amax(np.abs(tf_hidden_states - pt_hidden_states))
-            # Debug info (remove when fixed)
-            if max_diff >= 2e-2:
-                print("===")
-                print(model_class)
-                print(config)
-                print(inputs_dict)
-                print(pt_inputs_dict)
             self.assertLessEqual(max_diff, 2e-2)
 
             # Check we can load pt model in tf and vice-versa with checkpoint => model functions

tests/test_tokenization_bert.py

@@ -84,7 +84,7 @@ class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         tokenizer = self.get_tokenizer()
         rust_tokenizer = self.get_rust_tokenizer(add_special_tokens=False)
 
-        sequence = "UNwant\u00E9d,running"
+        sequence = u"UNwant\u00E9d,running"
 
         tokens = tokenizer.tokenize(sequence)
         rust_tokens = rust_tokenizer.tokenize(sequence)

tests/test_tokenization_gpt2.py

@@ -96,7 +96,7 @@ class GPT2TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         tokenizer = self.get_tokenizer()
         rust_tokenizer = self.get_rust_tokenizer(add_special_tokens=False, add_prefix_space=True)
 
-        sequence = "lower newer"
+        sequence = u"lower newer"
 
         # Testing tokenization
         tokens = tokenizer.tokenize(sequence, add_prefix_space=True)