make style and quality

Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>

setup.py

@@ -86,7 +86,7 @@ setup(
     packages=find_packages("src"),
     install_requires=[
         "numpy",
-        "tokenizers == 0.0.11",
+        "tokenizers == 0.2.1",
         # accessing files from S3 directly
         "boto3",
         # filesystem locks e.g. to prevent parallel downloads

src/transformers/modeling_ctrl.py

@@ -30,7 +30,9 @@ from .modeling_utils import Conv1D, PreTrainedModel
 
 logger = logging.getLogger(__name__)
 
-CTRL_PRETRAINED_MODEL_ARCHIVE_MAP = {"ctrl": "https://storage.googleapis.com/sf-ctrl/pytorch/seqlen256_v1.bin"}
+CTRL_PRETRAINED_MODEL_ARCHIVE_MAP = {
+    "ctrl": "https://s3.amazonaws.com/models.huggingface.co/bert/ctrl-pytorch_model.bin"
+}
 
 
 def angle_defn(pos, i, d_model_size):

src/transformers/tokenization_auto.py

@@ -36,14 +36,14 @@ from .configuration_auto import (
 )
 from .configuration_utils import PretrainedConfig
 from .tokenization_albert import AlbertTokenizer
-from .tokenization_bert import BertTokenizer
+from .tokenization_bert import BertTokenizer, BertTokenizerFast
 from .tokenization_bert_japanese import BertJapaneseTokenizer
 from .tokenization_camembert import CamembertTokenizer
-from .tokenization_ctrl import CTRLTokenizer
+from .tokenization_ctrl import CTRLTokenizer, CTRLTokenizerFast
-from .tokenization_distilbert import DistilBertTokenizer
+from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFast
-from .tokenization_gpt2 import GPT2Tokenizer
+from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
-from .tokenization_openai import OpenAIGPTTokenizer
+from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
-from .tokenization_roberta import RobertaTokenizer
+from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
 from .tokenization_t5 import T5Tokenizer
 from .tokenization_transfo_xl import TransfoXLTokenizer
 from .tokenization_xlm import XLMTokenizer
@@ -56,19 +56,19 @@ logger = logging.getLogger(__name__)
 
 TOKENIZER_MAPPING = OrderedDict(
     [
-        (T5Config, T5Tokenizer),
+        (T5Config, (T5Tokenizer, None)),
-        (DistilBertConfig, DistilBertTokenizer),
+        (DistilBertConfig, (DistilBertTokenizer, DistilBertTokenizerFast)),
-        (AlbertConfig, AlbertTokenizer),
+        (AlbertConfig, (AlbertTokenizer, None)),
-        (CamembertConfig, CamembertTokenizer),
+        (CamembertConfig, (CamembertTokenizer, None)),
-        (XLMRobertaConfig, XLMRobertaTokenizer),
+        (XLMRobertaConfig, (XLMRobertaTokenizer, None)),
-        (RobertaConfig, RobertaTokenizer),
+        (RobertaConfig, (RobertaTokenizer, RobertaTokenizerFast)),
-        (BertConfig, BertTokenizer),
+        (BertConfig, (BertTokenizer, BertTokenizerFast)),
-        (OpenAIGPTConfig, OpenAIGPTTokenizer),
+        (OpenAIGPTConfig, (OpenAIGPTTokenizer, OpenAIGPTTokenizerFast)),
-        (GPT2Config, GPT2Tokenizer),
+        (GPT2Config, (GPT2Tokenizer, GPT2TokenizerFast)),
-        (TransfoXLConfig, TransfoXLTokenizer),
+        (TransfoXLConfig, (TransfoXLTokenizer, None)),
-        (XLNetConfig, XLNetTokenizer),
+        (XLNetConfig, (XLNetTokenizer, None)),
-        (XLMConfig, XLMTokenizer),
+        (XLMConfig, (XLMTokenizer, None)),
-        (CTRLConfig, CTRLTokenizer),
+        (CTRLConfig, (CTRLTokenizer, CTRLTokenizerFast)),
     ]
 )
 
@@ -174,9 +174,12 @@ class AutoTokenizer(object):
         if "bert-base-japanese" in pretrained_model_name_or_path:
             return BertJapaneseTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
 
-        for config_class, tokenizer_class in TOKENIZER_MAPPING.items():
+        for config_class, (tokenizer_class_py, tokenizer_class_ru) in TOKENIZER_MAPPING.items():
             if isinstance(config, config_class):
-                return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+                if tokenizer_class_ru:
+                    return tokenizer_class_ru.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+                else:
+                    return tokenizer_class_py.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
 
         raise ValueError(
             "Unrecognized configuration class {} to build an AutoTokenizer.\n"

src/transformers/tokenization_bert.py

@@ -555,6 +555,15 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
         **kwargs
     ):
         super().__init__(
+            tk.implementations.BertWordPieceTokenizer(
+                vocab_file,
+                add_special_tokens,
+                unk_token,
+                sep_token,
+                cls_token,
+                handle_chinese_chars=tokenize_chinese_chars,
+                lowercase=do_lower_case,
+            ),
             unk_token=unk_token,
             sep_token=sep_token,
             pad_token=pad_token,
@@ -562,33 +571,3 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
             mask_token=mask_token,
             **kwargs,
         )
-
-        self._tokenizer = tk.Tokenizer(tk.models.WordPiece.from_files(vocab_file, unk_token=unk_token))
-        self._update_special_tokens()
-        self._tokenizer.with_pre_tokenizer(
-            tk.pre_tokenizers.BertPreTokenizer.new(
-                do_basic_tokenize=do_basic_tokenize,
-                do_lower_case=do_lower_case,
-                tokenize_chinese_chars=tokenize_chinese_chars,
-                never_split=never_split if never_split is not None else [],
-            )
-        )
-        self._tokenizer.with_decoder(tk.decoders.WordPiece.new())
-
-        if add_special_tokens:
-            self._tokenizer.with_post_processor(
-                tk.processors.BertProcessing.new(
-                    (sep_token, self._tokenizer.token_to_id(sep_token)),
-                    (cls_token, self._tokenizer.token_to_id(cls_token)),
-                )
-            )
-        if max_length is not None:
-            self._tokenizer.with_truncation(max_length, stride=stride, strategy=truncation_strategy)
-        self._tokenizer.with_padding(
-            max_length=max_length if pad_to_max_length else None,
-            direction=self.padding_side,
-            pad_id=self.pad_token_id,
-            pad_type_id=self.pad_token_type_id,
-            pad_token=self.pad_token,
-        )
-        self._decoder = tk.decoders.WordPiece.new()

src/transformers/tokenization_ctrl.py

@@ -20,8 +20,9 @@ import logging
 import os
 
 import regex as re
+from tokenizers import BPETokenizer
 
-from .tokenization_utils import PreTrainedTokenizer
+from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
 
 
 logger = logging.getLogger(__name__)
@@ -32,8 +33,8 @@ VOCAB_FILES_NAMES = {
 }
 
 PRETRAINED_VOCAB_FILES_MAP = {
-    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
+    "vocab_file": {"ctrl": "https://s3.amazonaws.com/models.huggingface.co/bert/ctrl-vocab.json"},
-    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
+    "merges_file": {"ctrl": "https://s3.amazonaws.com/models.huggingface.co/bert/ctrl-merges.txt"},
 }
 
 PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
@@ -148,14 +149,14 @@ class CTRLTokenizer(PreTrainedTokenizer):
         return len(self.encoder)
 
     def bpe(self, token):
-        if token in self.cache:
-            return self.cache[token]
         word = tuple(token)
         word = tuple(list(word[:-1]) + [word[-1] + "</w>"])
+        if token in self.cache:
+            return self.cache[token]
         pairs = get_pairs(word)
 
         if not pairs:
-            return token
+            return token + "</w>"
 
         while True:
             bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
@@ -186,8 +187,9 @@ class CTRLTokenizer(PreTrainedTokenizer):
                 break
             else:
                 pairs = get_pairs(word)
-        word = "@@ ".join(word)
+        word = " ".join(word)
-        word = word[:-4]
+        if word == "\n  </w>":
+            word = "\n</w>"
         self.cache[token] = word
         return word
 
@@ -212,7 +214,7 @@ class CTRLTokenizer(PreTrainedTokenizer):
 
     def convert_tokens_to_string(self, tokens):
         """ Converts a sequence of tokens (string) in a single string. """
-        out_string = " ".join(tokens).replace("@@ ", "").strip()
+        out_string = "".join(tokens).replace("</w>", " ").strip()
         return out_string
 
     def save_vocabulary(self, save_directory):
@@ -246,3 +248,13 @@ class CTRLTokenizer(PreTrainedTokenizer):
     #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
     #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
     #     return ''.join(tokens_generated_so_far)
+
+
+class CTRLTokenizerFast(PreTrainedTokenizerFast):
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    control_codes = CONTROL_CODES
+
+    def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
+        super().__init__(BPETokenizer(vocab_file, merges_file, unk_token), **kwargs)

src/transformers/tokenization_distilbert.py

@@ -17,7 +17,7 @@
 
 import logging
 
-from .tokenization_bert import BertTokenizer
+from .tokenization_bert import BertTokenizer, BertTokenizerFast
 
 
 logger = logging.getLogger(__name__)
@@ -68,3 +68,10 @@ class DistilBertTokenizer(BertTokenizer):
     pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
     max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
+
+
+class DistilBertTokenizerFast(BertTokenizerFast):
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION

src/transformers/tokenization_gpt2.py

@@ -22,6 +22,7 @@ from functools import lru_cache
 
 import regex as re
 import tokenizers as tk
+from tokenizers import ByteLevelBPETokenizer
 
 from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
 
@@ -268,19 +269,25 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
         truncation_strategy="longest_first",
         **kwargs
     ):
-        super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
+        super().__init__(
-
+            ByteLevelBPETokenizer(vocab_file, merges_file, add_prefix_space),
-        self._tokenizer = tk.Tokenizer(tk.models.BPE.from_files(vocab_file, merges_file))
+            bos_token=bos_token,
-        self._update_special_tokens()
+            eos_token=eos_token,
-        self._tokenizer.with_pre_tokenizer(tk.pre_tokenizers.ByteLevel.new(add_prefix_space=add_prefix_space))
+            unk_token=unk_token,
-        self._tokenizer.with_decoder(tk.decoders.ByteLevel.new())
+            **kwargs,
-        if max_length:
-            self._tokenizer.with_truncation(max_length, stride=stride, strategy=truncation_strategy)
-        self._tokenizer.with_padding(
-            max_length=max_length if pad_to_max_length else None,
-            direction=self.padding_side,
-            pad_id=self.pad_token_id if self.pad_token_id is not None else 0,
-            pad_type_id=self.pad_token_type_id,
-            pad_token=self.pad_token if self.pad_token is not None else "",
         )
-        self._decoder = tk.decoders.ByteLevel.new()
+
+        # self._tokenizer = tk.Tokenizer(tk.models.BPE.from_files(vocab_file, merges_file))
+        # self._update_special_tokens()
+        # self._tokenizer.with_pre_tokenizer(tk.pre_tokenizers.ByteLevel.new(add_prefix_space=add_prefix_space))
+        # self._tokenizer.with_decoder(tk.decoders.ByteLevel.new())
+        # if max_length:
+        #     self._tokenizer.with_truncation(max_length, stride=stride, strategy=truncation_strategy)
+        # self._tokenizer.with_padding(
+        #     max_length=max_length if pad_to_max_length else None,
+        #     direction=self.padding_side,
+        #     pad_id=self.pad_token_id if self.pad_token_id is not None else 0,
+        #     pad_type_id=self.pad_token_type_id,
+        #     pad_token=self.pad_token if self.pad_token is not None else "",
+        # )
+        # self._decoder = tk.decoders.ByteLevel.new()

src/transformers/tokenization_openai.py

@@ -20,8 +20,10 @@ import logging
 import os
 import re
 
+from tokenizers import BPETokenizer
+
 from .tokenization_bert import BasicTokenizer
-from .tokenization_utils import PreTrainedTokenizer
+from .tokenization_utils import PreTrainedTokenizer, PreTrainedTokenizerFast
 
 
 logger = logging.getLogger(__name__)
@@ -213,3 +215,12 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer):
                 index += 1
 
         return vocab_file, merge_file
+
+
+class OpenAIGPTTokenizerFast(PreTrainedTokenizerFast):
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+
+    def __init__(self, vocab_file, merges_file, unk_token="<unk>", **kwargs):
+        super().__init__(BPETokenizer(vocab_file, merges_file, unk_token), **kwargs)

src/transformers/tokenization_roberta.py

@@ -17,7 +17,7 @@
 
 import logging
 
-from .tokenization_gpt2 import GPT2Tokenizer
+from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
 
 
 logger = logging.getLogger(__name__)
@@ -154,3 +154,30 @@ class RobertaTokenizer(GPT2Tokenizer):
         if token_ids_1 is None:
             return len(cls + token_ids_0 + sep) * [0]
         return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]
+
+
+class RobertaTokenizerFast(GPT2TokenizerFast):
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+
+    def __init__(
+        self,
+        vocab_file,
+        merges_file,
+        errors="replace",
+        bos_token="<s>",
+        eos_token="</s>",
+        sep_token="</s>",
+        cls_token="<s>",
+        unk_token="<unk>",
+        pad_token="<pad>",
+        mask_token="<mask>",
+        **kwargs
+    ):
+        kwargs["pad_token"] = pad_token
+        kwargs["sep_token"] = sep_token
+        kwargs["cls_token"] = cls_token
+        kwargs["mask_token"] = mask_token
+
+        super().__init__(vocab_file, merges_file, unk_token, bos_token, eos_token, add_prefix_space=True)

src/transformers/tokenization_utils.py

@@ -21,6 +21,9 @@ import json
 import logging
 import os
 import re
+from contextlib import contextmanager
+
+from tokenizers.implementations import BaseTokenizer
 
 from .file_utils import cached_path, hf_bucket_url, is_remote_url, is_tf_available, is_torch_available
 
@@ -37,6 +40,56 @@ ADDED_TOKENS_FILE = "added_tokens.json"
 TOKENIZER_CONFIG_FILE = "tokenizer_config.json"
 
 
+@contextmanager
+def truncate_and_pad(
+    tokenizer: BaseTokenizer,
+    max_length: int,
+    stride: int,
+    strategy: str,
+    pad_to_max_length: bool,
+    padding_side: str,
+    pad_token_id: int,
+    pad_token_type_id: int,
+    pad_token: str,
+):
+    """
+    This contextmanager is in charge of defining the truncation and the padding strategies and then
+    restore the tokenizer settings afterwards.
+
+    :param tokenizer:
+    :param max_length:
+    :param stride:
+    :param strategy:
+    :param pad_to_max_length:
+    :param padding_side:
+    :param pad_token_id:
+    :param pad_token_type_id:
+    :param pad_token:
+    :return:
+    """
+
+    # Handle all the truncation and padding stuff
+    if max_length is not None:
+        tokenizer.enable_truncation(max_length, stride=stride, strategy=strategy)
+
+        if pad_to_max_length:
+            tokenizer.enable_padding(
+                max_length=max_length,
+                direction=padding_side,
+                pad_id=pad_token_id,
+                pad_type_id=pad_token_type_id,
+                pad_token=pad_token,
+            )
+
+    yield
+
+    if max_length is not None:
+        tokenizer.no_truncation()
+
+        if pad_to_max_length:
+            tokenizer.no_padding()
+
+
 class PreTrainedTokenizer(object):
     """ Base class for all tokenizers.
     Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading pretrained tokenizers as well as adding tokens to the vocabulary.
@@ -832,6 +885,7 @@ class PreTrainedTokenizer(object):
         return_attention_mask=True,
         return_overflowing_tokens=False,
         return_special_tokens_mask=False,
+        return_offsets_mapping=False,
         **kwargs
     ):
         """
@@ -905,6 +959,9 @@ class PreTrainedTokenizer(object):
                     "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
                 )
 
+        if return_offsets_mapping:
+            logger.warning("offset mapping is not available on Python tokenizers.")
+
         first_ids = get_input_ids(text)
         second_ids = get_input_ids(text_pair) if text_pair is not None else None
 
@@ -1417,30 +1474,27 @@ class PreTrainedTokenizer(object):
 
 
 class PreTrainedTokenizerFast(PreTrainedTokenizer):
-    _tokenizer = None
+    def __init__(self, tokenizer: BaseTokenizer, **kwargs):
-    _decoder = None
+        if tokenizer is None:
+            raise ValueError("Provided tokenizer cannot be None")
+        self._tokenizer = tokenizer
 
-    def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
     @property
     def tokenizer(self):
-        if self._tokenizer is None:
-            raise NotImplementedError
         return self._tokenizer
 
     @property
     def decoder(self):
-        if self._decoder is None:
+        return self._tokenizer._tokenizer.decoder
-            raise NotImplementedError
-        return self._decoder
 
     @property
     def vocab_size(self):
-        return self.tokenizer.get_vocab_size(with_added_tokens=False)
+        return self._tokenizer.get_vocab_size(with_added_tokens=False)
 
     def __len__(self):
-        return self.tokenizer.get_vocab_size(with_added_tokens=True)
+        return self._tokenizer.get_vocab_size(with_added_tokens=True)
 
     @PreTrainedTokenizer.bos_token.setter
     def bos_token(self, value):
@@ -1494,36 +1548,59 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer):
         return_attention_mask=True,
         return_overflowing_tokens=False,
         return_special_tokens_mask=False,
+        return_offsets_mapping=False,
+        pad_token_id: int = 0,
+        pad_to_length: int = -1,
     ):
+        if return_overflowing_tokens and encoding.overflowing is not None:
+            encodings = [encoding] + encoding.overflowing
+        else:
+            encodings = [encoding]
+
         encoding_dict = {
-            "input_ids": encoding.ids,
+            "input_ids": [e.ids for e in encodings],
         }
+
         if return_token_type_ids:
-            encoding_dict["token_type_ids"] = encoding.type_ids
+            encoding_dict["token_type_ids"] = [e.type_ids for e in encodings]
         if return_attention_mask:
-            encoding_dict["attention_mask"] = encoding.attention_mask
+            encoding_dict["attention_mask"] = [e.attention_mask for e in encodings]
-        if return_overflowing_tokens:
-            overflowing = encoding.overflowing
-            encoding_dict["overflowing_tokens"] = overflowing.ids if overflowing is not None else []
         if return_special_tokens_mask:
-            encoding_dict["special_tokens_mask"] = encoding.special_tokens_mask
+            encoding_dict["special_tokens_mask"] = [e.special_tokens_mask for e in encodings]
+        if return_offsets_mapping:
+            encoding_dict["offset_mapping"] = [e.offsets for e in encodings]
+
+        if pad_to_length > 0:
+            for i in range(len(encoding_dict["input_ids"])):
+                if len(encoding_dict["input_ids"][i]) < pad_to_length:
+                    padding = pad_to_length - len(encoding_dict["input_ids"][i])
+                    encoding_dict["input_ids"][i] += [pad_token_id] * padding
+
+                    if return_attention_mask:
+                        encoding_dict["attention_mask"][i] += [0] * padding
+
+                    if return_special_tokens_mask:
+                        encoding_dict["special_tokens_mask"][i] += [1] * padding
+
+                    if return_token_type_ids:
+                        encoding_dict["token_type_ids"][i] += [1] * padding
 
         # Prepare inputs as tensors if asked
         if return_tensors == "tf" and is_tf_available():
-            encoding_dict["input_ids"] = tf.constant([encoding_dict["input_ids"]])
+            encoding_dict["input_ids"] = tf.constant(encoding_dict["input_ids"])
             if "token_type_ids" in encoding_dict:
-                encoding_dict["token_type_ids"] = tf.constant([encoding_dict["token_type_ids"]])
+                encoding_dict["token_type_ids"] = tf.constant(encoding_dict["token_type_ids"])
 
             if "attention_mask" in encoding_dict:
-                encoding_dict["attention_mask"] = tf.constant([encoding_dict["attention_mask"]])
+                encoding_dict["attention_mask"] = tf.constant(encoding_dict["attention_mask"])
 
         elif return_tensors == "pt" and is_torch_available():
-            encoding_dict["input_ids"] = torch.tensor([encoding_dict["input_ids"]])
+            encoding_dict["input_ids"] = torch.tensor(encoding_dict["input_ids"])
             if "token_type_ids" in encoding_dict:
-                encoding_dict["token_type_ids"] = torch.tensor([encoding_dict["token_type_ids"]])
+                encoding_dict["token_type_ids"] = torch.tensor(encoding_dict["token_type_ids"])
 
             if "attention_mask" in encoding_dict:
-                encoding_dict["attention_mask"] = torch.tensor([encoding_dict["attention_mask"]])
+                encoding_dict["attention_mask"] = torch.tensor(encoding_dict["attention_mask"])
         elif return_tensors is not None:
             logger.warning(
                 "Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(
@@ -1531,73 +1608,110 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer):
                 )
             )
 
-        return encoding_dict
+        return {k: v if len(v) > 1 else v[0] for k, v in encoding_dict.items()}
-
-    def encode_plus(
-        self,
-        text,
-        text_pair=None,
-        return_tensors=None,
-        return_token_type_ids=True,
-        return_attention_mask=True,
-        return_overflowing_tokens=False,
-        return_special_tokens_mask=False,
-        **kwargs
-    ):
-        encoding = self.tokenizer.encode(text, text_pair)
-        return self._convert_encoding(
-            encoding,
-            return_tensors=return_tensors,
-            return_token_type_ids=return_token_type_ids,
-            return_attention_mask=return_attention_mask,
-            return_overflowing_tokens=return_overflowing_tokens,
-            return_special_tokens_mask=return_special_tokens_mask,
-        )
-
-    def tokenize(self, text):
-        return self.tokenizer.encode(text).tokens
 
     def _convert_token_to_id_with_added_voc(self, token):
-        id = self.tokenizer.token_to_id(token)
+        id = self._tokenizer.token_to_id(token)
         if id is None:
             return self.unk_token_id
         return id
 
     def _convert_id_to_token(self, index):
-        return self.tokenizer.id_to_token(int(index))
+        return self._tokenizer.id_to_token(int(index))
 
     def convert_tokens_to_string(self, tokens):
-        return self.decoder.decode(tokens)
+        return self._tokenizer.decode(tokens)
 
     def add_tokens(self, new_tokens):
-        self.tokenizer.add_tokens(new_tokens)
+        self._tokenizer.add_tokens(new_tokens)
 
     def add_special_tokens(self, special_tokens_dict):
         added = super().add_special_tokens(special_tokens_dict)
         self._update_special_tokens()
         return added
 
-    def encode_batch(
+    def encode_plus(
         self,
-        texts,
+        text,
+        text_pair=None,
+        add_special_tokens=True,
+        max_length=None,
+        stride=0,
+        truncation_strategy="longest_first",
+        pad_to_max_length=False,
         return_tensors=None,
         return_token_type_ids=True,
         return_attention_mask=True,
         return_overflowing_tokens=False,
         return_special_tokens_mask=False,
+        return_offsets_mapping=False,
+        **kwargs
     ):
-        return [
+        # Ensure we have text defined as [str]
+        if text is not None and not isinstance(text, list):
+            text = [text]
+
+        if text_pair is not None and not isinstance(text_pair, list):
+            text_pair = [text_pair]
+
+            # Ensure we have all the pairs
+            if len(text_pair) != len(text):
+                raise ValueError(
+                    "Number of text_pair ({}) doesn't match number of text ({})".format(len(text_pair), len(text))
+                )
+
+        # Set the truncation and padding strategy and restore the initial configuration
+        with truncate_and_pad(
+            self._tokenizer,
+            max_length,
+            stride,
+            truncation_strategy,
+            pad_to_max_length,
+            self.padding_side,
+            self.pad_token_id,
+            self.pad_token_type_id,
+            self._pad_token,
+        ):
+
+            if text_pair is None:
+                tokens = self._tokenizer.encode_batch(text)
+            else:
+                tokens = self._tokenizer.encode_batch(list(zip(text, text_pair)))
+
+        # Convert encoding to dict
+        max_length = max(map(lambda e: len(e.ids), tokens))
+        tokens = [
             self._convert_encoding(
                 encoding,
-                return_tensors=return_tensors,
+                return_tensors,
-                return_token_type_ids=return_token_type_ids,
+                return_token_type_ids,
-                return_attention_mask=return_attention_mask,
+                return_attention_mask,
-                return_overflowing_tokens=return_overflowing_tokens,
+                return_overflowing_tokens,
-                return_special_tokens_mask=return_special_tokens_mask,
+                return_special_tokens_mask,
+                return_offsets_mapping,
+                self.pad_token_id,
+                max_length,
             )
-            for encoding in self.tokenizer.encode_batch(texts)
+            for encoding in tokens
         ]
 
+        # Unwrap from the list if only on sample
+        if len(tokens) == 1:
+            return tokens[0]
+
+        # Sanitize the output to have dict[list] from list[dict]
+        sanitized = {}
+        for key in tokens[0].keys():
+            stack = [item[key] for item in tokens]
+
+            if return_tensors == "tf":
+                stack = tf.concat(stack, axis=0)
+            elif return_tensors == "pt":
+                stack = torch.cat(stack, dim=0)
+
+            sanitized[key] = stack
+        return sanitized
+
     def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
         text = self.tokenizer.decode(token_ids, skip_special_tokens)
 
@@ -1607,8 +1721,5 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer):
         else:
             return text
 
-    def decode_batch(self, ids_batch, skip_special_tokens=False, clear_up_tokenization_spaces=True):
+    def save_vocabulary(self, save_directory):
-        return [
+        self._tokenizer.save(save_directory)
-            self.clean_up_tokenization(text) if clear_up_tokenization_spaces else text
-            for text in self.tokenizer.decode_batch(ids_batch, skip_special_tokens)
-        ]

tests/test_tokenization_fast.py

@@ -0,0 +1,100 @@
+import unittest
+
+from transformers import (
+    BertTokenizer,
+    BertTokenizerFast,
+    CTRLTokenizer,
+    DistilBertTokenizer,
+    GPT2Tokenizer,
+    GPT2TokenizerFast,
+    OpenAIGPTTokenizer,
+    RobertaTokenizer,
+)
+from transformers.tokenization_ctrl import CTRLTokenizerFast
+from transformers.tokenization_distilbert import DistilBertTokenizerFast
+from transformers.tokenization_openai import OpenAIGPTTokenizerFast
+from transformers.tokenization_roberta import RobertaTokenizerFast
+
+
+class FastTokenizerMatchingTest(unittest.TestCase):
+    def setUp(self) -> None:
+        with open("fixtures/sample_text.txt") as f_data:
+            self._data = f_data.read()
+
+    def _tokenize_inputs_and_check_matching(self, tokenizer_p, tokenizer_r):
+        # Ensure basic input match
+        input_p = tokenizer_p.encode_plus(self._data)
+        input_r = tokenizer_r.encode_plus(self._data)
+
+        self.assertSequenceEqual(input_p["input_ids"], input_r["input_ids"])
+        self.assertSequenceEqual(input_p["token_type_ids"], input_r["token_type_ids"])
+        self.assertSequenceEqual(input_p["attention_mask"], input_r["attention_mask"])
+
+        input_pairs_p = tokenizer_p.encode_plus(self._data, self._data)
+        input_pairs_r = tokenizer_r.encode_plus(self._data, self._data)
+
+        self.assertSequenceEqual(input_pairs_p["input_ids"], input_pairs_r["input_ids"])
+        self.assertSequenceEqual(input_pairs_p["token_type_ids"], input_pairs_r["token_type_ids"])
+        self.assertSequenceEqual(input_pairs_p["attention_mask"], input_pairs_r["attention_mask"])
+
+        # Ensure truncation match
+        input_p = tokenizer_p.encode_plus(self._data, max_length=512, pad_to_max_length=True)
+        input_r = tokenizer_r.encode_plus(self._data, max_length=512, pad_to_max_length=True)
+
+        self.assertSequenceEqual(input_p["input_ids"], input_r["input_ids"])
+        self.assertSequenceEqual(input_p["token_type_ids"], input_r["token_type_ids"])
+        self.assertSequenceEqual(input_p["attention_mask"], input_r["attention_mask"])
+
+        # Ensure truncation with stride match
+        # input_p = tokenizer_p.encode_plus(self._data, max_length=512, stride=3, return_overflowing_tokens=True)
+        # input_r = tokenizer_r.encode_plus(self._data, max_length=512, stride=3, return_overflowing_tokens=True)
+        #
+        # self.assertSequenceEqual(input_p['input_ids'], input_r['input_ids'])
+        # self.assertSequenceEqual(input_p['token_type_ids'], input_r['token_type_ids'])
+        # self.assertSequenceEqual(input_p['attention_mask'], input_r['attention_mask'])
+
+    def test_bert(self):
+        for tokenizer_name in BertTokenizer.pretrained_vocab_files_map["vocab_file"].keys():
+            tokenizer_p = BertTokenizer.from_pretrained(tokenizer_name)
+            tokenizer_r = BertTokenizerFast.from_pretrained(tokenizer_name)
+
+            self._tokenize_inputs_and_check_matching(tokenizer_p, tokenizer_r)
+
+    def test_ctrl(self):
+        for tokenizer_name in CTRLTokenizer.pretrained_vocab_files_map["vocab_file"].keys():
+            tokenizer_p = CTRLTokenizer.from_pretrained(tokenizer_name)
+            tokenizer_r = CTRLTokenizerFast.from_pretrained(tokenizer_name)
+
+            self._tokenize_inputs_and_check_matching(tokenizer_p, tokenizer_r)
+
+    def test_distilbert(self):
+        for tokenizer_name in DistilBertTokenizer.pretrained_vocab_files_map["vocab_file"].keys():
+            tokenizer_p = DistilBertTokenizer.from_pretrained(tokenizer_name)
+            tokenizer_r = DistilBertTokenizerFast.from_pretrained(tokenizer_name)
+
+            self._tokenize_inputs_and_check_matching(tokenizer_p, tokenizer_r)
+
+    def test_gpt2(self):
+        for tokenizer_name in GPT2Tokenizer.pretrained_vocab_files_map["vocab_file"].keys():
+            tokenizer_p = GPT2Tokenizer.from_pretrained(tokenizer_name)
+            tokenizer_r = GPT2TokenizerFast.from_pretrained(tokenizer_name)
+
+            self._tokenize_inputs_and_check_matching(tokenizer_p, tokenizer_r)
+
+    def test_roberta(self):
+        for tokenizer_name in RobertaTokenizer.pretrained_vocab_files_map["vocab_file"].keys():
+            tokenizer_p = RobertaTokenizer.from_pretrained(tokenizer_name)
+            tokenizer_r = RobertaTokenizerFast.from_pretrained(tokenizer_name)
+
+            self._tokenize_inputs_and_check_matching(tokenizer_p, tokenizer_r)
+
+    def test_openai(self):
+        for tokenizer_name in OpenAIGPTTokenizer.pretrained_vocab_files_map["vocab_file"].keys():
+            tokenizer_p = OpenAIGPTTokenizer.from_pretrained(tokenizer_name)
+            tokenizer_r = OpenAIGPTTokenizerFast.from_pretrained(tokenizer_name)
+
+            self._tokenize_inputs_and_check_matching(tokenizer_p, tokenizer_r)
+
+
+if __name__ == "__main__":
+    unittest.main()