kwargs are passed to both model and configuration in AutoModels

Warning on `add_special_tokens` when passed to `encode`, `encode_plus` and `batch_encode_plus`

.gitignore

@@ -142,3 +142,6 @@ debug.env
 
 # vim
 .*.swp
+
+#ctags
+tags

examples/README.md

@@ -442,14 +442,14 @@ This example code fine-tunes XLNet on both SQuAD1.0 and SQuAD2.0 dataset. See ab
 ```bash
 export SQUAD_DIR=/path/to/SQUAD
 
-python /data/home/hlu/transformers/examples/run_squad.py \
+python run_squad.py \
     --model_type xlnet \
     --model_name_or_path xlnet-large-cased \
     --do_train \
     --do_eval \
     --do_lower_case \
-    --train_file /data/home/hlu/notebooks/NLP/examples/question_answering/train-v1.1.json \
-    --predict_file /data/home/hlu/notebooks/NLP/examples/question_answering/dev-v1.1.json \
+    --train_file $SQUAD_DIR/train-v1.1.json \
+    --predict_file $SQUAD_DIR/dev-v1.1.json \
     --learning_rate 3e-5 \
     --num_train_epochs 2 \
     --max_seq_length 384 \

model_cards/bert-base-german-cased-README.md

@@ -1,10 +1,10 @@
 ---
 language: german
-thumbnail: https://thumb.tildacdn.com/tild3162-6462-4566-b663-376630376138/-/format/webp/Screenshot_from_2020.png
+thumbnail: https://static.tildacdn.com/tild6438-3730-4164-b266-613634323466/german_bert.png
 ---
 
 # German BERT
-
+![bert_image](https://static.tildacdn.com/tild6438-3730-4164-b266-613634323466/german_bert.png)
 ## Overview
 **Language model:** bert-base-cased   
 **Language:** German  
@@ -68,4 +68,4 @@ Some of our work:
 - [Haystack](https://github.com/deepset-ai/haystack/)
 
 Get in touch:
-[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai)  
+[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai)  

model_cards/mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es/README.md

@@ -89,7 +89,37 @@ The model was trained on a Tesla P100 GPU and 25GB of RAM with the following com
 
 So, yes, this version is even more accurate.
 
-### Model in action (in a Colab Notebook)
+### Model in action
+
+Fast usage with **pipelines**:
+
+```python
+from transformers import *
+
+# Important!: By now the QA pipeline is not compatible with fast tokenizer, but they are working on it. So that pass the object to the tokenizer {"use_fast": False} as in the following example:
+
+nlp = pipeline(
+    'question-answering', 
+    model='mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es',
+    tokenizer=(
+        'mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es',  
+        {"use_fast": False}
+    )
+)
+
+nlp(
+    {
+        'question': '¿Para qué lenguaje está trabajando?',
+        'context': 'Manuel Romero está colaborando activamente con huggingface/transformers ' +
+                    'para traer el poder de las últimas técnicas de procesamiento de lenguaje natural al idioma español'
+    }
+)
+# Output: {'answer': 'español', 'end': 169, 'score': 0.67530957344621, 'start': 163}
+```
+
+Play with this model and ```pipelines``` in a Colab:
+
+<a href="https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/Using_Spanish_BERT_fine_tuned_for_Q%26A_pipelines.ipynb" target="_parent"><img src="https://camo.githubusercontent.com/52feade06f2fecbf006889a904d221e6a730c194/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Open In Colab" data-canonical-src="https://colab.research.google.com/assets/colab-badge.svg"></a>
 
 <details>
 
@@ -100,13 +130,12 @@ So, yes, this version is even more accurate.
 2.  Run predictions:
 
 ![Run the model](https://media.giphy.com/media/WT453aptcbCP7hxWTZ/giphy.gif)
-
-3. Using **Pipelines**
-
-![qa-pipelines](https://media.giphy.com/media/L4YiWhinC3P1GF31t0/giphy.gif)
-
 </details>
 
+More about ``` Huggingface pipelines```? check this Colab out:
+
+<a href="https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/Huggingface_pipelines_demo.ipynb" target="_parent"><img src="https://camo.githubusercontent.com/52feade06f2fecbf006889a904d221e6a730c194/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Open In Colab" data-canonical-src="https://colab.research.google.com/assets/colab-badge.svg"></a>
+
 > Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488)
 
 > Made with <span style="color: #e25555;">&hearts;</span> in Spain

src/transformers/modeling_auto.py

@@ -359,16 +359,12 @@ class AutoModel(object):
                 Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
 
             kwargs: (`optional`) Remaining dictionary of keyword arguments:
-                Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded:
-
-                - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)
-                - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.
+                These arguments will be passed to the configuration and the model.
 
         Examples::
 
             model = AutoModel.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
             model = AutoModel.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
-            model = AutoModel.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading
             assert model.config.output_attention == True
             # Loading from a TF checkpoint file instead of a PyTorch model (slower)
             config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
@@ -503,24 +499,12 @@ class AutoModelForPreTraining(object):
             output_loading_info: (`optional`) boolean:
                 Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
             kwargs: (`optional`) Remaining dictionary of keyword arguments:
-                Can be used to update the configuration object (after it being loaded) and initiate the model.
-                (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or
-                automatically loaded:
-
-                - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the
-                  underlying model's ``__init__`` method (we assume all relevant updates to the configuration have
-                  already been done)
-                - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class
-                  initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of
-                  ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute
-                  with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration
-                  attribute will be passed to the underlying model's ``__init__`` function.
+                These arguments will be passed to the configuration and the model.
 
         Examples::
 
             model = AutoModelForPreTraining.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
             model = AutoModelForPreTraining.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
-            model = AutoModelForPreTraining.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading
             assert model.config.output_attention == True
             # Loading from a TF checkpoint file instead of a PyTorch model (slower)
             config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
@@ -657,24 +641,12 @@ class AutoModelWithLMHead(object):
             output_loading_info: (`optional`) boolean:
                 Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
             kwargs: (`optional`) Remaining dictionary of keyword arguments:
-                Can be used to update the configuration object (after it being loaded) and initiate the model.
-                (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or
-                automatically loaded:
-
-                - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the
-                  underlying model's ``__init__`` method (we assume all relevant updates to the configuration have
-                  already been done)
-                - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class
-                  initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of
-                  ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute
-                  with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration
-                  attribute will be passed to the underlying model's ``__init__`` function.
+                These arguments will be passed to the configuration and the model.
 
         Examples::
 
             model = AutoModelWithLMHead.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
             model = AutoModelWithLMHead.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
-            model = AutoModelWithLMHead.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading
             assert model.config.output_attention == True
             # Loading from a TF checkpoint file instead of a PyTorch model (slower)
             config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
@@ -814,16 +786,12 @@ class AutoModelForSequenceClassification(object):
                 Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
 
             kwargs: (`optional`) Remaining dictionary of keyword arguments:
-                Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded:
-
-                - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)
-                - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.
+                These arguments will be passed to the configuration and the model.
 
         Examples::
 
             model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
             model = AutoModelForSequenceClassification.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
-            model = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading
             assert model.config.output_attention == True
             # Loading from a TF checkpoint file instead of a PyTorch model (slower)
             config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
@@ -957,16 +925,12 @@ class AutoModelForQuestionAnswering(object):
                 Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
 
             kwargs: (`optional`) Remaining dictionary of keyword arguments:
-                Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded:
-
-                - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)
-                - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.
+                These arguments will be passed to the configuration and the model.
 
         Examples::
 
             model = AutoModelForQuestionAnswering.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
             model = AutoModelForQuestionAnswering.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
-            model = AutoModelForQuestionAnswering.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading
             assert model.config.output_attention == True
             # Loading from a TF checkpoint file instead of a PyTorch model (slower)
             config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
@@ -1101,16 +1065,12 @@ class AutoModelForTokenClassification:
                 Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.
 
             kwargs: (`optional`) Remaining dictionary of keyword arguments:
-                Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded:
-
-                - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)
-                - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.
+                These arguments will be passed to the configuration and the model.
 
         Examples::
 
             model = AutoModelForTokenClassification.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.
             model = AutoModelForTokenClassification.from_pretrained('./test/bert_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`
-            model = AutoModelForTokenClassification.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading
             assert model.config.output_attention == True
             # Loading from a TF checkpoint file instead of a PyTorch model (slower)
             config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')

src/transformers/modeling_bart.py

@@ -470,10 +470,6 @@ class BartDecoder(nn.Module):
         """
         # embed positions
         positions = self.embed_positions(input_ids)
-
-        if decoder_cached_states is not None:
-            input_ids = input_ids[:, -1:]
-            positions = positions[:, -1:]
         x = self.embed_tokens(input_ids)
 
         if positions is not None:
@@ -491,7 +487,7 @@ class BartDecoder(nn.Module):
             decoder_layer  # type: DecoderLayer
             # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
             dropout_probability = random.uniform(0, 1)
-            if self.training and (dropout_probability > self.layerdrop):
+            if self.training and (dropout_probability < self.layerdrop):
                 continue
             layer_state = decoder_cached_states[i] if decoder_cached_states is not None else None
             x, layer_self_attn, layer_past = decoder_layer.forward(
@@ -940,7 +936,7 @@ class BartForMaskedLM(PretrainedBartModel):
 
     @staticmethod
     def prepare_inputs_for_generation(input_ids, past, **kwargs):
-        return {"input_ids": input_ids, "decoder_cached_states": past, "decoder_input_ids": input_ids}
+        return {"input_ids": input_ids, "decoder_cached_states": past, "decoder_input_ids": input_ids[:, -1:]}
 
     def get_output_embeddings(self):
         return self.lm_head

src/transformers/modeling_encoder_decoder.py

@@ -18,7 +18,6 @@
 import logging
 import os
 
-import torch
 from torch import nn
 
 from .modeling_auto import AutoModel, AutoModelWithLMHead
@@ -294,21 +293,3 @@ class Model2Model(PreTrainedEncoderDecoder):
         )
 
         return model
-
-
-class Model2LSTM(PreTrainedEncoderDecoder):
-    @classmethod
-    def from_pretrained(cls, *args, **kwargs):
-        if kwargs.get("decoder_model", None) is None:
-            # We will create a randomly initilized LSTM model as decoder
-            if "decoder_config" not in kwargs:
-                raise ValueError(
-                    "To load an LSTM in Encoder-Decoder model, please supply either: "
-                    "    - a torch.nn.LSTM model as `decoder_model` parameter (`decoder_model=lstm_model`), or"
-                    "    - a dictionary of configuration parameters that will be used to initialize a"
-                    "      torch.nn.LSTM model as `decoder_config` keyword argument. "
-                    "      E.g. `decoder_config={'input_size': 768, 'hidden_size': 768, 'num_layers': 2}`"
-                )
-            kwargs["decoder_model"] = torch.nn.LSTM(kwargs.pop("decoder_config"))
-        model = super().from_pretrained(*args, **kwargs)
-        return model

src/transformers/tokenization_albert.py

@@ -114,6 +114,11 @@ class AlbertTokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return len(self.sp_model)
 
+    def get_vocab(self):
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
     def __getstate__(self):
         state = self.__dict__.copy()
         state["sp_model"] = None

src/transformers/tokenization_bert.py

@@ -195,6 +195,9 @@ class BertTokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return len(self.vocab)
 
+    def get_vocab(self):
+        return dict(self.vocab, **self.added_tokens_encoder)
+
     def _tokenize(self, text):
         split_tokens = []
         if self.do_basic_tokenize:

src/transformers/tokenization_ctrl.py

@@ -147,6 +147,9 @@ class CTRLTokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return len(self.encoder)
 
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+
     def bpe(self, token):
         if token in self.cache:
             return self.cache[token]

src/transformers/tokenization_gpt2.py

@@ -149,6 +149,9 @@ class GPT2Tokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return len(self.encoder)
 
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+
     def bpe(self, token):
         if token in self.cache:
             return self.cache[token]

src/transformers/tokenization_openai.py

@@ -125,6 +125,9 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return len(self.encoder)
 
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+
     def bpe(self, token):
         word = tuple(token[:-1]) + (token[-1] + "</w>",)
         if token in self.cache:

src/transformers/tokenization_t5.py

@@ -119,6 +119,11 @@ class T5Tokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return self.sp_model.get_piece_size() + self._extra_ids
 
+    def get_vocab(self):
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
     def __getstate__(self):
         state = self.__dict__.copy()
         state["sp_model"] = None

src/transformers/tokenization_transfo_xl.py

@@ -273,6 +273,9 @@ class TransfoXLTokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return len(self.idx2sym)
 
+    def get_vocab(self):
+        return dict(self.sym2idx, **self.added_tokens_encoder)
+
     def _tokenize(self, line, add_eos=False, add_double_eos=False):
         line = line.strip()
         # convert to lower case

src/transformers/tokenization_utils.py

@@ -286,6 +286,10 @@ class PreTrainedTokenizer(object):
         """ Ids of all the additional special tokens in the vocabulary (list of integers). Log an error if used while not having been set. """
         return self.convert_tokens_to_ids(self.additional_special_tokens)
 
+    def get_vocab(self):
+        """ Returns the vocabulary as a dict of {token: index} pairs. `tokenizer.get_vocab()[token]` is equivalent to `tokenizer.convert_tokens_to_ids(token)` when `token` is in the vocab. """
+        raise NotImplementedError()
+
     def __init__(self, max_len=None, **kwargs):
         self._bos_token = None
         self._eos_token = None
@@ -1229,7 +1233,10 @@ class PreTrainedTokenizer(object):
             token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else [])
 
         if return_special_tokens_mask:
-            encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids)
+            if add_special_tokens:
+                encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids)
+            else:
+                encoded_inputs["special_tokens_mask"] = [0] * len(sequence)
 
         encoded_inputs["input_ids"] = sequence
         if return_token_type_ids:

src/transformers/tokenization_xlm.py

@@ -662,6 +662,9 @@ class XLMTokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return len(self.encoder)
 
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+
     def bpe(self, token):
         word = tuple(token[:-1]) + (token[-1] + "</w>",)
         if token in self.cache:

src/transformers/tokenization_xlm_roberta.py

@@ -190,6 +190,11 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return len(self.sp_model) + len(self.fairseq_tokens_to_ids)
 
+    def get_vocab(self):
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
     def _tokenize(self, text):
         return self.sp_model.EncodeAsPieces(text)
 

src/transformers/tokenization_xlnet.py

@@ -114,6 +114,11 @@ class XLNetTokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return len(self.sp_model)
 
+    def get_vocab(self):
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
     def __getstate__(self):
         state = self.__dict__.copy()
         state["sp_model"] = None

tests/test_modeling_bart.py

@@ -251,6 +251,7 @@ class BartHeadTests(unittest.TestCase):
             output_past=True,
         )
         lm_model = BartForMaskedLM(config)
+        lm_model.eval()
         new_input_ids = lm_model.generate(input_ids)
         self.assertEqual(new_input_ids.shape, (input_ids.shape[0], 20))
 

tests/test_tokenization_common.py

@@ -542,3 +542,23 @@ class TokenizerTesterMixin:
         print(new_tokenizer.init_kwargs)
         assert tokenizer.init_kwargs["random_argument"] is True
         assert new_tokenizer.init_kwargs["random_argument"] is False
+
+    def test_get_vocab(self):
+        tokenizer = self.get_tokenizer()
+        vocab = tokenizer.get_vocab()
+
+        self.assertIsInstance(vocab, dict)
+        self.assertEqual(len(vocab), len(tokenizer))
+
+        for word, ind in vocab.items():
+            self.assertEqual(tokenizer.convert_tokens_to_ids(word), ind)
+            self.assertEqual(tokenizer.convert_ids_to_tokens(ind), word)
+
+        tokenizer.add_tokens(["asdfasdfasdfasdf"])
+        vocab = tokenizer.get_vocab()
+        self.assertIsInstance(vocab, dict)
+        self.assertEqual(len(vocab), len(tokenizer))
+
+        for word, ind in vocab.items():
+            self.assertEqual(tokenizer.convert_tokens_to_ids(word), ind)
+            self.assertEqual(tokenizer.convert_ids_to_tokens(ind), word)