Actually changing the default resolution order to account for model

defaults

Adding a new tests for each pipeline to check that pipeline(task) works
too without manually adding the framework too.

docs/source/model_doc/blenderbot.rst

@@ -1,5 +1,5 @@
 Blenderbot
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+-----------------------------------------------------------------------------------------------------------------------
 **DISCLAIMER:** If you see something strange,
 file a `Github Issue <https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ .
 

examples/lightning_base.py

@@ -119,7 +119,7 @@ class BaseTransformer(pl.LightningModule):
     def get_lr_scheduler(self):
         get_schedule_func = arg_to_scheduler[self.hparams.lr_scheduler]
         scheduler = get_schedule_func(
-            self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=self.total_steps
+            self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=self.total_steps()
         )
         scheduler = {"scheduler": scheduler, "interval": "step", "frequency": 1}
         return scheduler
@@ -159,19 +159,20 @@ class BaseTransformer(pl.LightningModule):
     def test_epoch_end(self, outputs):
         return self.validation_end(outputs)
 
-    @property
     def total_steps(self) -> int:
         """The number of total training steps that will be run. Used for lr scheduler purposes."""
         num_devices = max(1, self.hparams.gpus)  # TODO: consider num_tpu_cores
         effective_batch_size = self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
-        dataset_size = len(self.train_loader.dataset)
-        return (dataset_size / effective_batch_size) * self.hparams.max_epochs
+        return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs
 
     def setup(self, mode):
-        if mode == "fit":
+        if mode == "test":
+            self.dataset_size = len(self.test_dataloader().dataset)
+        else:
             self.train_loader = self.get_dataloader("train", self.hparams.train_batch_size, shuffle=True)
+            self.dataset_size = len(self.train_loader.dataset)
 
-    def get_dataloader(self, type_path, batch_size, shuffle=False):
+    def get_dataloader(self, type_path: str, batch_size: int, shuffle: bool = False):
         raise NotImplementedError("You must implement this for your task")
 
     def train_dataloader(self):

examples/requirements.txt

@@ -5,7 +5,7 @@ psutil
 sacrebleu
 rouge-score
 tensorflow_datasets
-pytorch-lightning==0.8.5
+pytorch-lightning==0.9.0
 matplotlib
 git-python==1.0.3
 faiss-cpu

examples/seq2seq/README.md

@@ -12,7 +12,6 @@ For `bertabs` instructions, see [`bertabs/README.md`](bertabs/README.md).
 - `MBartForConditionalGeneration`
 - `FSMTForConditionalGeneration`
 - `T5ForConditionalGeneration`
-    
 
 ## Datasets
 
@@ -100,7 +99,7 @@ All finetuning bash scripts call finetune.py (or distillation.py) with reasonabl
 To see all the possible command line options, run:
 
 ```bash
- ./finetune.py --help 
+./finetune.py --help
 ```
 
 ### Finetuning Training Params
@@ -192,7 +191,7 @@ model = AutoModelForSeq2SeqLM.from_pretrained(f'{output_dir}/best_tfmr')
 ### Fine-tuning using Seq2SeqTrainer
 To use `Seq2SeqTrainer` for fine-tuning you should use the `finetune_trainer.py` script. It subclasses `Trainer` to extend it for seq2seq training. Except the `Trainer` releated `TrainingArguments`, it shares the same argument names as that of `finetune.py` file. One notable difference is that, calculating generative metrics (BLEU, ROUGE) is optional and is controlled using the `--predict_with_generate` argument, set this argument to calculate BLEU and ROUGE metrics.
 
-With PyTorch 1.6+ it'll automatically use `native AMP` when `--fp16` is set. 
+With PyTorch 1.6+ it'll automatically use `native AMP` when `--fp16` is set.
 
 To see all the possible command line options, run:
 
@@ -265,6 +264,7 @@ export DATA_DIR=cnn_dm
     --fp16 \
     --bs 32
 ```
+
 ### Multi-GPU Evaluation
 here is a command to run xsum evaluation on 8 GPUS. It is more than linearly faster than run_eval.py in some cases 
 because it uses SortishSampler to minimize padding. You can also use it on 1 GPU. `data_dir` must have 
@@ -391,6 +391,17 @@ runtime: 13H on V-100 16GB GPU.
 pytest examples/seq2seq/
 ```
 
+### Converting pytorch-lightning checkpoints
+pytorch lightning ``-do_predict`` often fails, after you are done training, the best way to evaluate your model is to convert it.
+
+This should be done for you, with a file called `{save_dir}/best_tfmr`. 
+
+If that file doesn't exist but you have a lightning `.ckpt` file, you can run
+```bash
+python convert_pl_checkpoint_to_hf.py PATH_TO_CKPT  randomly_initialized_hf_model_path save_dir/best_tfmr
+```
+Then either `run_eval` or `run_distributed_eval` with `save_dir/best_tfmr` (see previous sections)
+
 
 ## Experimental Features 
 These features are harder to use and not always useful.
@@ -419,4 +430,3 @@ uses 12,723 batches of length 48 and takes slightly more time 9.5 minutes.
 The feature is still experimental, because:
 + we can make it much more robust if we have memory mapped/preprocessed datasets.
 + The speedup over sortish sampler is not that large at the moment.
-

examples/seq2seq/distillation.py

@@ -17,7 +17,7 @@ from finetune import main as ft_main
 from make_student import create_student_by_copying_alternating_layers, get_layers_to_supervise
 from transformers import AutoModelForSeq2SeqLM, MBartTokenizer, T5ForConditionalGeneration
 from transformers.modeling_bart import shift_tokens_right
-from utils import calculate_bleu, freeze_params, label_smoothed_nll_loss, pickle_load, use_task_specific_params
+from utils import calculate_bleu, freeze_params, label_smoothed_nll_loss, use_task_specific_params
 
 
 # need the parent dir module
@@ -264,30 +264,6 @@ def create_module(args):
     return model
 
 
-def evaluate_checkpoint(ckpt_path: Path, dest_dir=None):
-    # TODO(SS): DELETE? Better to convert_pl_ckpt_to_hf and run_eval.py
-    exp_dir = ckpt_path.parent
-    if dest_dir is None:
-        dest_dir = exp_dir
-    clash = list(dest_dir.glob("test_generations*"))
-    if clash:
-        print(f"SKIPPING to avoid overwriting {clash}")
-    ckpt = torch.load(ckpt_path, map_location="cpu")
-    if "hparams" in ckpt:
-        args = argparse.Namespace(**ckpt["hparams"])
-    else:
-        args = argparse.Namespace(**pickle_load(exp_dir / "hparams.pkl"))
-    args.resume_from_checkpoint = str(ckpt_path)
-    args.do_train = False
-    args.output_dir = str(dest_dir)
-    args.n_gpu = 1
-    args.eval_batch_size = 16
-    Path(args.output_dir).mkdir(exist_ok=True)
-    model = create_module(args)
-    trainer: pl.Trainer = generic_train(model, args, early_stopping_callback=False)
-    trainer.test(model)
-
-
 def distill_main(args):
     Path(args.output_dir).mkdir(exist_ok=True)
     if len(os.listdir(args.output_dir)) > 3 and args.do_train:

examples/seq2seq/finetune.py

@@ -181,6 +181,7 @@ class SummarizationModule(BaseTransformer):
         return self._generative_step(batch)
 
     def validation_epoch_end(self, outputs, prefix="val") -> Dict:
+
         self.step_count += 1
         losses = {k: torch.stack([x[k] for x in outputs]).mean() for k in self.loss_names}
         loss = losses["loss"]

examples/seq2seq/test_seq2seq_examples.py

@@ -13,7 +13,7 @@ import torch
 
 import lightning_base
 from convert_pl_checkpoint_to_hf import convert_pl_to_hf
-from distillation import distill_main, evaluate_checkpoint
+from distillation import distill_main
 from finetune import SummarizationModule, main
 from run_eval import generate_summaries_or_translations, run_generate
 from run_eval_search import run_search
@@ -178,7 +178,6 @@ class TestSummarizationDistiller(unittest.TestCase):
         generate_summaries_or_translations(examples, out_path, str(model.output_dir / "best_tfmr"))
         self.assertTrue(Path(out_path).exists())
 
-        evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
         out_path_new = tempfile.mkdtemp()
         convert_pl_to_hf(ckpts[0], transformer_ckpts[0].parent, out_path_new)
         assert os.path.exists(os.path.join(out_path_new, "pytorch_model.bin"))
@@ -227,8 +226,6 @@ class TestSummarizationDistiller(unittest.TestCase):
         assert len(all_files) > 2
         self.assertEqual(len(transformer_ckpts), 2)
 
-        evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
-
     def test_distill_t5(self):
         updates = dict(
             student_encoder_layers=1,

examples/test_examples.py

@@ -116,8 +116,8 @@ class ExamplesTests(TestCasePlus):
             testargs.append("--fp16")
 
         with patch.object(sys, "argv", testargs):
-            result = run_pl_glue.main()
-            # for now just testing that the script can run to a completion
+            result = run_pl_glue.main()[0]
+            # for now just testing that the script can run to completion
             self.assertGreater(result["acc"], 0.25)
             #
             # TODO: this fails on CI - doesn't get acc/f1>=0.75:

model_cards/distilbert-base-cased-README.md

@@ -0,0 +1,3 @@
+---
+license: apache-2.0
+---

model_cards/distilbert-base-cased-distilled-squad-README.md

@@ -4,4 +4,5 @@ datasets:
 - squad
 metrics:
 - squad
+license: apache-2.0
 ---

model_cards/distilbert-base-german-cased-README.md

@@ -1,4 +1,5 @@
 ---
 language: de
+license: apache-2.0
 ---
 ## distilbert-base-german-cased

model_cards/distilbert-base-uncased-distilled-squad-README.md

@@ -6,4 +6,5 @@ widget:
   context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
 - text: "How many square kilometers of rainforest is covered in the basin?"
   context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species."
+license: apache-2.0
 ---

model_cards/distilbert-base-uncased-finetuned-sst-2-english-README.md

@@ -0,0 +1,3 @@
+---
+license: apache-2.0
+---

model_cards/facebook/bart-large-mnli/README.md

@@ -0,0 +1,3 @@
+---
+license: mit
+---

scripts/pegasus/build_test_sample_spm_no_bos.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+# this script builds a small sample spm file tests/fixtures/test_sentencepiece_no_bos.model, with features needed by pegasus 
+
+# 1. pip install sentencepiece
+# 
+# 2. wget https://raw.githubusercontent.com/google/sentencepiece/master/data/botchan.txt
+
+# 3. build
+import sentencepiece as spm
+
+# pegasus:
+# 1. no bos
+# 2. eos_id is 1
+# 3. unk_id is 2
+# build a sample spm file accordingly
+spm.SentencePieceTrainer.train('--input=botchan.txt --model_prefix=test_sentencepiece_no_bos --bos_id=-1 --unk_id=2  --eos_id=1  --vocab_size=1000')
+
+# 4. now update the fixture
+# mv test_sentencepiece_no_bos.model ../../tests/fixtures/

setup.py

@@ -6,6 +6,7 @@ To create the package for pypi.
 1. Change the version in __init__.py, setup.py as well as docs/source/conf.py. Remove the master from the links in
    the new models of the README:
    (https://huggingface.co/transformers/master/model_doc/ -> https://huggingface.co/transformers/model_doc/)
+   then run `make fix-copies` to fix the index of the documentation.
 
 2. Unpin specific versions from setup.py that use a git install.
 
@@ -133,9 +134,7 @@ setup(
         "sacremoses",
     ],
     extras_require=extras,
-    entry_points={
-        "console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]
-    },
+    entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
     python_requires=">=3.6.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",

src/transformers/integrations.py

@@ -2,13 +2,11 @@
 import math
 import os
 
-from .file_utils import is_torch_tpu_available
-from .trainer_callback import TrainerCallback
-from .trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun
-from .utils import logging
 
+# Import 3rd-party integrations first:
 
 try:
+    # Comet needs to be imported before any ML frameworks
     import comet_ml  # noqa: F401
 
     _has_comet = True
@@ -53,6 +51,14 @@ except ImportError:
     except ImportError:
         _has_tensorboard = False
 
+# No transformer imports above this point
+
+from .file_utils import is_torch_tpu_available
+from .trainer_callback import TrainerCallback
+from .trainer_utils import PREFIX_CHECKPOINT_DIR, BestRun
+from .utils import logging
+
+
 logger = logging.get_logger(__name__)
 
 

src/transformers/modeling_auto.py

@@ -532,7 +532,7 @@ AUTO_MODEL_PRETRAINED_DOCSTRING = r"""
 class AutoModel:
     r"""
     This is a generic model class that will be instantiated as one of the base model classes of the library
-    when created with the when created with the :meth:`~transformers.AutoModel.from_pretrained` class method or the
+    when created with the :meth:`~transformers.AutoModel.from_pretrained` class method or the
     :meth:`~transformers.AutoModel.from_config` class methods.
 
     This class cannot be instantiated directly using ``__init__()`` (throws an error).

src/transformers/modeling_bert.py

@@ -667,7 +667,7 @@ BERT_INPUTS_DOCSTRING = r"""
             Mask values selected in ``[0, 1]``:
 
             - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **maked**.
+            - 0 for tokens that are **masked**.
 
             `What are attention masks? <../glossary.html#attention-mask>`__
         token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
@@ -781,7 +781,7 @@ class BertModel(BertPreTrainedModel):
             Mask values selected in ``[0, 1]``:
 
             - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **maked**.
+            - 0 for tokens that are **masked**.
         """
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -1012,7 +1012,7 @@ class BertLMHeadModel(BertPreTrainedModel):
             Mask values selected in ``[0, 1]``:
 
             - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **maked**.
+            - 0 for tokens that are **masked**.
         labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
             Labels for computing the left-to-right language modeling loss (next word prediction).
             Indices should be in ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)

src/transformers/pipelines.py

@@ -85,31 +85,63 @@ if TYPE_CHECKING:
 logger = logging.get_logger(__name__)
 
 
-def get_framework(model=None):
+def get_framework(model):
     """
     Select framework (TensorFlow or PyTorch) to use.
 
     Args:
-        model (:obj:`str`, :class:`~transformers.PreTrainedModel` or :class:`~transformers.TFPreTrainedModel`, `optional`):
+        model (:obj:`str`, :class:`~transformers.PreTrainedModel` or :class:`~transformers.TFPreTrainedModel`):
             If both frameworks are installed, picks the one corresponding to the model passed (either a model class or
             the model name). If no specific model is provided, defaults to using PyTorch.
     """
-    if is_tf_available() and is_torch_available() and model is not None and not isinstance(model, str):
-        # Both framework are available but the user supplied a model class instance.
-        # Try to guess which framework to use from the model classname
-        framework = "tf" if model.__class__.__name__.startswith("TF") else "pt"
-    elif not is_tf_available() and not is_torch_available():
+    if not is_tf_available() and not is_torch_available():
         raise RuntimeError(
             "At least one of TensorFlow 2.0 or PyTorch should be installed. "
             "To install TensorFlow 2.0, read the instructions at https://www.tensorflow.org/install/ "
             "To install PyTorch, read the instructions at https://pytorch.org/."
         )
-    else:
-        # framework = 'tf' if is_tf_available() else 'pt'
-        framework = "pt" if is_torch_available() else "tf"
+    if isinstance(model, str):
+        if is_torch_available() and not is_tf_available():
+            model = AutoModel.from_pretrained(model)
+        elif is_tf_available() and not is_torch_available():
+            model = TFAutoModel.from_pretrained(model)
+        else:
+            try:
+                model = AutoModel.from_pretrained(model)
+            except OSError:
+                model = TFAutoModel.from_pretrained(model)
+
+    framework = "tf" if model.__class__.__name__.startswith("TF") else "pt"
     return framework
 
 
+def get_default_model(targeted_task: Dict, framework: Optional[str]) -> str:
+    """
+    Select a default model to use for a given task. Defaults to pytorch if ambiguous.
+
+    Args:
+        targeted_task (:obj:`Dict` ):
+           Dictionnary representing the given task, that should contain default models
+
+        framework (:obj:`str`, None)
+           "pt", "tf" or None, representing a specific framework if it was specified, or None if we don't know yet.
+
+    Returns
+
+        :obj:`str` The model string representing the default model for this pipeline
+    """
+    if is_torch_available() and not is_tf_available():
+        framework = "pt"
+    elif is_tf_available() and not is_torch_available():
+        framework = "tf"
+
+    default_models = targeted_task["default"]["model"]
+    if framework is None:
+        framework = "pt"
+
+    return default_models[framework]
+
+
 class PipelineException(Exception):
     """
     Raised by a :class:`~transformers.Pipeline` when handling __call__.
@@ -2678,14 +2710,16 @@ def pipeline(
     if task not in SUPPORTED_TASKS:
         raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys())))
 
-    framework = framework or get_framework(model)
-
     targeted_task = SUPPORTED_TASKS[task]
-    task_class, model_class = targeted_task["impl"], targeted_task[framework]
 
     # Use default model/config/tokenizer for the task if no model is provided
     if model is None:
-        model = targeted_task["default"]["model"][framework]
+        # At that point framework might still be undetermined
+        model = get_default_model(targeted_task, framework)
+
+    framework = framework or get_framework(model)
+
+    task_class, model_class = targeted_task["impl"], targeted_task[framework]
 
     # Try to infer tokenizer from model or config name (if provided as str)
     if tokenizer is None:

src/transformers/testing_utils.py

@@ -184,13 +184,23 @@ def require_faiss(test_case):
         return test_case
 
 
-def get_tests_dir():
+def get_tests_dir(append_path=None):
     """
-    returns the full path to the `tests` dir, so that the tests can be invoked from anywhere
+    Args:
+        append_path: optional path to append to the tests dir path
+
+    Return:
+        The full path to the `tests` dir, so that the tests can be invoked from anywhere.
+        Optionally `append_path` is joined after the `tests` dir the former is provided.
+
     """
     # this function caller's __file__
     caller__file__ = inspect.stack()[1][1]
-    return os.path.abspath(os.path.dirname(caller__file__))
+    tests_dir = os.path.abspath(os.path.dirname(caller__file__))
+    if append_path:
+        return os.path.join(tests_dir, append_path)
+    else:
+        return tests_dir
 
 
 #

src/transformers/tokenization_pegasus.py

@@ -49,7 +49,7 @@ class PegasusTokenizer(ReformerTokenizer):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        # Dont use reserved words added_token_encoder, added_tokens_decoder because of
+        # Don't use reserved words added_token_encoder, added_tokens_decoder because of
         # AssertionError: Non-consecutive added token '1' found. in from_pretrained
         assert len(self.added_tokens_decoder) == 0
         self.encoder: Dict[int, str] = {0: self.pad_token, 1: self.eos_token}
@@ -58,7 +58,7 @@ class PegasusTokenizer(ReformerTokenizer):
         self.decoder: Dict[str, int] = {v: k for k, v in self.encoder.items()}
 
     def _convert_token_to_id(self, token: str) -> int:
-        """ Converts a token (str) in an id using the vocab. """
+        """ Converts a token (str) to an id using the vocab. """
         if token in self.decoder:
             return self.decoder[token]
         elif token in self.added_tokens_decoder:
@@ -67,7 +67,7 @@ class PegasusTokenizer(ReformerTokenizer):
         return sp_id + self.offset
 
     def _convert_id_to_token(self, index: int) -> str:
-        """Converts an index (integer) in a token (str) using the vocab."""
+        """Converts an index (integer) to a token (str) using the vocab."""
         if index in self.encoder:
             return self.encoder[index]
         elif index in self.added_tokens_encoder:
@@ -81,11 +81,6 @@ class PegasusTokenizer(ReformerTokenizer):
     def vocab_size(self) -> int:
         return len(self.sp_model) + self.offset
 
-    def get_vocab(self) -> Dict[str, int]:
-        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
-        vocab.update(self.added_tokens_encoder)
-        return vocab
-
     def num_special_tokens_to_add(self, pair=False):
         """Just EOS"""
         return 1
@@ -109,12 +104,12 @@ class PegasusTokenizer(ReformerTokenizer):
 
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
         """
-        Build model inputs from a sequence or a pair of sequence for sequence classification tasks
+        Build model inputs from a sequence or a pair of sequences for sequence classification tasks
         by concatenating and adding special tokens.
         A Pegasus sequence has the following format, where ``X`` represents the sequence:
 
         - single sequence: ``X </s>``
-        - pair of sequences: ``A B </s>``  (not intended use)
+        - pair of sequences: ``A B </s>`` (not intended use)
 
         BOS is never used.
         Pairs of sequences are not the expected use case, but they will be handled without a separator.

src/transformers/tokenization_reformer.py

@@ -17,6 +17,7 @@
 
 import os
 from shutil import copyfile
+from typing import Dict
 
 from .tokenization_utils import PreTrainedTokenizer
 from .tokenization_utils_fast import PreTrainedTokenizerFast
@@ -119,7 +120,7 @@ class ReformerTokenizer(PreTrainedTokenizer):
     def vocab_size(self):
         return self.sp_model.get_piece_size()
 
-    def get_vocab(self):
+    def get_vocab(self) -> Dict[str, int]:
         vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
         vocab.update(self.added_tokens_encoder)
         return vocab

src/transformers/tokenization_utils.py

@@ -186,7 +186,7 @@ class PreTrainedTokenizer(PreTrainedTokenizerBase):
 
             num_added_toks = tokenizer.add_tokens(['new_tok1', 'my_new-tok2'])
             print('We have added', num_added_toks, 'tokens')
-            # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e. the length of the tokenizer.
+            # Note: resize_token_embeddings expects to receive the full size of the new vocabulary, i.e. the length of the tokenizer.
             model.resize_token_embeddings(len(tokenizer))
         """
         new_tokens = [str(tok) for tok in new_tokens]

test_tokenization_blenderbot.py

@@ -1,50 +0,0 @@
-import json
-import os
-import unittest
-
-from transformers.testing_utils import slow
-from transformers.tokenization_blenderbot import VOCAB_FILES_NAMES, BlenderbotTokenizer, BlenderbotSmallTokenizer
-
-from .test_tokenization_common import TokenizerTesterMixin
-
-class BlenderbotSmallTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
-    
-    tokenizer_class = BlenderbotSmallTokenizer
-
-    def setUp(self):
-        super().setUp()
-
-        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
-        vocab = ["adapt", "react", "read@@", "ap@@", "t", "__unk__", "__start__", "__end__", "__null__"]
-        vocab_tokens = dict(zip(vocab, range(len(vocab))))
-        merges = ["#version: 0.2", "a p", "ap t</w>", "r e", "a d", "ad apt</w>", ""]
-        self.special_tokens_map = {"bos_token": "__start", "eos_token": "__end__", "pad_token": "__null__", "unk_token": "__unk__"}
-
-        self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
-        self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["merges_file"])
-        with open(self.vocab_file, "w", encoding="utf-8") as fp:
-            fp.write(json.dumps(vocab_tokens) + "\n")
-        with open(self.merges_file, "w", encoding="utf-8") as fp:
-            fp.write("\n".join(merges))
-
-    def get_tokenizer(self, **kwargs):
-        kwargs.update(self.special_tokens_map)
-        return BlenderbotSmallTokenizer.from_pretrained(self.tmpdirname, **kwargs)
-
-    def get_input_output_texts(self, tokenizer):
-        input_text = "adapt react readapt apt"
-        output_text = "adapt react readapt apt"
-        return input_text, output_text
-    
-    def test_full_blenderbot_small_tokenizer(self):
-        tokenizer = BlenderbotSmallTokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map)
-        text = "adapt react readapt apt"
-        bpe_tokens = ['adapt', 'react', 'read@@', 'ap@@', 't', 'ap@@', 't']
-        tokens = tokenizer.tokenize(text)
-        self.assertListEqual(tokens, bpe_tokens)
-
-        input_tokens = [tokenizer.bos_token] + tokens + [tokenizer.eos_token]
-        print(input_tokens)
-
-        # input_bpe_tokens = [0, 1, 2, 4, 5, 1, 0, 3, 6]
-        # self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens)

tests/test_pipelines.py

@@ -10,6 +10,7 @@ DEFAULT_DEVICE_NUM = -1 if torch_device == "cpu" else 0
 VALID_INPUTS = ["A simple string", ["list of strings"]]
 
 NER_FINETUNED_MODELS = ["sshleifer/tiny-dbmdz-bert-large-cased-finetuned-conll03-english"]
+TF_NER_FINETUNED_MODELS = ["Narsil/small"]
 
 # xlnet-base-cased disabled for now, since it crashes TF2
 FEATURE_EXTRACT_FINETUNED_MODELS = ["sshleifer/tiny-distilbert-base-cased"]
@@ -804,6 +805,14 @@ class NerPipelineTests(unittest.TestCase):
             nlp = pipeline(task="ner", model=model_name, tokenizer=model_name, framework="tf", grouped_entities=True)
             self._test_ner_pipeline(nlp, mandatory_keys)
 
+    @require_tf
+    def test_tf_only_ner(self):
+        mandatory_keys = {"entity", "word", "score"}
+        for model_name in TF_NER_FINETUNED_MODELS:
+            # We don't specificy framework='tf' but it gets detected automatically
+            nlp = pipeline(task="ner", model=model_name, tokenizer=model_name)
+            self._test_ner_pipeline(nlp, mandatory_keys)
+
 
 class PipelineCommonTests(unittest.TestCase):
     pipelines = SUPPORTED_TASKS.keys()
@@ -815,6 +824,7 @@ class PipelineCommonTests(unittest.TestCase):
         for task in self.pipelines:
             with self.subTest(msg="Testing TF defaults with TF and {}".format(task)):
                 pipeline(task, framework="tf")
+                pipeline(task)
 
     @require_torch
     @slow
@@ -823,3 +833,4 @@ class PipelineCommonTests(unittest.TestCase):
         for task in self.pipelines:
             with self.subTest(msg="Testing Torch defaults with PyTorch and {}".format(task)):
                 pipeline(task, framework="pt")
+                pipeline(task)

tests/test_tokenization_pegasus.py

@@ -1,13 +1,15 @@
 import unittest
-from pathlib import Path
 
 from transformers.file_utils import cached_property
-from transformers.testing_utils import require_torch
+from transformers.testing_utils import get_tests_dir, require_torch
 from transformers.tokenization_pegasus import PegasusTokenizer, PegasusTokenizerFast
 
 from .test_tokenization_common import TokenizerTesterMixin
 
 
+SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece_no_bos.model")
+
+
 class PegasusTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = PegasusTokenizer
@@ -17,11 +19,9 @@ class PegasusTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
     def setUp(self):
         super().setUp()
 
-        save_dir = Path(self.tmpdirname)
-        spm_file = PegasusTokenizer.vocab_files_names["vocab_file"]
-        if not (save_dir / spm_file).exists():
-            tokenizer = self.pegasus_large_tokenizer
-            tokenizer.save_pretrained(self.tmpdirname)
+        # We have a SentencePiece fixture for testing
+        tokenizer = PegasusTokenizer(SAMPLE_VOCAB)
+        tokenizer.save_pretrained(self.tmpdirname)
 
     @cached_property
     def pegasus_large_tokenizer(self):
@@ -32,10 +32,7 @@ class PegasusTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
         pass
 
     def get_tokenizer(self, **kwargs) -> PegasusTokenizer:
-        if not kwargs:
-            return self.pegasus_large_tokenizer
-        else:
-            return PegasusTokenizer.from_pretrained(self.tmpdirname, **kwargs)
+        return PegasusTokenizer.from_pretrained(self.tmpdirname, **kwargs)
 
     def get_input_output_texts(self, tokenizer):
         return ("This is a test", "This is a test")

tests/test_tokenization_t5.py

@@ -14,19 +14,18 @@
 # limitations under the License.
 
 
-import os
 import unittest
 
 from transformers import BatchEncoding
 from transformers.file_utils import cached_property
-from transformers.testing_utils import _torch_available
+from transformers.testing_utils import _torch_available, get_tests_dir
 from transformers.tokenization_t5 import T5Tokenizer, T5TokenizerFast
 from transformers.tokenization_xlnet import SPIECE_UNDERLINE
 
 from .test_tokenization_common import TokenizerTesterMixin
 
 
-SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
+SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")
 
 FRAMEWORK = "pt" if _torch_available else "tf"
 

tests/test_trainer.py

@@ -195,7 +195,9 @@ class TrainerIntegrationTest(unittest.TestCase):
         args = TrainingArguments("./regression")
         dict1, dict2 = args.to_dict(), trainer.args.to_dict()
         for key in dict1.keys():
-            self.assertEqual(dict1[key], dict2[key])
+            # Logging dir can be slightly different as they default to something with the time.
+            if key != "loggin_dir":
+                self.assertEqual(dict1[key], dict2[key])
 
     def test_reproducible_training(self):
         # Checks that training worked, model trained and seed made a reproducible training.