boom boom

2020-03-24 00:27:01 -04:00 · 2020-03-24 00:26:06 -04:00 · 2020-03-24 00:23:47 -04:00 · 2020-03-24 00:06:37 -04:00 · 2020-03-24 00:03:38 -04:00 · 2020-03-23 23:54:44 -04:00
51 changed files with 732 additions and 1860 deletions
@@ -85,8 +85,6 @@ jobs:
        parallelism: 1
        steps:
            - checkout
-            # we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
-            - run: sudo pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
            - run: sudo pip install .[tf,torch,quality]
            - run: black --check --line-length 119 --target-version py35 examples templates tests src utils
            - run: isort --check-only --recursive examples templates tests src utils
@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'2.6.0'
+release = u'2.5.1'


 # -- General configuration ---------------------------------------------------
@@ -375,31 +375,24 @@ def print_summary_statistics(summary: MemorySummary):
        "\nLines by line memory consumption:\n"
        + "\n".join(
            f"{state.frame.filename}:{state.frame.line_number}: mem {state.cpu_gpu}: {state.frame.line_text}"
-            for state in summary.relative_mem_list
+            for state in summary.sequential
        )
    )
    print(
-        "\nLines with top memory increase:\n"
+        "\nLines with top memory consumption:\n"
        + "\n".join(
-            f"=> {state.frame.filename}:{state.frame.line_number}: mem {state.cpu_gpu_with_units}: {state.frame.line_text}"
-            for state in summary.relative_mem_sorted[:6]
+            f"=> {state.frame.filename}:{state.frame.line_number}: mem {state.cpu_gpu}: {state.frame.line_text}"
+            for state in summary.cumulative[:6]
        )
    )
    print(
-        "\nLines with lowest memory increase:\n"
+        "\nLines with lowest memory consumption:\n"
        + "\n".join(
-            f"=> {state.frame.filename}:{state.frame.line_number}: mem {state.cpu_gpu_with_units}: {state.frame.line_text}"
-            for state in summary.relative_mem_sorted[-6:]
+            f"=> {state.frame.filename}:{state.frame.line_number}: mem {state.cpu_gpu}: {state.frame.line_text}"
+            for state in summary.cumulative[-6:]
        )
    )
-    print(
-        "\nLines with peak memory used:\n"
-        + "\n".join(
-            f"=> {state.frame.filename}:{state.frame.line_number}: mem {state.cpu_gpu_with_units}: {state.frame.line_text}"
-            for state in summary.absolute_mem_sorted[:6]
-        )
-    )
-    print(f"\nTotal memory increase: {summary.relative_mem_total.cpu_gpu_with_units}")
+    print(f"\nTotal memory increase: {summary.total}")


 def _compute_pytorch(
@@ -460,7 +453,7 @@ def _compute_pytorch(
                            if verbose:
                                print_summary_statistics(summary)

-                            dictionary[model_name]["memory"][batch_size][slice_size] = summary.relative_mem_total.cpu_gpu_with_units
+                            dictionary[model_name]["memory"][batch_size][slice_size] = str(summary.total)
                        else:
                            dictionary[model_name]["memory"][batch_size][slice_size] = "N/A"

@@ -31,7 +31,6 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
-    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
    WEIGHTS_NAME,
    AdamW,
    AutoConfig,
@@ -39,6 +38,7 @@ from transformers import (
    AutoTokenizer,
    get_linear_schedule_with_warmup,
 )
+from transformers.modeling_auto import MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
 from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file


@@ -52,7 +52,6 @@ logger = logging.getLogger(__name__)

 MODEL_CONFIG_CLASSES = list(MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys())
 MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
-
 ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), ())

 TOKENIZER_ARGS = ["do_lower_case", "strip_accents", "keep_accents", "use_fast"]
@@ -13,11 +13,16 @@ from seqeval import metrics

 from transformers import (
    TF2_WEIGHTS_NAME,
-    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
-    AutoConfig,
-    AutoTokenizer,
+    BertConfig,
+    BertTokenizer,
+    DistilBertConfig,
+    DistilBertTokenizer,
    GradientAccumulator,
-    TFAutoModelForTokenClassification,
+    RobertaConfig,
+    RobertaTokenizer,
+    TFBertForTokenClassification,
+    TFDistilBertForTokenClassification,
+    TFRobertaForTokenClassification,
    create_optimizer,
 )
 from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file
@@ -29,17 +34,22 @@ except ImportError:
    from fastprogress.fastprogress import master_bar, progress_bar


-MODEL_CONFIG_CLASSES = list(TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys())
-MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+ALL_MODELS = sum(
+    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)), ()
+)

-ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)
+MODEL_CLASSES = {
+    "bert": (BertConfig, TFBertForTokenClassification, BertTokenizer),
+    "roberta": (RobertaConfig, TFRobertaForTokenClassification, RobertaTokenizer),
+    "distilbert": (DistilBertConfig, TFDistilBertForTokenClassification, DistilBertTokenizer),
+}


 flags.DEFINE_string(
    "data_dir", None, "The input data dir. Should contain the .conll files (or other data files) " "for the task."
 )

-flags.DEFINE_string("model_type", None, "Model type selected in the list: " + ", ".join(MODEL_TYPES))
+flags.DEFINE_string("model_type", None, "Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))

 flags.DEFINE_string(
    "model_name_or_path",
@@ -499,7 +509,8 @@ def main(_):
    labels = get_labels(args["labels"])
    num_labels = len(labels) + 1
    pad_token_label_id = 0
-    config = AutoConfig.from_pretrained(
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args["model_type"]]
+    config = config_class.from_pretrained(
        args["config_name"] if args["config_name"] else args["model_name_or_path"],
        num_labels=num_labels,
        cache_dir=args["cache_dir"] if args["cache_dir"] else None,
@@ -509,14 +520,14 @@ def main(_):

    # Training
    if args["do_train"]:
-        tokenizer = AutoTokenizer.from_pretrained(
+        tokenizer = tokenizer_class.from_pretrained(
            args["tokenizer_name"] if args["tokenizer_name"] else args["model_name_or_path"],
            do_lower_case=args["do_lower_case"],
            cache_dir=args["cache_dir"] if args["cache_dir"] else None,
        )

        with strategy.scope():
-            model = TFAutoModelForTokenClassification.from_pretrained(
+            model = model_class.from_pretrained(
                args["model_name_or_path"],
                from_pt=bool(".bin" in args["model_name_or_path"]),
                config=config,
@@ -551,7 +562,7 @@ def main(_):

    # Evaluation
    if args["do_eval"]:
-        tokenizer = AutoTokenizer.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
+        tokenizer = tokenizer_class.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
        checkpoints = []
        results = []

@@ -573,7 +584,7 @@ def main(_):
            global_step = checkpoint.split("-")[-1] if re.match(".*checkpoint-[0-9]", checkpoint) else "final"

            with strategy.scope():
-                model = TFAutoModelForTokenClassification.from_pretrained(checkpoint)
+                model = model_class.from_pretrained(checkpoint)

            y_true, y_pred, eval_loss = evaluate(
                args, strategy, model, tokenizer, labels, pad_token_label_id, mode="dev"
@@ -600,8 +611,8 @@ def main(_):
                        writer.write("\n")

    if args["do_predict"]:
-        tokenizer = AutoTokenizer.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
-        model = TFAutoModelForTokenClassification.from_pretrained(args["output_dir"])
+        tokenizer = tokenizer_class.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
+        model = model_class.from_pretrained(args["output_dir"])
        eval_batch_size = args["per_device_eval_batch_size"] * args["n_device"]
        predict_dataset, _ = load_and_cache_examples(
            args, tokenizer, labels, pad_token_label_id, eval_batch_size, mode="test"
@@ -30,12 +30,32 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
-    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    WEIGHTS_NAME,
    AdamW,
-    AutoConfig,
-    AutoModelForSequenceClassification,
-    AutoTokenizer,
+    AlbertConfig,
+    AlbertForSequenceClassification,
+    AlbertTokenizer,
+    BertConfig,
+    BertForSequenceClassification,
+    BertTokenizer,
+    DistilBertConfig,
+    DistilBertForSequenceClassification,
+    DistilBertTokenizer,
+    FlaubertConfig,
+    FlaubertForSequenceClassification,
+    FlaubertTokenizer,
+    RobertaConfig,
+    RobertaForSequenceClassification,
+    RobertaTokenizer,
+    XLMConfig,
+    XLMForSequenceClassification,
+    XLMRobertaConfig,
+    XLMRobertaForSequenceClassification,
+    XLMRobertaTokenizer,
+    XLMTokenizer,
+    XLNetConfig,
+    XLNetForSequenceClassification,
+    XLNetTokenizer,
    get_linear_schedule_with_warmup,
 )
 from transformers import glue_compute_metrics as compute_metrics
@@ -52,10 +72,33 @@ except ImportError:

 logger = logging.getLogger(__name__)

-MODEL_CONFIG_CLASSES = list(MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys())
-MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+ALL_MODELS = sum(
+    (
+        tuple(conf.pretrained_config_archive_map.keys())
+        for conf in (
+            BertConfig,
+            XLNetConfig,
+            XLMConfig,
+            RobertaConfig,
+            DistilBertConfig,
+            AlbertConfig,
+            XLMRobertaConfig,
+            FlaubertConfig,
+        )
+    ),
+    (),
+)

-ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)
+MODEL_CLASSES = {
+    "bert": (BertConfig, BertForSequenceClassification, BertTokenizer),
+    "xlnet": (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
+    "xlm": (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
+    "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
+    "distilbert": (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer),
+    "albert": (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer),
+    "xlmroberta": (XLMRobertaConfig, XLMRobertaForSequenceClassification, XLMRobertaTokenizer),
+    "flaubert": (FlaubertConfig, FlaubertForSequenceClassification, FlaubertTokenizer),
+}


 def set_seed(args):
@@ -399,7 +442,7 @@ def main():
        default=None,
        type=str,
        required=True,
-        help="Model type selected in the list: " + ", ".join(MODEL_TYPES),
+        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
    )
    parser.add_argument(
        "--model_name_or_path",
@@ -579,18 +622,19 @@ def main():
        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

    args.model_type = args.model_type.lower()
-    config = AutoConfig.from_pretrained(
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(
        args.config_name if args.config_name else args.model_name_or_path,
        num_labels=num_labels,
        finetuning_task=args.task_name,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    tokenizer = AutoTokenizer.from_pretrained(
+    tokenizer = tokenizer_class.from_pretrained(
        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
        do_lower_case=args.do_lower_case,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    model = AutoModelForSequenceClassification.from_pretrained(
+    model = model_class.from_pretrained(
        args.model_name_or_path,
        from_tf=bool(".ckpt" in args.model_name_or_path),
        config=config,
@@ -629,14 +673,14 @@ def main():
        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))

        # Load a trained model and vocabulary that you have fine-tuned
-        model = AutoModelForSequenceClassification.from_pretrained(args.output_dir)
-        tokenizer = AutoTokenizer.from_pretrained(args.output_dir)
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
        model.to(args.device)

    # Evaluation
    results = {}
    if args.do_eval and args.local_rank in [-1, 0]:
-        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
        checkpoints = [args.output_dir]
        if args.eval_all_checkpoints:
            checkpoints = list(
@@ -648,7 +692,7 @@ def main():
            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
            prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""

-            model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
+            model = model_class.from_pretrained(checkpoint)
            model.to(args.device)
            result = evaluate(args, model, tokenizer, prefix=prefix)
            result = dict((k + "_{}".format(global_step), v) for k, v in result.items())
@@ -38,14 +38,28 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
-    MODEL_WITH_LM_HEAD_MAPPING,
    WEIGHTS_NAME,
    AdamW,
-    AutoConfig,
-    AutoModelWithLMHead,
-    AutoTokenizer,
+    BertConfig,
+    BertForMaskedLM,
+    BertTokenizer,
+    CamembertConfig,
+    CamembertForMaskedLM,
+    CamembertTokenizer,
+    DistilBertConfig,
+    DistilBertForMaskedLM,
+    DistilBertTokenizer,
+    GPT2Config,
+    GPT2LMHeadModel,
+    GPT2Tokenizer,
+    OpenAIGPTConfig,
+    OpenAIGPTLMHeadModel,
+    OpenAIGPTTokenizer,
    PreTrainedModel,
    PreTrainedTokenizer,
+    RobertaConfig,
+    RobertaForMaskedLM,
+    RobertaTokenizer,
    get_linear_schedule_with_warmup,
 )

@@ -59,8 +73,14 @@ except ImportError:
 logger = logging.getLogger(__name__)


-MODEL_CONFIG_CLASSES = list(MODEL_WITH_LM_HEAD_MAPPING.keys())
-MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+MODEL_CLASSES = {
+    "gpt2": (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer),
+    "openai-gpt": (OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer),
+    "bert": (BertConfig, BertForMaskedLM, BertTokenizer),
+    "roberta": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
+    "distilbert": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
+    "camembert": (CamembertConfig, CamembertForMaskedLM, CamembertTokenizer),
+}


 class TextDataset(Dataset):
@@ -673,26 +693,23 @@ def main():
    if args.local_rank not in [-1, 0]:
        torch.distributed.barrier()  # Barrier to make sure only the first process in distributed training download model & vocab

+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+
    if args.config_name:
-        config = AutoConfig.from_pretrained(args.config_name, cache_dir=args.cache_dir)
+        config = config_class.from_pretrained(args.config_name, cache_dir=args.cache_dir)
    elif args.model_name_or_path:
-        config = AutoConfig.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
+        config = config_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
    else:
-        # When we release a pip version exposing CONFIG_MAPPING,
-        # we can do `config = CONFIG_MAPPING[args.model_type]()`.
-        raise ValueError(
-            "You are instantiating a new config instance from scratch. This is not supported, but you can do it from another script, save it,"
-            "and load it from here, using --config_name"
-        )
+        config = config_class()

    if args.tokenizer_name:
-        tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir)
    elif args.model_name_or_path:
-        tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
    else:
        raise ValueError(
-            "You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
-            "and load it from here, using --tokenizer_name"
+            "You are instantiating a new {} tokenizer. This is not supported, but you can do it from another script, save it,"
+            "and load it from here, using --tokenizer_name".format(tokenizer_class.__name__)
        )

    if args.block_size <= 0:
@@ -702,7 +719,7 @@ def main():
        args.block_size = min(args.block_size, tokenizer.max_len)

    if args.model_name_or_path:
-        model = AutoModelWithLMHead.from_pretrained(
+        model = model_class.from_pretrained(
            args.model_name_or_path,
            from_tf=bool(".ckpt" in args.model_name_or_path),
            config=config,
@@ -710,7 +727,7 @@ def main():
        )
    else:
        logger.info("Training new model from scratch")
-        model = AutoModelWithLMHead.from_config(config)
+        model = model_class(config=config)

    model.to(args.device)

@@ -751,8 +768,8 @@ def main():
        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))

        # Load a trained model and vocabulary that you have fine-tuned
-        model = AutoModelWithLMHead.from_pretrained(args.output_dir)
-        tokenizer = AutoTokenizer.from_pretrained(args.output_dir)
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
        model.to(args.device)

    # Evaluation
@@ -769,7 +786,7 @@ def main():
            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
            prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""

-            model = AutoModelWithLMHead.from_pretrained(checkpoint)
+            model = model_class.from_pretrained(checkpoint)
            model.to(args.device)
            result = evaluate(args, model, tokenizer, prefix=prefix)
            result = dict((k + "_{}".format(global_step), v) for k, v in result.items())
@@ -30,12 +30,29 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
-    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
    WEIGHTS_NAME,
    AdamW,
-    AutoConfig,
-    AutoModelForQuestionAnswering,
-    AutoTokenizer,
+    AlbertConfig,
+    AlbertForQuestionAnswering,
+    AlbertTokenizer,
+    BertConfig,
+    BertForQuestionAnswering,
+    BertTokenizer,
+    CamembertConfig,
+    CamembertForQuestionAnswering,
+    CamembertTokenizer,
+    DistilBertConfig,
+    DistilBertForQuestionAnswering,
+    DistilBertTokenizer,
+    RobertaConfig,
+    RobertaForQuestionAnswering,
+    RobertaTokenizer,
+    XLMConfig,
+    XLMForQuestionAnswering,
+    XLMTokenizer,
+    XLNetConfig,
+    XLNetForQuestionAnswering,
+    XLNetTokenizer,
    get_linear_schedule_with_warmup,
    squad_convert_examples_to_features,
 )
@@ -55,10 +72,23 @@ except ImportError:

 logger = logging.getLogger(__name__)

-MODEL_CONFIG_CLASSES = list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
-MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+ALL_MODELS = sum(
+    (
+        tuple(conf.pretrained_config_archive_map.keys())
+        for conf in (BertConfig, CamembertConfig, RobertaConfig, XLNetConfig, XLMConfig)
+    ),
+    (),
+)

-ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)
+MODEL_CLASSES = {
+    "bert": (BertConfig, BertForQuestionAnswering, BertTokenizer),
+    "camembert": (CamembertConfig, CamembertForQuestionAnswering, CamembertTokenizer),
+    "roberta": (RobertaConfig, RobertaForQuestionAnswering, RobertaTokenizer),
+    "xlnet": (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer),
+    "xlm": (XLMConfig, XLMForQuestionAnswering, XLMTokenizer),
+    "distilbert": (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer),
+    "albert": (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer),
+}


 def set_seed(args):
@@ -483,7 +513,7 @@ def main():
        default=None,
        type=str,
        required=True,
-        help="Model type selected in the list: " + ", ".join(MODEL_TYPES),
+        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
    )
    parser.add_argument(
        "--model_name_or_path",
@@ -727,16 +757,17 @@ def main():
        torch.distributed.barrier()

    args.model_type = args.model_type.lower()
-    config = AutoConfig.from_pretrained(
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(
        args.config_name if args.config_name else args.model_name_or_path,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    tokenizer = AutoTokenizer.from_pretrained(
+    tokenizer = tokenizer_class.from_pretrained(
        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
        do_lower_case=args.do_lower_case,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    model = AutoModelForQuestionAnswering.from_pretrained(
+    model = model_class.from_pretrained(
        args.model_name_or_path,
        from_tf=bool(".ckpt" in args.model_name_or_path),
        config=config,
@@ -786,8 +817,8 @@ def main():
        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))

        # Load a trained model and vocabulary that you have fine-tuned
-        model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)  # , force_download=True)
-        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        model = model_class.from_pretrained(args.output_dir)  # , force_download=True)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
        model.to(args.device)

    # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory
@@ -811,7 +842,7 @@ def main():
        for checkpoint in checkpoints:
            # Reload the model
            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
-            model = AutoModelForQuestionAnswering.from_pretrained(checkpoint)  # , force_download=True)
+            model = model_class.from_pretrained(checkpoint)  # , force_download=True)
            model.to(args.device)

            # Evaluate
@@ -14,19 +14,6 @@ python evaluate_cnn.py <path_to_test.source> cnn_test_summaries.txt
 ```
 the default batch size, 8, fits in 16GB GPU memory, but may need to be adjusted to fit your system.

-
-### Training
-
-
-
-After downloading the CNN and Daily Mail datasets, preprocess the dataset:
-```commandline
-git clone https://github.com/artmatsak/cnn-dailymail
-cd cnn-dailymail && python make_datafiles.py ../cnn/stories/ ../dailymail/stories/
-```
-
-Run the training script: `run_train.sh`
-
 ### Where is the code?
 The core model is in `src/transformers/modeling_bart.py`. This directory only contains examples.

@@ -46,7 +46,7 @@ def generate_summaries(lns, out_file, batch_size=8, device=DEFAULT_DEVICE):
 def _run_generate():
    parser = argparse.ArgumentParser()
    parser.add_argument(
-        "source_path", type=str, help="like cnn_dm/test.source",
+        "DATA_PATH", type=str, help="like cnn_dm/test.source",
    )
    parser.add_argument(
        "output_path", type=str, help="where to save summaries",
@@ -1,172 +0,0 @@
-import argparse
-import glob
-import logging
-import os
-import time
-
-import torch
-from torch.utils.data import DataLoader
-
-from transformer_base import BaseTransformer, add_generic_args, generic_train, get_linear_schedule_with_warmup
-from utils import SummarizationDataset
-
-
-logger = logging.getLogger(__name__)
-
-
-class BartSystem(BaseTransformer):
-
-    mode = "language-modeling"
-
-    def __init__(self, hparams):
-        super(BartSystem, self).__init__(hparams, num_labels=None, mode=self.mode)
-
-    def forward(
-        self, input_ids, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, lm_labels=None
-    ):
-        return self.model(
-            input_ids,
-            attention_mask=attention_mask,
-            decoder_input_ids=decoder_input_ids,
-            decoder_attention_mask=decoder_attention_mask,
-            lm_labels=lm_labels,
-        )
-
-    def _step(self, batch):
-        y = batch["target_ids"]
-        y_ids = y[:, :-1].contiguous()
-        lm_labels = y[:, 1:].clone()
-        lm_labels[y[:, 1:] == self.tokenizer.pad_token_id] = -100
-        outputs = self(
-            input_ids=batch["source_ids"],
-            attention_mask=batch["source_mask"],
-            decoder_input_ids=y_ids,
-            lm_labels=lm_labels,
-        )
-
-        loss = outputs[0]
-
-        return loss
-
-    def training_step(self, batch, batch_idx):
-        loss = self._step(batch)
-
-        tensorboard_logs = {"train_loss": loss}
-        return {"loss": loss, "log": tensorboard_logs}
-
-    def validation_step(self, batch, batch_idx):
-        loss = self._step(batch)
-        return {"val_loss": loss}
-
-    def validation_end(self, outputs):
-        avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean()
-        tensorboard_logs = {"val_loss": avg_loss}
-        return {"avg_val_loss": avg_loss, "log": tensorboard_logs}
-
-    def test_step(self, batch, batch_idx):
-        generated_ids = self.model.generate(
-            batch["source_ids"],
-            attention_mask=batch["source_mask"],
-            num_beams=1,
-            max_length=80,
-            repetition_penalty=2.5,
-            length_penalty=1.0,
-            early_stopping=True,
-        )
-        preds = [
-            self.tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True)
-            for g in generated_ids
-        ]
-        target = [
-            self.tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True)
-            for t in batch["target_ids"]
-        ]
-        loss = self._step(batch)
-
-        return {"val_loss": loss, "preds": preds, "target": target}
-
-    def test_end(self, outputs):
-        return self.validation_end(outputs)
-
-    def test_epoch_end(self, outputs):
-        output_test_predictions_file = os.path.join(self.hparams.output_dir, "test_predictions.txt")
-        output_test_targets_file = os.path.join(self.hparams.output_dir, "test_targets.txt")
-        # write predictions and targets for later rouge evaluation.
-        with open(output_test_predictions_file, "w+") as p_writer, open(output_test_targets_file, "w+") as t_writer:
-            for output_batch in outputs:
-                p_writer.writelines(s + "\n" for s in output_batch["preds"])
-                t_writer.writelines(s + "\n" for s in output_batch["target"])
-            p_writer.close()
-            t_writer.close()
-
-        return self.test_end(outputs)
-
-    def train_dataloader(self):
-        train_dataset = SummarizationDataset(
-            self.tokenizer, data_dir=self.hparams.data_dir, type_path="train", block_size=self.hparams.max_seq_length
-        )
-        dataloader = DataLoader(train_dataset, batch_size=self.hparams.train_batch_size)
-        t_total = (
-            (len(dataloader.dataset) // (self.hparams.train_batch_size * max(1, self.hparams.n_gpu)))
-            // self.hparams.gradient_accumulation_steps
-            * float(self.hparams.num_train_epochs)
-        )
-        scheduler = get_linear_schedule_with_warmup(
-            self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=t_total
-        )
-        self.lr_scheduler = scheduler
-        return dataloader
-
-    def val_dataloader(self):
-        val_dataset = SummarizationDataset(
-            self.tokenizer, data_dir=self.hparams.data_dir, type_path="val", block_size=self.hparams.max_seq_length
-        )
-        return DataLoader(val_dataset, batch_size=self.hparams.eval_batch_size)
-
-    def test_dataloader(self):
-        test_dataset = SummarizationDataset(
-            self.tokenizer, data_dir=self.hparams.data_dir, type_path="test", block_size=self.hparams.max_seq_length
-        )
-        return DataLoader(test_dataset, batch_size=self.hparams.eval_batch_size)
-
-    @staticmethod
-    def add_model_specific_args(parser, root_dir):
-        BaseTransformer.add_model_specific_args(parser, root_dir)
-        # Add BART specific options
-        parser.add_argument(
-            "--max_seq_length",
-            default=1024,
-            type=int,
-            help="The maximum total input sequence length after tokenization. Sequences longer "
-            "than this will be truncated, sequences shorter will be padded.",
-        )
-
-        parser.add_argument(
-            "--data_dir",
-            default=None,
-            type=str,
-            required=True,
-            help="The input data dir. Should contain the dataset files for the CNN/DM summarization task.",
-        )
-        return parser
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    add_generic_args(parser, os.getcwd())
-    parser = BartSystem.add_model_specific_args(parser, os.getcwd())
-    args = parser.parse_args()
-
-    # If output_dir not provided, a folder will be generated in pwd
-    if args.output_dir is None:
-        args.output_dir = os.path.join("./results", f"{args.task}_{args.model_type}_{time.strftime('%Y%m%d_%H%M%S')}",)
-        os.makedirs(args.output_dir)
-
-    model = BartSystem(args)
-    trainer = generic_train(model, args)
-
-    # Optionally, predict on dev set and write to output_dir
-    if args.do_predict:
-        checkpoints = list(sorted(glob.glob(os.path.join(args.output_dir, "checkpointepoch=*.ckpt"), recursive=True)))
-        BartSystem.load_from_checkpoint(checkpoints[-1])
-        trainer.test(model)
@@ -1,23 +0,0 @@
-# Install newest ptl.
-pip install -U git+http://github.com/PyTorchLightning/pytorch-lightning/
-
-
-export OUTPUT_DIR_NAME=bart_sum
-export CURRENT_DIR=${PWD}
-export OUTPUT_DIR=${CURRENT_DIR}/${OUTPUT_DIR_NAME}
-
-# Make output directory if it doesn't exist
-mkdir -p $OUTPUT_DIR
-
-# Add parent directory to python path to access transformer_base.py
-export PYTHONPATH="../../":"${PYTHONPATH}"
-
-python run_bart_sum.py \
--data_dir=./cnn-dailymail/cnn_dm \
--model_type=bart \
--model_name_or_path=bart-large \
--learning_rate=3e-5 \
--train_batch_size=4 \
--eval_batch_size=4 \
--output_dir=$OUTPUT_DIR \
--do_train
@@ -1,43 +0,0 @@
-import os
-
-from torch.utils.data import Dataset
-
-
-class SummarizationDataset(Dataset):
-    def __init__(self, tokenizer, data_dir="./cnn-dailymail/cnn_dm/", type_path="train", block_size=1024):
-        super(SummarizationDataset,).__init__()
-        self.tokenizer = tokenizer
-
-        self.source = []
-        self.target = []
-
-        print("loading " + type_path + " source.")
-
-        with open(os.path.join(data_dir, type_path + ".source"), "r") as f:
-            for text in f.readlines():  # each text is a line and a full story
-                tokenized = tokenizer.batch_encode_plus(
-                    [text], max_length=block_size, pad_to_max_length=True, return_tensors="pt"
-                )
-                self.source.append(tokenized)
-            f.close()
-
-        print("loading " + type_path + " target.")
-
-        with open(os.path.join(data_dir, type_path + ".target"), "r") as f:
-            for text in f.readlines():  # each text is a line and a summary
-                tokenized = tokenizer.batch_encode_plus(
-                    [text], max_length=56, pad_to_max_length=True, return_tensors="pt"
-                )
-                self.target.append(tokenized)
-            f.close()
-
-    def __len__(self):
-        return len(self.source)
-
-    def __getitem__(self, index):
-        source_ids = self.source[index]["input_ids"].squeeze()
-        target_ids = self.target[index]["input_ids"].squeeze()
-
-        src_mask = self.source[index]["attention_mask"].squeeze()  # might need to squeeze
-
-        return {"source_ids": source_ids, "source_mask": src_mask, "target_ids": target_ids}
@@ -53,9 +53,10 @@ class BaseTransformer(pl.LightningModule):
        super(BaseTransformer, self).__init__()
        self.hparams = hparams
        self.hparams.model_type = self.hparams.model_type.lower()
+
        config = AutoConfig.from_pretrained(
            self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path,
-            **({"num_labels": num_labels} if num_labels is not None else {}),
+            num_labels=num_labels,
            cache_dir=self.hparams.cache_dir if self.hparams.cache_dir else None,
        )
        tokenizer = AutoTokenizer.from_pretrained(
@@ -320,9 +320,7 @@ def convert_examples_to_features(
            else:
                text_b = example.question + " " + ending

-            inputs = tokenizer.encode_plus(
-                text_a, text_b, add_special_tokens=True, max_length=max_length, return_token_type_ids=True
-            )
+            inputs = tokenizer.encode_plus(text_a, text_b, add_special_tokens=True, max_length=max_length,)
            if "num_truncated_tokens" in inputs and inputs["num_truncated_tokens"] > 0:
                logger.info(
                    "Attention! you are cropping tokens (swag task is ok). "
@@ -1,76 +0,0 @@
---
-language: turkish
---
-
-# 🤗 + 📚 dbmdz Turkish BERT model
-
-In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
-Library open sources a cased model for Turkish 🎉
-
-# 🇹🇷 BERTurk
-
-BERTurk is a community-driven cased BERT model for Turkish.
-
-Some datasets used for pretraining and evaluation are contributed from the
-awesome Turkish NLP community, as well as the decision for the model name: BERTurk.
-
-## Stats
-
-The current version of the model is trained on a filtered and sentence
-segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/),
-a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a
-special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/).
-
-The final training corpus has a size of 35GB and 44,04,976,662 tokens.
-
-Thanks to Google's TensorFlow Research Cloud (TFRC) we could train a cased model
-on a TPU v3-8 for 2M steps.
-
-For this model we use a vocab size of 128k.
-
-## Model weights
-
-Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
-compatible weights are available. If you need access to TensorFlow checkpoints,
-please raise an issue!
-
-| Model                                | Downloads
-| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------
-| `dbmdz/bert-base-turkish-128k-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/vocab.txt)
-
-## Usage
-
-With Transformers >= 2.3 our BERTurk cased model can be loaded like:
-
-```python
-from transformers import AutoModel, AutoTokenizer
-
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-128k-cased")
-model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-128k-cased")
-```
-
-## Results
-
-For results on PoS tagging or NER tasks, please refer to
-[this repository](https://github.com/stefan-it/turkish-bert).
-
-# Huggingface model hub
-
-All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
-
-# Contact (Bugs, Feedback, Contribution and more)
-
-For questions about our BERT models just open an issue
-[here](https://github.com/dbmdz/berts/issues/new) 🤗
-
-# Acknowledgments
-
-Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us
-additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing
-us the Turkish NER dataset for evaluation.
-
-Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
-Thanks for providing access to the TFRC ❤️
-
-Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
-it is possible to download both cased and uncased models from their S3 storage 🤗
@@ -1,76 +0,0 @@
---
-language: turkish
---
-
-# 🤗 + 📚 dbmdz Turkish BERT model
-
-In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
-Library open sources an uncased model for Turkish 🎉
-
-# 🇹🇷 BERTurk
-
-BERTurk is a community-driven uncased BERT model for Turkish.
-
-Some datasets used for pretraining and evaluation are contributed from the
-awesome Turkish NLP community, as well as the decision for the model name: BERTurk.
-
-## Stats
-
-The current version of the model is trained on a filtered and sentence
-segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/),
-a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a
-special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/).
-
-The final training corpus has a size of 35GB and 44,04,976,662 tokens.
-
-Thanks to Google's TensorFlow Research Cloud (TFRC) we could train an uncased model
-on a TPU v3-8 for 2M steps.
-
-For this model we use a vocab size of 128k.
-
-## Model weights
-
-Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
-compatible weights are available. If you need access to TensorFlow checkpoints,
-please raise an issue!
-
-| Model                                  | Downloads
-| -------------------------------------- | ---------------------------------------------------------------------------------------------------------------
-| `dbmdz/bert-base-turkish-128k-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/vocab.txt)
-
-## Usage
-
-With Transformers >= 2.3 our BERTurk uncased model can be loaded like:
-
-```python
-from transformers import AutoModel, AutoTokenizer
-
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-128k-uncased")
-model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-128k-uncased")
-```
-
-## Results
-
-For results on PoS tagging or NER tasks, please refer to
-[this repository](https://github.com/stefan-it/turkish-bert).
-
-# Huggingface model hub
-
-All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
-
-# Contact (Bugs, Feedback, Contribution and more)
-
-For questions about our BERT models just open an issue
-[here](https://github.com/dbmdz/berts/issues/new) 🤗
-
-# Acknowledgments
-
-Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us
-additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing
-us the Turkish NER dataset for evaluation.
-
-Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
-Thanks for providing access to the TFRC ❤️
-
-Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
-it is possible to download both cased and uncased models from their S3 storage 🤗
@@ -1,74 +0,0 @@
---
-language: turkish
---
-
-# 🤗 + 📚 dbmdz Turkish BERT model
-
-In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
-Library open sources an uncased model for Turkish 🎉
-
-# 🇹🇷 BERTurk
-
-BERTurk is a community-driven uncased BERT model for Turkish.
-
-Some datasets used for pretraining and evaluation are contributed from the
-awesome Turkish NLP community, as well as the decision for the model name: BERTurk.
-
-## Stats
-
-The current version of the model is trained on a filtered and sentence
-segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/),
-a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a
-special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/).
-
-The final training corpus has a size of 35GB and 44,04,976,662 tokens.
-
-Thanks to Google's TensorFlow Research Cloud (TFRC) we could train an uncased model
-on a TPU v3-8 for 2M steps.
-
-## Model weights
-
-Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
-compatible weights are available. If you need access to TensorFlow checkpoints,
-please raise an issue!
-
-| Model                             | Downloads
-| --------------------------------- | ---------------------------------------------------------------------------------------------------------------
-| `dbmdz/bert-base-turkish-uncased`   | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/vocab.txt)
-
-## Usage
-
-With Transformers >= 2.3 our BERTurk uncased model can be loaded like:
-
-```python
-from transformers import AutoModel, AutoTokenizer
-
-tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-uncased")
-model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-uncased")
-```
-
-## Results
-
-For results on PoS tagging or NER tasks, please refer to
-[this repository](https://github.com/stefan-it/turkish-bert).
-
-# Huggingface model hub
-
-All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
-
-# Contact (Bugs, Feedback, Contribution and more)
-
-For questions about our BERT models just open an issue
-[here](https://github.com/dbmdz/berts/issues/new) 🤗
-
-# Acknowledgments
-
-Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us
-additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing
-us the Turkish NER dataset for evaluation.
-
-Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
-Thanks for providing access to the TFRC ❤️
-
-Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
-it is possible to download both cased and uncased models from their S3 storage 🤗
@@ -1,74 +0,0 @@
-## CS224n SQuAD2.0 Project Dataset
-The goal of this model is to save CS224n students GPU time when establising
-baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
-The training set used to fine-tune this model is the same as
-the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
-evaluation and model selection were performed using roughly half of the official
-dev set, 6078 examples, picked at random. The data files can be found at
-<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
-version. Given that the official SQuAD2.0 dev set contains the project's test
-set, students must make sure not to use the official SQuAD2.0 dev set in any way
-— including the use of models fine-tuned on the official SQuAD2.0, since they
-used the official SQuAD2.0 dev set for model selection.
-
-## Results
-```json
-{
-  "exact": 78.94044093451794,
-  "f1": 81.7724930324639,
-  "total": 6078,
-  "HasAns_exact": 76.28865979381443,
-  "HasAns_f1": 82.20385314478195,
-  "HasAns_total": 2910,
-  "NoAns_exact": 81.37626262626263,
-  "NoAns_f1": 81.37626262626263,
-  "NoAns_total": 3168,
-  "best_exact": 78.95689371503784,
-  "best_exact_thresh": 0.0,
-  "best_f1": 81.78894581298378,
-  "best_f1_thresh": 0.0
-}
-```
-
-## Notable Arguments
-```json
-{
-  "do_lower_case": true,
-  "doc_stride": 128,
-  "fp16": false,
-  "fp16_opt_level": "O1",
-  "gradient_accumulation_steps": 24,
-  "learning_rate": 3e-05,
-  "max_answer_length": 30,
-  "max_grad_norm": 1,
-  "max_query_length": 64,
-  "max_seq_length": 384,
-  "model_name_or_path": "albert-base-v2",
-  "model_type": "albert",
-  "num_train_epochs": 3,
-  "per_gpu_train_batch_size": 8,
-  "save_steps": 5000,
-  "seed": 42,
-  "train_batch_size": 8,
-  "version_2_with_negative": true,
-  "warmup_steps": 0,
-  "weight_decay": 0
-}
-```
-
-## Environment Setup
-```json
-{
-  "transformers": "2.5.1",
-  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
-  "python": "3.6.5=hc3d631a_2",
-  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
-  "gpu": "Tesla V100-SXM2-16GB"
-}
-```
-
-## Related Models
-* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
-* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
-* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
-* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)
@@ -1,74 +0,0 @@
-## CS224n SQuAD2.0 Project Dataset
-The goal of this model is to save CS224n students GPU time when establising
-baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
-The training set used to fine-tune this model is the same as
-the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
-evaluation and model selection were performed using roughly half of the official
-dev set, 6078 examples, picked at random. The data files can be found at
-<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
-version. Given that the official SQuAD2.0 dev set contains the project's test
-set, students must make sure not to use the official SQuAD2.0 dev set in any way
-— including the use of models fine-tuned on the official SQuAD2.0, since they
-used the official SQuAD2.0 dev set for model selection.
-
-## Results
-```json
-{
-  "exact": 79.2694965449161,
-  "f1": 82.50844352970152,
-  "total": 6078,
-  "HasAns_exact": 74.87972508591065,
-  "HasAns_f1": 81.64478342732858,
-  "HasAns_total": 2910,
-  "NoAns_exact": 83.30176767676768,
-  "NoAns_f1": 83.30176767676768,
-  "NoAns_total": 3168,
-  "best_exact": 79.2694965449161,
-  "best_exact_thresh": 0.0,
-  "best_f1": 82.50844352970155,
-  "best_f1_thresh": 0.0
-}
-```
-
-## Notable Arguments
-```json
-{
-  "do_lower_case": true,
-  "doc_stride": 128,
-  "fp16": false,
-  "fp16_opt_level": "O1",
-  "gradient_accumulation_steps": 1,
-  "learning_rate": 3e-05,
-  "max_answer_length": 30,
-  "max_grad_norm": 1,
-  "max_query_length": 64,
-  "max_seq_length": 384,
-  "model_name_or_path": "albert-large-v2",
-  "model_type": "albert",
-  "num_train_epochs": 5,
-  "per_gpu_train_batch_size": 8,
-  "save_steps": 5000,
-  "seed": 42,
-  "train_batch_size": 8,
-  "version_2_with_negative": true,
-  "warmup_steps": 0,
-  "weight_decay": 0
-}
-```
-
-## Environment Setup
-```json
-{
-  "transformers": "2.5.1",
-  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
-  "python": "3.6.5=hc3d631a_2",
-  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
-  "gpu": "Tesla V100-SXM2-16GB"
-}
-```
-
-## Related Models
-* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
-* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
-* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
-* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)
@@ -1,74 +0,0 @@
-## CS224n SQuAD2.0 Project Dataset
-The goal of this model is to save CS224n students GPU time when establising
-baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
-The training set used to fine-tune this model is the same as
-the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
-evaluation and model selection were performed using roughly half of the official
-dev set, 6078 examples, picked at random. The data files can be found at
-<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
-version. Given that the official SQuAD2.0 dev set contains the project's test
-set, students must make sure not to use the official SQuAD2.0 dev set in any way
-— including the use of models fine-tuned on the official SQuAD2.0, since they
-used the official SQuAD2.0 dev set for model selection.
-
-## Results
-```json
-{
-  "exact": 85.93287265547877,
-  "f1": 88.91258331187983,
-  "total": 6078,
-  "HasAns_exact": 84.36426116838489,
-  "HasAns_f1": 90.58786301361013,
-  "HasAns_total": 2910,
-  "NoAns_exact": 87.37373737373737,
-  "NoAns_f1": 87.37373737373737,
-  "NoAns_total": 3168,
-  "best_exact": 85.93287265547877,
-  "best_exact_thresh": 0.0,
-  "best_f1": 88.91258331187993,
-  "best_f1_thresh": 0.0
-}
-```
-
-## Notable Arguments
-```json
-{
-  "do_lower_case": true,
-  "doc_stride": 128,
-  "fp16": false,
-  "fp16_opt_level": "O1",
-  "gradient_accumulation_steps": 24,
-  "learning_rate": 3e-05,
-  "max_answer_length": 30,
-  "max_grad_norm": 1,
-  "max_query_length": 64,
-  "max_seq_length": 512,
-  "model_name_or_path": "albert-xxlarge-v1",
-  "model_type": "albert",
-  "num_train_epochs": 4,
-  "per_gpu_train_batch_size": 1,
-  "save_steps": 1000,
-  "seed": 42,
-  "train_batch_size": 1,
-  "version_2_with_negative": true,
-  "warmup_steps": 814,
-  "weight_decay": 0
-}
-```
-
-## Environment Setup
-```json
-{
-  "transformers": "2.5.1",
-  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
-  "python": "3.6.5=hc3d631a_2",
-  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
-  "gpu": "Tesla V100-SXM2-16GB"
-}
-```
-
-## Related Models
-* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
-* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
-* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
-* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)
@@ -1,74 +0,0 @@
-## CS224n SQuAD2.0 Project Dataset
-The goal of this model is to save CS224n students GPU time when establising
-baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
-The training set used to fine-tune this model is the same as
-the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
-evaluation and model selection were performed using roughly half of the official
-dev set, 6078 examples, picked at random. The data files can be found at
-<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
-version. Given that the official SQuAD2.0 dev set contains the project's test
-set, students must make sure not to use the official SQuAD2.0 dev set in any way
-— including the use of models fine-tuned on the official SQuAD2.0, since they
-used the official SQuAD2.0 dev set for model selection.
-
-## Results
-```json
-{
-  "exact": 65.16946363935504,
-  "f1": 67.87348075352251,
-  "total": 6078,
-  "HasAns_exact": 69.51890034364261,
-  "HasAns_f1": 75.16667217179045,
-  "HasAns_total": 2910,
-  "NoAns_exact": 61.17424242424242,
-  "NoAns_f1": 61.17424242424242,
-  "NoAns_total": 3168,
-  "best_exact": 65.16946363935504,
-  "best_exact_thresh": 0.0,
-  "best_f1": 67.87348075352243,
-  "best_f1_thresh": 0.0
-}
-```
-
-## Notable Arguments
-```json
-{
-  "do_lower_case": true,
-  "doc_stride": 128,
-  "fp16": false,
-  "fp16_opt_level": "O1",
-  "gradient_accumulation_steps": 24,
-  "learning_rate": 3e-05,
-  "max_answer_length": 30,
-  "max_grad_norm": 1,
-  "max_query_length": 64,
-  "max_seq_length": 384,
-  "model_name_or_path": "distilbert-base-uncased-distilled-squad",
-  "model_type": "distilbert",
-  "num_train_epochs": 4,
-  "per_gpu_train_batch_size": 32,
-  "save_steps": 5000,
-  "seed": 42,
-  "train_batch_size": 32,
-  "version_2_with_negative": true,
-  "warmup_steps": 0,
-  "weight_decay": 0
-}
-```
-
-## Environment Setup
-```json
-{
-  "transformers": "2.5.1",
-  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
-  "python": "3.6.5=hc3d631a_2",
-  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
-  "gpu": "Tesla V100-SXM2-16GB"
-}
-```
-
-## Related Models
-* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
-* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
-* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
-* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)
@@ -1,74 +0,0 @@
-## CS224n SQuAD2.0 Project Dataset
-The goal of this model is to save CS224n students GPU time when establising
-baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
-The training set used to fine-tune this model is the same as
-the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
-evaluation and model selection were performed using roughly half of the official
-dev set, 6078 examples, picked at random. The data files can be found at
-<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
-version. Given that the official SQuAD2.0 dev set contains the project's test
-set, students must make sure not to use the official SQuAD2.0 dev set in any way
-— including the use of models fine-tuned on the official SQuAD2.0, since they
-used the official SQuAD2.0 dev set for model selection.
-
-## Results
-```json
-{
-  "exact": 75.32082922013821,
-  "f1": 78.66699523704254,
-  "total": 6078,
-  "HasAns_exact": 74.84536082474227,
-  "HasAns_f1": 81.83436324767868,
-  "HasAns_total": 2910,
-  "NoAns_exact": 75.75757575757575,
-  "NoAns_f1": 75.75757575757575,
-  "NoAns_total": 3168,
-  "best_exact": 75.32082922013821,
-  "best_exact_thresh": 0.0,
-  "best_f1": 78.66699523704266,
-  "best_f1_thresh": 0.0
-}
-```
-
-## Notable Arguments
-```json
-{
-  "do_lower_case": true,
-  "doc_stride": 128,
-  "fp16": false,
-  "fp16_opt_level": "O1",
-  "gradient_accumulation_steps": 24,
-  "learning_rate": 3e-05,
-  "max_answer_length": 30,
-  "max_grad_norm": 1,
-  "max_query_length": 64,
-  "max_seq_length": 384,
-  "model_name_or_path": "roberta-base",
-  "model_type": "roberta",
-  "num_train_epochs": 4,
-  "per_gpu_train_batch_size": 16,
-  "save_steps": 5000,
-  "seed": 42,
-  "train_batch_size": 16,
-  "version_2_with_negative": true,
-  "warmup_steps": 0,
-  "weight_decay": 0
-}
-```
-
-## Environment Setup
-```json
-{
-  "transformers": "2.5.1",
-  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
-  "python": "3.6.5=hc3d631a_2",
-  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
-  "gpu": "Tesla V100-SXM2-16GB"
-}
-```
-
-## Related Models
-* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
-* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
-* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
-* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
@@ -1,37 +0,0 @@
-# BioBERT-NLI
-
-This is the model [BioBERT](https://github.com/dmis-lab/biobert) [1] fine-tuned on the [SNLI](https://nlp.stanford.edu/projects/snli/) and the [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) datasets using the [`sentence-transformers` library](https://github.com/UKPLab/sentence-transformers/) to produce universal sentence embeddings [2].
-
-The model uses the original BERT wordpiece vocabulary and was trained using the **average pooling strategy** and a **softmax loss**.
-
-**Base model**: `monologg/biobert_v1.1_pubmed` from HuggingFace's `AutoModel`.
-
-**Training time**: ~6 hours on the NVIDIA Tesla P100 GPU provided in Kaggle Notebooks.
-
-**Parameters**:
-
-| Parameter        | Value |
-|------------------|-------|
-| Batch size       | 64    |
-| Training steps   | 30000 |
-| Warmup steps     | 1450  |
-| Lowercasing      | False |
-| Max. Seq. Length | 128   |
-
-**Performances**: The performance was evaluated on the test portion of the [STS dataset](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) using Spearman rank correlation and compared to the performances of a general BERT base model obtained with the same procedure to verify their similarity.
-
-| Model                         | Score       |
-|-------------------------------|-------------|
-| `biobert-nli` (this)          | 73.40       |
-| `gsarti/scibert-nli`          | 74.50       |
-| `bert-base-nli-mean-tokens`[3]| 77.12       |
-
-An example usage for similarity-based scientific paper retrieval is provided in the [Covid Papers Browser](https://github.com/gsarti/covid-papers-browser) repository.
-
-**References:**
-
-[1] J. Lee et al, [BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://academic.oup.com/bioinformatics/article/36/4/1234/5566506)
-
-[2] A. Conneau et al., [Supervised Learning of Universal Sentence Representations from Natural Language Inference Data](https://www.aclweb.org/anthology/D17-1070/)
-
-[3] N. Reimers et I. Gurevych, [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://www.aclweb.org/anthology/D19-1410/)
@@ -2,28 +2,26 @@

 This is the model [SciBERT](https://github.com/allenai/scibert) [1] fine-tuned on the [SNLI](https://nlp.stanford.edu/projects/snli/) and the [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) datasets using the [`sentence-transformers` library](https://github.com/UKPLab/sentence-transformers/) to produce universal sentence embeddings [2].

-The model uses the original `scivocab` wordpiece vocabulary and was trained using the **average pooling strategy** and a **softmax loss**.
+The model uses the original `scivocab` wordpiece vocabulary and was trained using the **average pooling strategy** and a **softmax loss**. 

-**Base model**: `allenai/scibert-scivocab-cased` from HuggingFace's `AutoModel`.
-
-**Training time**: ~4 hours on the NVIDIA Tesla P100 GPU provided in Kaggle Notebooks.
+**Base model**: `allenai/scibert-scivocab-cased` from HuggingFace AutoModel

 **Parameters**:

-| Parameter        | Value |
-|------------------|-------|
-| Batch size       | 64    |
-| Training steps   | 20000 |
-| Warmup steps     | 1450  |
-| Lowercasing      | True  |
-| Max. Seq. Length | 128   |
+| Parameter      | Value |
+|----------------|-------|
+| Batch size     | 64    |
+| Training steps | 20000 |
+| Warmup steps   | 1450  |

 **Performances**: The performance was evaluated on the test portion of the [STS dataset](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) using Spearman rank correlation and compared to the performances of a general BERT base model obtained with the same procedure to verify their similarity.

-| Model                         | Score       |
-|-------------------------------|-------------|
-| `scibert-nli` (this)          | 74.50       |
-| `bert-base-nli-mean-tokens`[3]| 77.12       |
+
+| Model                       | Score       |
+|-----------------------------|-------------|
+| `scibert-nli` (ours)        | 74.50       |
+| `bert-base-nli-mean-tokens` | 77.12       |
+

 An example usage for similarity-based scientific paper retrieval is provided in the [Covid Papers Browser](https://github.com/gsarti/covid-papers-browser) repository.

@@ -32,5 +30,3 @@ An example usage for similarity-based scientific paper retrieval is provided in
 [1] I. Beltagy et al, [SciBERT: A Pretrained Language Model for Scientific Text](https://www.aclweb.org/anthology/D19-1371/)

 [2] A. Conneau et al., [Supervised Learning of Universal Sentence Representations from Natural Language Inference Data](https://www.aclweb.org/anthology/D17-1070/)
-
-[3] N. Reimers et I. Gurevych, [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://www.aclweb.org/anthology/D19-1410/)
@@ -32,54 +32,13 @@ Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess
 You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this:  

 ```python
-from transformers import AlbertTokenizer, BertModel
+from transformers import XLNetTokenizer, BertModel

 model = BertModel.from_pretrained('huseinzol05/bert-base-bahasa-cased')
-tokenizer = AlbertTokenizer.from_pretrained(
-    'huseinzol05/bert-base-bahasa-cased',
-    unk_token = '[UNK]',
-    pad_token = '[PAD]',
-    do_lower_case = False,
-)
+tokenizer = XLNetTokenizer.from_pretrained('huseinzol05/bert-base-bahasa-cased')
 ```

-We use [google/sentencepiece](https://github.com/google/sentencepiece) to train the tokenizer, so to use it, need to load from `AlbertTokenizer`.
-
-## Example using AutoModelWithLMHead
-
-```python
-from transformers import AlbertTokenizer, AutoModelWithLMHead, pipeline
-
-model = AutoModelWithLMHead.from_pretrained('huseinzol05/bert-base-bahasa-cased')
-tokenizer = AlbertTokenizer.from_pretrained(
-    'huseinzol05/bert-base-bahasa-cased',
-    unk_token = '[UNK]',
-    pad_token = '[PAD]',
-    do_lower_case = False,
-)
-fill_mask = pipeline('fill-mask', model = model, tokenizer = tokenizer)
-print(fill_mask('makan ayam dengan [MASK]'))
-```
-
-Output is,
-
-```text
-[{'sequence': '[CLS] makan ayam dengan rendang[SEP]',
-  'score': 0.10812027007341385,
-  'token': 2446},
- {'sequence': '[CLS] makan ayam dengan kicap[SEP]',
-  'score': 0.07653367519378662,
-  'token': 12928},
- {'sequence': '[CLS] makan ayam dengan nasi[SEP]',
-  'score': 0.06839974224567413,
-  'token': 450},
- {'sequence': '[CLS] makan ayam dengan ayam[SEP]',
-  'score': 0.059544261544942856,
-  'token': 638},
- {'sequence': '[CLS] makan ayam dengan sayur[SEP]',
-  'score': 0.05294966697692871,
-  'token': 1639}]
-```
+We use [google/sentencepiece](https://github.com/google/sentencepiece) to train the tokenizer, so to use it, need to load from `XLNetTokenizer`.

 ## Results

@@ -1,64 +0,0 @@
---
-language: malay
---
-
-# Bahasa XLNet Model
-
-Pretrained XLNet base language model for Malay and Indonesian. 
-
-## Pretraining Corpus
-
-`XLNET-base-bahasa-cased` model was pretrained on ~1.8 Billion words. We trained on both standard and social media language structures, and below is list of data we trained on,
-
-1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1).
-2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram).
-3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1).
-4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news).
-5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament).
-6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text).
-7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession).
-8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad).
-9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf).
-
-Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess).
-
-## Pretraining details
-
- This model was trained using zihangdai XLNet's github [repository](https://github.com/zihangdai/xlnet) on 3 Titan V100 32GB VRAM.
- All steps can reproduce from here, [Malaya/pretrained-model/xlnet](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/xlnet).
-
-## Load Pretrained Model
-
-You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this:  
-
-```python
-from transformers import XLNetTokenizer, XLNetModel
-
-model = XLNetModel.from_pretrained('huseinzol05/xlnet-base-bahasa-cased')
-tokenizer = XLNetTokenizer.from_pretrained(
-    'huseinzol05/xlnet-base-bahasa-cased', do_lower_case = False
-)
-```
-
-## Example using AutoModelWithLMHead
-
-```python
-from transformers import AlbertTokenizer, AutoModelWithLMHead, pipeline
-
-model = AutoModelWithLMHead.from_pretrained('huseinzol05/xlnet-base-bahasa-cased')
-tokenizer = XLNetTokenizer.from_pretrained(
-    'huseinzol05/xlnet-base-bahasa-cased', do_lower_case = False
-)
-fill_mask = pipeline('fill-mask', model = model, tokenizer = tokenizer)
-print(fill_mask('makan ayam dengan [MASK]'))
-```
-
-## Results
-
-For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models.
-
-## Acknowledgement
-
-Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train XLNet for Bahasa. 
-
-
@@ -1,60 +0,0 @@
---
-language: english
-thumbnail:
---
-
-# GPT-2 + CORD19 dataset : 🦠 ✍ ⚕
-
-**GPT-2** fine-tuned on **biorxiv_medrxiv** and **comm_use_subset files** from [CORD-19](https://www.kaggle.com/allen-institute-for-ai/CORD-19-research-challenge) dataset.
-
-
-## Datasets details:
-
-| Dataset                | # Files |
-| ---------------------- | ----- |
-| biorxiv_medrxiv        | 885  |
-| comm_use_subse         | 9K   |
-
-## Model training
-
-The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
-
-```bash
-
-export TRAIN_FILE=/path/to/dataset/train.txt
-
-python run_language_modeling.py \
-    --model_type gpt2 \
-    --model_name_or_path gpt2 \
-    --do_train \
-    --train_data_file $TRAIN_FILE \
-    --num_train_epochs 4 \
-    --output_dir model_output \
-    --overwrite_output_dir \
-    --save_steps 10000 \
-    --per_gpu_train_batch_size 3
-```
-
-<img alt="training loss" src="https://svgshare.com/i/JTf.svg' title='GTP-2-finetuned-CORDS19-loss" width="600" height="300" />
-
-## Model in action / Example of usage: ✒
-
-You can get the following script [here](https://github.com/huggingface/transformers/blob/master/examples/run_generation.py)
-
-```bash
-python run_generation.py \
-    --model_type gpt2 \
-    --model_name_or_path mrm8488/GPT-2-finetuned-CORD19 \
-    --length 200
-```
-```txt
-# Input: the effects of COVID-19 on the lungs
-# Output: === GENERATED SEQUENCE 1 ===
-the effects of COVID-19 on the lungs are currently debated (86). The role of this virus in the pathogenesis of pneumonia and lung cancer is still debated. MERS-CoV is also known to cause acute respiratory distress syndrome (87) and is associated with increased expression of pulmonary fibrosis markers (88). Thus, early airway inflammation may play an important role in the pathogenesis of coronavirus pneumonia and may contribute to the severe disease and/or mortality observed in coronavirus patients.
-Pneumonia is an acute, often fatal disease characterized by severe edema, leakage of oxygen and bronchiolar inflammation. Viruses include coronaviruses, and the role of oxygen depletion is complicated by lung injury and fibrosis in the lung, in addition to susceptibility to other lung diseases. The progression of the disease may be variable, depending on the lung injury, pathologic role, prognosis, and the immune status of the patient. Inflammatory responses to respiratory viruses cause various pathologies of the respiratory
-```
-
-
-> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
-
-> Made with <span style="color: #e25555;">&hearts;</span> in Spain
@@ -93,8 +93,8 @@ from transformers import pipeline

 qa_pipeline = pipeline(
    "question-answering",
-    model="mrm8488/xlm-multi-finetuned-xquadv1",
-    tokenizer="mrm8488/xlm-multi-finetuned-xquadv1"
+    model="mrm8488/bert-multi-uncased-finetuned-xquadv1",
+    tokenizer="bert-multi-uncased-finetuned-xquadv1"
 )

 # English
@@ -114,7 +114,7 @@ qa_pipeline({

 #Output: {'answer': 'работал в репозитории hugginface /','end': 76, 'score': 0.00012340750456964894, 'start': 42}
 ```
-Try it on a Colab (*Do not forget to change the model and tokenizer path in the Colab if necessary*):
+Try it on a Colab:

 <a href="https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/Try_mrm8488_xquad_finetuned_uncased_model.ipynb" target="_parent"><img src="https://camo.githubusercontent.com/52feade06f2fecbf006889a904d221e6a730c194/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Open In Colab" data-canonical-src="https://colab.research.google.com/assets/colab-badge.svg"></a>

@@ -1,24 +1,22 @@
-This model is [ALBERT base v2](https://huggingface.co/albert-base-v2) trained on SQuAD v2 as:
+This model is ALBERT base v2 trained on SQuAD v2 as:

 ```
-export SQUAD_DIR=../../squad2
-python3 run_squad.py 
-    --model_type albert 
-    --model_name_or_path albert-base-v2 
-    --do_train 
-    --do_eval 
-    --overwrite_cache 
-    --do_lower_case 
-    --version_2_with_negative 
-    --save_steps 100000 
-    --train_file $SQUAD_DIR/train-v2.0.json 
-    --predict_file $SQUAD_DIR/dev-v2.0.json 
-    --per_gpu_train_batch_size 8 
-    --num_train_epochs 3 
-    --learning_rate 3e-5 
-    --max_seq_length 384 
-    --doc_stride 128 
-    --output_dir ./tmp/albert_fine/
+python run_squad.py 
+--model_type albert 
+--model_name_or_path albert-base-v2 
+--do_train 
+--do_eval 
+--overwrite_cache 
+--do_lower_case 
+--version_2_with_negative 
+--train_file $SQUAD_DIR/train-v2.0.json 
+--predict_file $SQUAD_DIR/dev-v2.0.json 
+--per_gpu_train_batch_size 8 
+--num_train_epochs 3 
+--learning_rate 3e-5 
+--max_seq_length 384 
+--doc_stride 128 
+--output_dir ./tmp/albert_base_fine/
 ```

 Performance on a dev subset is close to the original paper:
@@ -1,24 +1,22 @@
-This model is [BERT base uncased](https://huggingface.co/bert-base-uncased) trained on SQuAD v2 as:
+This model is BERT base uncased trained on SQuAD v2 as:

 ```
-export SQUAD_DIR=../../squad2
-python3 run_squad.py 
-    --model_type bert 
-    --model_name_or_path bert-base-uncased 
-    --do_train 
-    --do_eval 
-    --overwrite_cache 
-    --do_lower_case 
-    --version_2_with_negative 
-    --save_steps 100000 
-    --train_file $SQUAD_DIR/train-v2.0.json 
-    --predict_file $SQUAD_DIR/dev-v2.0.json 
-    --per_gpu_train_batch_size 8 
-    --num_train_epochs 3 
-    --learning_rate 3e-5 
-    --max_seq_length 384 
-    --doc_stride 128 
-    --output_dir ./tmp/bert_fine_tuned/
+python run_squad.py 
+--model_type bert 
+--model_name_or_path bert-base-uncased
+--do_train 
+--do_eval 
+--overwrite_cache 
+--do_lower_case 
+--version_2_with_negative 
+--train_file $SQUAD_DIR/train-v2.0.json 
+--predict_file $SQUAD_DIR/dev-v2.0.json 
+--per_gpu_train_batch_size 8 
+--num_train_epochs 3 
+--learning_rate 3e-5 
+--max_seq_length 384 
+--doc_stride 128 
+--output_dir ./tmp/bert_base_fine/
 ```

 Performance on a dev subset is close to the original paper:
@@ -1,45 +0,0 @@
-This model is [Distilbert base uncased](https://huggingface.co/distilbert-base-uncased) trained on SQuAD v2 as:
-
-```
-export SQUAD_DIR=../../squad2
-python3 run_squad.py 
-    --model_type distilbert 
-    --model_name_or_path distilbert-base-uncased
-    --do_train 
-    --do_eval 
-    --overwrite_cache 
-    --do_lower_case 
-    --version_2_with_negative 
-    --save_steps 100000 
-    --train_file $SQUAD_DIR/train-v2.0.json 
-    --predict_file $SQUAD_DIR/dev-v2.0.json 
-    --per_gpu_train_batch_size 8 
-    --num_train_epochs 3 
-    --learning_rate 3e-5 
-    --max_seq_length 384 
-    --doc_stride 128 
-    --output_dir ./tmp/distilbert_fine_tuned/
-```
-
-Performance on a dev subset is close to the original paper:
-
-```
-Results: 
-{
-    'exact': 64.88976637051661, 
-    'f1': 68.1776176526635, 
-    'total': 6078, 
-    'HasAns_exact': 69.7594501718213, 
-    'HasAns_f1': 76.62665295288285, 
-    'HasAns_total': 2910, 
-    'NoAns_exact': 60.416666666666664, 
-    'NoAns_f1': 60.416666666666664, 
-    'NoAns_total': 3168, 
-    'best_exact': 64.88976637051661, 
-    'best_exact_thresh': 0.0, 
-    'best_f1': 68.17761765266337, 
-    'best_f1_thresh': 0.0
-}
-```
-
-We are hopeful this might save you time, energy, and compute. Cheers!
@@ -1,44 +0,0 @@
-This model is [Distilroberta base](https://huggingface.co/distilroberta-base) trained on SQuAD v2 as:
-
-```
-export SQUAD_DIR=../../squad2
-python3 run_squad.py 
-    --model_type robberta 
-    --model_name_or_path distilroberta-base 
-    --do_train 
-    --do_eval 
-    --overwrite_cache 
-    --do_lower_case 
-    --version_2_with_negative 
-    --save_steps 100000 
-    --train_file $SQUAD_DIR/train-v2.0.json 
-    --predict_file $SQUAD_DIR/dev-v2.0.json 
-    --per_gpu_train_batch_size 8 
-    --num_train_epochs 3 
-    --learning_rate 3e-5 
-    --max_seq_length 384 
-    --doc_stride 128 
-    --output_dir ./tmp/distilroberta_fine_tuned/
-```
-
-Performance on a dev subset is close to the original paper:
-
-```
-Results: 
-{
-    'exact': 70.9279368213228, 
-    'f1': 74.60439802429168, 
-    'total': 6078, 
-    'HasAns_exact': 67.62886597938144, 
-    'HasAns_f1': 75.30774267754136, 
-    'HasAns_total': 2910, 
-    'NoAns_exact': 73.95833333333333, 
-    'NoAns_f1': 73.95833333333333, 'NoAns_total': 3168, 
-    'best_exact': 70.94438960184272, 
-    'best_exact_thresh': 0.0, 
-    'best_f1': 74.62085080481161, 
-    'best_f1_thresh': 0.0
-}
-```
-
-We are hopeful this might save you time, energy, and compute. Cheers!
@@ -64,7 +64,7 @@ if stale_egg_info.exists():
 extras = {}

 extras["mecab"] = ["mecab-python3"]
-extras["sklearn"] = ["scikit-learn"]
+extras["sklearn"] = ["scikit-learn==0.22.1"]
 extras["tf"] = ["tensorflow"]
 extras["tf-cpu"] = ["tensorflow-cpu"]
 extras["torch"] = ["torch"]
@@ -76,14 +76,14 @@ extras["testing"] = ["pytest", "pytest-xdist"]
 extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme"]
 extras["quality"] = [
    "black",
-    "isort",
+    "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
    "flake8",
 ]
 extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3", "scikit-learn", "tensorflow", "torch"]

 setup(
    name="transformers",
-    version="2.6.0",
+    version="2.5.1",
    author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
    author_email="thomas@huggingface.co",
    description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
@@ -97,8 +97,6 @@ setup(
    install_requires=[
        "numpy",
        "tokenizers == 0.5.2",
-        # dataclasses for Python versions that don't have it
-        "dataclasses;python_version<'3.7'",
        # accessing files from S3 directly
        "boto3",
        # filesystem locks e.g. to prevent parallel downloads
@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.

-__version__ = "2.6.0"
+__version__ = "2.5.1"

 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -22,15 +22,17 @@ import logging
 # Benchmarking
 from .benchmark_utils import (
    Frame,
+    Memory,
    MemoryState,
    MemorySummary,
    MemoryTrace,
+    UsedMemoryState,
    bytes_to_human_readable,
    start_memory_tracing,
    stop_memory_tracing,
 )
 from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
-from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, AutoConfig
+from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, AutoConfig
 from .configuration_bart import BartConfig
 from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig
 from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
@@ -114,11 +116,10 @@ from .pipelines import (
    SummarizationPipeline,
    TextClassificationPipeline,
    TokenClassificationPipeline,
-    TranslationPipeline,
    pipeline,
 )
 from .tokenization_albert import AlbertTokenizer
-from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
+from .tokenization_auto import AutoTokenizer
 from .tokenization_bart import BartTokenizer
 from .tokenization_bert import BasicTokenizer, BertTokenizer, BertTokenizerFast, WordpieceTokenizer
 from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer
@@ -157,12 +158,6 @@ if is_torch_available():
        AutoModelWithLMHead,
        AutoModelForTokenClassification,
        ALL_PRETRAINED_MODEL_ARCHIVE_MAP,
-        MODEL_MAPPING,
-        MODEL_FOR_PRETRAINING_MAPPING,
-        MODEL_WITH_LM_HEAD_MAPPING,
-        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
-        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
-        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
    )

    from .modeling_bert import (
@@ -322,12 +317,6 @@ if is_tf_available():
        TFAutoModelWithLMHead,
        TFAutoModelForTokenClassification,
        TF_ALL_PRETRAINED_MODEL_ARCHIVE_MAP,
-        TF_MODEL_MAPPING,
-        TF_MODEL_FOR_PRETRAINING_MAPPING,
-        TF_MODEL_WITH_LM_HEAD_MAPPING,
-        TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
-        TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
-        TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
    )

    from .modeling_tf_bert import (
@@ -0,0 +1,67 @@
+from transformers import *
+import torch
+DEFAULT_DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+def runner(source_path, out_file, batch_size=8, device=DEFAULT_DEVICE, prof_generate=False):
+
+    tokenizer = BartTokenizer.from_pretrained('bart-large')
+    lns = [" " + x.rstrip() for x in open(source_path).readlines()][:batch_size]
+
+    dct = tokenizer.batch_encode_plus(lns, max_length=1024, return_tensors="pt", pad_to_max_length=True)
+    ids = dct['input_ids'].to(DEFAULT_DEVICE)
+    msk = dct['attention_mask'].to(DEFAULT_DEVICE)
+    model = BartForConditionalGeneration.from_pretrained('bart-large-cnn', output_past=prof_generate).to(DEFAULT_DEVICE)
+    model.log_mem('starting')
+    if prof_generate:
+
+        summaries = model.generate(
+            input_ids=ids,
+            attention_mask=msk,
+            num_beams=4,
+            length_penalty=2.0,
+            max_length=140 + 2,  # +2 from original because we start at step=1 and stop before max_length
+            min_length=55 + 1,  # +1 from original because we start at step=1
+            no_repeat_ngram_size=3,
+            early_stopping=True,
+            do_sample=False,
+            decoder_start_token_id=model.config.eos_token_ids[0],
+        )
+        model.log_mem('done')
+        dec = [tokenizer.decode(s) for s in summaries]
+        print(dec[0])
+    else:
+        #model.decoder.generation_mode = Fals
+        with torch.no_grad():
+            model(
+                input_ids=ids,
+                attention_mask=msk,
+            )
+
+    log_df = model.combine_logs()
+    log_df.to_csv(out_file)
+
+
+import argparse
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "output_path", type=str, help="where to save summaries",
+    )
+    parser.add_argument(
+        "--DATA_PATH", type=str, default="/home/shleifer/transformers_fork/notebooks/test.source",
+        help="like cnn_dm/test.source", required=False
+    )
+    parser.add_argument(
+        "--device", type=str, required=False, default=DEFAULT_DEVICE, help="cuda, cuda:1, cpu etc.",
+    )
+    parser.add_argument(
+        "--bs", type=int, default=8, required=False, help="batch size: how many to summarize at a time",
+    )
+    parser.add_argument(
+        "--do-generate", action='store_true', required=False, help="batch size: how many to summarize at a time",
+    )
+    args = parser.parse_args()
+    runner(args.source_path, args.output_path, batch_size=args.bs, device=args.device, prof_generate=args.do_generate)
+
+
+
@@ -9,8 +9,7 @@ import logging
 import os
 import sys
 from collections import defaultdict
-from dataclasses import dataclass
-from typing import Iterable, List, Optional, Union
+from typing import Iterable, List, NamedTuple, Optional, Union

 from .file_utils import is_tf_available, is_torch_available

@@ -32,14 +31,14 @@ def is_memory_tracing_enabled():
    return _is_memory_tracing_enabled


-@dataclass(frozen=True)
-class Frame:
-    """ `Frame` is used to gather the current frame state:
-        - 'filename' (string): Name of the file currently executed
-        - 'module' (string): Name of the module currently executed
-        - 'line_number' (int): Number of the line currently executed
-        - 'event' (string): Event that triggered the tracing (default will be "line")
-        - 'line_text' (string): Text of the line in the python script
+class Frame(NamedTuple):
+    """ `Frame` is a NamedTuple used to gather the current frame state.
+            `Frame` has the following fields:
+            - 'filename' (string): Name of the file currently executed
+            - 'module' (string): Name of the module currently executed
+            - 'line_number' (int): Number of the line currently executed
+            - 'event' (string): Event that triggered the tracing (default will be "line")
+            - 'line_text' (string): Text of the line in the python script
    """

    filename: str
@@ -49,67 +48,61 @@ class Frame:
    line_text: str


-@dataclass
-class MemoryState:
-    """ `MemoryState` lists frame + CPU/GPU memory:
-        - `cpu`: CPU memory at or before the current frame as a `Memory` named tuple
-        - `gpu`: GPU memory at or before during the current frame as a `Memory` named tuple
-        - `frame` (`Frame`): the current frame
-        Also provide a few properties:
-            `cpu_gpu`: sum of the CPU + GPU memory at or before during the current frame as a `Memory` named tuple
-            `cpu_with_units`: CPU memory as a human readable string
-            `gpu_with_units`: GPU memory as a human readable string
-            `cpu_gpu_with_units`: CPU+GPU memory as a human readable string
+class UsedMemoryState(NamedTuple):
+    """ `UsedMemoryState` are named tuples with the following fields:
+        - 'frame': a `Frame` namedtuple (see below) storing information on the current tracing frame (current file, location in current file)
+        - 'cpu_memory': CPU RSS memory state *before* executing the line
+        - 'gpu_memory': GPU used memory *before* executing the line (sum for all GPUs or for only `gpus_to_trace` if provided)
    """

-    cpu: int
-    gpu: int
-    frame: Optional[Frame] = None
-
-    @property
-    def cpu_gpu(self) -> int:
-        return self.cpu + self.gpu
-
-    @property
-    def cpu_with_units(self) -> str:
-        return bytes_to_human_readable(self.cpu)
-
-    @property
-    def gpu_with_units(self) -> str:
-        return bytes_to_human_readable(self.gpu)
-
-    @property
-    def cpu_gpu_with_units(self) -> str:
-        return bytes_to_human_readable(self.cpu + self.gpu)
+    frame: Frame
+    cpu_memory: int
+    gpu_memory: int


-@dataclass
-class MemorySummary:
+class Memory(NamedTuple):
+    """ `Memory` NamedTuple have a single field `bytes` and
+        you can get a human readable string of the number of bytes by calling `__repr__`
+            - `byte` (integer): number of bytes,
+    """
+
+    bytes: int
+
+    def __repr__(self) -> str:
+        return bytes_to_human_readable(self.bytes)
+
+
+class MemoryState(NamedTuple):
+    """ `MemoryState` are namedtuples listing frame + CPU/GPU memory with the following fields:
+        - `frame` (`Frame`): the current frame (see above)
+        - `cpu`: CPU memory consumed at during the current frame as a `Memory` named tuple
+        - `gpu`: GPU memory consumed at during the current frame as a `Memory` named tuple
+        - `cpu_gpu`: CPU + GPU memory consumed at during the current frame as a `Memory` named tuple
+    """
+
+    frame: Frame
+    cpu: Memory
+    gpu: Memory
+    cpu_gpu: Memory
+
+
+class MemorySummary(NamedTuple):
    """ `MemorySummary` namedtuple otherwise with the fields:
-        - `absolute_mem_list`: total CPU/GPU memory used at each line
-            a list of `MemoryState` namedtuple (see below)
-        - `relative_mem_list`: relative difference in CPU/GPU memory at each line
-            a list of `MemoryState` namedtuple (see below) computed from the provided `memory_trace`
+        - `sequential`: a list of `MemoryState` namedtuple (see below) computed from the provided `memory_trace`
            by substracting the memory after executing each line from the memory before executing said line.
-        - `absolute_mem_sorted`: total CPU/GPU memory used at each line sorted by lines (max among all the times a line is executed)
-            a list of `MemoryState` namedtuple (see below)
-            The list is sorted from the frame with the largest memory consumption to the frame with the smallest (can be negative if memory is released)
-        - `relative_mem_sorted`: relative difference in CPU/GPU memory sorted by lines (cumulative increase among all the times a line is executed)
-            a list of `MemoryState` namedtuple (see below) with cumulative increase in memory for each line
+        - `cumulative`: a list of `MemoryState` namedtuple (see below) with cumulative increase in memory for each line
            obtained by summing repeted memory increase for a line if it's executed several times.
            The list is sorted from the frame with the largest memory consumption to the frame with the smallest (can be negative if memory is released)
        - `total`: total memory increase during the full tracing as a `Memory` named tuple (see below).
            Line with memory release (negative consumption) are ignored if `ignore_released_memory` is `True` (default).
    """

-    absolute_mem_list: List[MemoryState]
-    relative_mem_list: List[MemoryState]
-    absolute_mem_sorted: List[MemoryState]
-    relative_mem_sorted: List[MemoryState]
-    relative_mem_total: MemoryState
+    sequential: List[MemoryState]
+    cumulative: List[MemoryState]
+    total: Memory


-MemoryTrace = List[MemoryState]
+MemoryTrace = List[UsedMemoryState]


 def start_memory_tracing(
@@ -136,14 +129,13 @@ def start_memory_tracing(
            - `gpus_to_trace`: (optional list, default None) list of GPUs to trace. Default to tracing all GPUs

        Return:
-            - `memory_trace` is a list of `MemoryState` for each event (default each line of the traced script).
-                - `MemoryState` are simple classes with the following attributes:
+            - `memory_trace` is a list of `UsedMemoryState` for each event (default each line of the traced script).
+                - `UsedMemoryState` are named tuples with the following fields:
                    - 'frame': a `Frame` namedtuple (see below) storing information on the current tracing frame (current file, location in current file)
-                    - 'cpu': CPU RSS memory state *before* executing the line
-                    - 'gpu': GPU used memory *before* executing the line (sum for all GPUs or for only `gpus_to_trace` if provided)
-                    - `cpu_gpu`: CPU + GPU memory *before* executing the line
+                    - 'cpu_memory': CPU RSS memory state *before* executing the line
+                    - 'gpu_memory': GPU used memory *before* executing the line (sum for all GPUs or for only `gpus_to_trace` if provided)

-        `Frame` is a namedtuple used by `MemoryState` to list the current frame state.
+        `Frame` is a namedtuple used by `UsedMemoryState` to list the current frame state.
            `Frame` has the following fields:
            - 'filename' (string): Name of the file currently executed
            - 'module' (string): Name of the module currently executed
@@ -248,7 +240,7 @@ def start_memory_tracing(
                gpu_mem += meminfo.used
            py3nvml.nvmlShutdown()

-        mem_state = MemoryState(cpu_mem, gpu_mem, traced_state)
+        mem_state = UsedMemoryState(traced_state, cpu_mem, gpu_mem)
        memory_trace.append(mem_state)

        return traceit
@@ -302,39 +294,40 @@ def stop_memory_tracing(
    _is_memory_tracing_enabled = False

    if memory_trace is not None and len(memory_trace) > 1:
-        init_mem = memory_trace[0]
-        absolute_mem_list = []
-        relative_mem_list = []
-        absolute_mem_dict = defaultdict(lambda: [])
-        relative_mem_dict = defaultdict(lambda: [])
-        for line, next_line in zip(memory_trace[:-1], memory_trace[1:]):
-            absolute_mem = MemoryState(line.cpu - init_mem.cpu, line.gpu - init_mem.gpu, line.frame)
-            relative_mem = MemoryState(next_line.cpu - line.cpu, next_line.gpu - line.gpu, line.frame)
-            absolute_mem_list.append(absolute_mem)
-            relative_mem_list.append(relative_mem)
-            absolute_mem_dict[line.frame].append(absolute_mem)
-            relative_mem_dict[line.frame].append(relative_mem)
+        memory_diff_trace = []
+        cumulative_memory_dict = defaultdict(lambda: [0, 0, 0])
+        for (frame, cpu_mem, gpu_mem), (next_frame, next_cpu_mem, next_gpu_mem) in zip(
+            memory_trace[:-1], memory_trace[1:]
+        ):
+            cpu_mem_inc = next_cpu_mem - cpu_mem
+            gpu_mem_inc = next_gpu_mem - gpu_mem
+            cpu_gpu_mem_inc = cpu_mem_inc + gpu_mem_inc
+            memory_diff_trace.append(
+                MemoryState(
+                    frame=frame, cpu=Memory(cpu_mem_inc), gpu=Memory(gpu_mem_inc), cpu_gpu=Memory(cpu_gpu_mem_inc),
+                )
+            )
+            cumulative_memory_dict[frame][0] += cpu_mem_inc
+            cumulative_memory_dict[frame][1] += gpu_mem_inc
+            cumulative_memory_dict[frame][2] += cpu_gpu_mem_inc

-        relative_mem_sorted = list(MemoryState(sum(v.cpu for v in l), sum(v.gpu for v in l), k) for k, l in relative_mem_dict.items())
-        absolute_mem_sorted = list(MemoryState(max(v.cpu for v in l), max(v.gpu for v in l), k) for k, l in absolute_mem_dict.items())
-
-        relative_mem_sorted = sorted(relative_mem_sorted, key=lambda x: x.cpu_gpu, reverse=True)
-        absolute_mem_sorted = sorted(absolute_mem_sorted, key=lambda x: x.cpu_gpu, reverse=True)
-
-        to_sum = (
-            filter(lambda m: m.cpu_gpu > 0, relative_mem_list)
-            if ignore_released_memory
-            else relative_mem_list
-        )
-        relative_mem_total = MemoryState(sum(v.cpu for v in to_sum), sum(v.gpu for v in to_sum))
-        return MemorySummary(
-            absolute_mem_list=absolute_mem_list,
-            relative_mem_list=relative_mem_list,
-            relative_mem_sorted=relative_mem_sorted,
-            absolute_mem_sorted=absolute_mem_sorted,
-            relative_mem_total=relative_mem_total,
+        cumulative_memory = sorted(
+            list(cumulative_memory_dict.items()), key=lambda x: x[1][2], reverse=True
+        )  # order by the total CPU + GPU memory increase
+        cumulative_memory = list(
+            MemoryState(
+                frame=frame, cpu=Memory(cpu_mem_inc), gpu=Memory(gpu_mem_inc), cpu_gpu=Memory(cpu_gpu_mem_inc),
+            )
+            for frame, (cpu_mem_inc, gpu_mem_inc, cpu_gpu_mem_inc) in cumulative_memory
        )

+        if ignore_released_memory:
+            total_memory = sum(max(0, step_trace.cpu_gpu.bytes) for step_trace in memory_diff_trace)
+        else:
+            total_memory = sum(step_trace.cpu_gpu.bytes for step_trace in memory_diff_trace)
+        total_memory = Memory(total_memory)
+        return MemorySummary(sequential=memory_diff_trace, cumulative=cumulative_memory, total=total_memory)
+
    return None


@@ -78,6 +78,9 @@ class PretrainedConfig(object):
        self.top_k = kwargs.pop("top_k", 50)
        self.top_p = kwargs.pop("top_p", 1.0)
        self.repetition_penalty = kwargs.pop("repetition_penalty", 1.0)
+        self.bos_token_id = kwargs.pop("bos_token_id", None)
+        self.pad_token_id = kwargs.pop("pad_token_id", None)
+        self.eos_token_id = kwargs.pop("eos_token_id", None)
        self.length_penalty = kwargs.pop("length_penalty", 1.0)
        self.no_repeat_ngram_size = kwargs.pop("no_repeat_ngram_size", 0)
        self.num_return_sequences = kwargs.pop("num_return_sequences", 1)
@@ -91,16 +94,6 @@ class PretrainedConfig(object):
        self.label2id = kwargs.pop("label2id", dict(zip(self.id2label.values(), self.id2label.keys())))
        self.label2id = dict((key, int(value)) for key, value in self.label2id.items())

-        # Tokenizer arguments TODO: eventually tokenizer and models should share the same config
-        self.prefix = kwargs.pop("prefix", None)
-        self.bos_token_id = kwargs.pop("bos_token_id", None)
-        self.pad_token_id = kwargs.pop("pad_token_id", None)
-        self.eos_token_id = kwargs.pop("eos_token_id", None)
-        self.decoder_start_token_id = kwargs.pop("decoder_start_token_id", None)
-
-        # task specific arguments
-        self.task_specific_params = kwargs.pop("task_specific_params", None)
-
        # Additional attributes without default values
        for key, value in kwargs.items():
            try:
@@ -380,14 +373,3 @@ class PretrainedConfig(object):
        """
        with open(json_file_path, "w", encoding="utf-8") as writer:
            writer.write(self.to_json_string())
-
-    def update(self, config_dict: Dict):
-        """
-        Updates attributes of this class
-        with attributes from `config_dict`.
-
-        Args:
-            :obj:`Dict[str, any]`: Dictionary of attributes that shall be updated for this class.
-        """
-        for key, value in config_dict.items():
-            setattr(self, key, value)
@@ -139,7 +139,6 @@ def squad_convert_example_to_features(example, max_seq_length, doc_stride, max_q
            pad_to_max_length=True,
            stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens,
            truncation_strategy="only_second" if tokenizer.padding_side == "right" else "only_first",
-            return_token_type_ids=True,
        )

        paragraph_len = min(
@@ -16,11 +16,8 @@

 import copy
 import csv
-import dataclasses
 import json
 import logging
-from dataclasses import dataclass
-from typing import Optional

 from ...file_utils import is_tf_available, is_torch_available

@@ -28,8 +25,7 @@ from ...file_utils import is_tf_available, is_torch_available
 logger = logging.getLogger(__name__)


-@dataclass(frozen=True)
-class InputExample:
+class InputExample(object):
    """
    A single training/test example for simple sequence classification.

@@ -43,14 +39,23 @@ class InputExample:
            specified for train and dev examples, but not for test examples.
    """

-    guid: str
-    text_a: str
-    text_b: Optional[str] = None
-    label: Optional[str] = None
+    def __init__(self, guid, text_a, text_b=None, label=None):
+        self.guid = guid
+        self.text_a = text_a
+        self.text_b = text_b
+        self.label = label
+
+    def __repr__(self):
+        return str(self.to_json_string())
+
+    def to_dict(self):
+        """Serializes this instance to a Python dictionary."""
+        output = copy.deepcopy(self.__dict__)
+        return output

    def to_json_string(self):
        """Serializes this instance to a JSON string."""
-        return json.dumps(dataclasses.asdict(self), indent=2, sort_keys=True) + "\n"
+        return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n"


 class InputFeatures(object):
@@ -769,7 +769,9 @@ class BartModel(PretrainedBartModel):
        assert decoder_input_ids is not None
        if encoder_outputs is None:
            encoder_outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
+
        assert isinstance(encoder_outputs, tuple)
+
        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
        decoder_outputs = self.decoder(
            decoder_input_ids,
@@ -610,9 +610,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
        num_return_sequences = (
            num_return_sequences if num_return_sequences is not None else self.config.num_return_sequences
        )
-        decoder_start_token_id = (
-            decoder_start_token_id if decoder_start_token_id is not None else self.config.decoder_start_token_id
-        )
+        decoder_start_token_id = decoder_start_token_id if decoder_start_token_id is not None else bos_token_id

        if input_ids is not None:
            batch_size = shape_list(input_ids)[0]  # overriden by the input batch_size
@@ -637,6 +635,9 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
        assert (eos_token_id is None) or (
            isinstance(eos_token_id, int) and (eos_token_id >= 0)
        ), "`eos_token_id` should be a positive integer."
+        assert (
+            decoder_start_token_id is not None or self.config.is_encoder_decoder is False
+        ), "`decoder_start_token_id` has to be defined if model is encoder-decoder model"
        assert length_penalty > 0, "`length_penalty` should be strictely positive."
        assert (
            isinstance(num_return_sequences, int) and num_return_sequences > 0
@@ -707,12 +708,8 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
            )  # shape: (batch_size * num_return_sequences * num_beams, cur_len)

        if self.config.is_encoder_decoder:
-            if decoder_start_token_id is None:
-                decoder_start_token_id = bos_token_id

-            assert (
-                decoder_start_token_id is not None
-            ), "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation"
+            assert bos_token_id is not None, "Encoder Decoder Models need to have a bos_token_id"
            assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self)
            assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder)

@@ -999,12 +996,10 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
            # set eos token prob to zero if min_length is not reached
            if eos_token_id is not None and cur_len < min_length:
                # create eos_token_id boolean mask
-                num_batch_hypotheses = batch_size * num_beams
-
                is_token_logit_eos_token = tf.convert_to_tensor(
                    [True if token is eos_token_id else False for token in range(vocab_size)], dtype=tf.bool
                )
-                eos_token_indices_mask = tf.broadcast_to(is_token_logit_eos_token, [num_batch_hypotheses, vocab_size])
+                eos_token_indices_mask = tf.broadcast_to(is_token_logit_eos_token, [batch_size, vocab_size])

                scores = set_tensor_by_indices_to_value(scores, eos_token_indices_mask, -float("inf"))

@@ -809,9 +809,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
        num_return_sequences = (
            num_return_sequences if num_return_sequences is not None else self.config.num_return_sequences
        )
-        decoder_start_token_id = (
-            decoder_start_token_id if decoder_start_token_id is not None else self.config.decoder_start_token_id
-        )
+        decoder_start_token_id = decoder_start_token_id if decoder_start_token_id is not None else bos_token_id

        if input_ids is not None:
            batch_size = input_ids.shape[0]  # overriden by the input batch_size
@@ -833,6 +831,9 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
        assert pad_token_id is None or (
            isinstance(pad_token_id, int) and (pad_token_id >= 0)
        ), "`pad_token_id` should be a positive integer."
+        assert (
+            decoder_start_token_id is not None or self.config.is_encoder_decoder is False
+        ), "`decoder_start_token_id` has to be defined if model is encoder-decoder model"
        assert (eos_token_id is None) or (
            isinstance(eos_token_id, int) and (eos_token_id >= 0)
        ), "`eos_token_id` should be a positive integer."
@@ -911,12 +912,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
            )  # shape: (batch_size * num_return_sequences * num_beams, cur_len)

        if self.config.is_encoder_decoder:
-            if decoder_start_token_id is None:
-                decoder_start_token_id = bos_token_id
-
-            assert (
-                decoder_start_token_id is not None
-            ), "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation"
+            assert bos_token_id is not None, "Encoder Decoder Models need to have a bos_token_id"
            assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self)
            assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder)

@@ -924,6 +920,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
            encoder = self.get_encoder()

            encoder_outputs = encoder(input_ids, attention_mask=attention_mask)
+            self.log_mem(f'done encoder, outputs shaped {encoder_outputs[0].shape}')

            # create empty decoder_input_ids
            input_ids = torch.full(
@@ -31,7 +31,6 @@ from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, AutoConfig
 from .configuration_bart import BartConfig
 from .configuration_distilbert import DistilBertConfig
 from .configuration_roberta import RobertaConfig
-from .configuration_t5 import T5Config
 from .configuration_utils import PretrainedConfig
 from .configuration_xlm import XLMConfig
 from .data import SquadExample, squad_convert_examples_to_features
@@ -61,6 +60,7 @@ if is_torch_available():
        AutoModelForTokenClassification,
        AutoModelWithLMHead,
    )
+    from .modeling_bart import BartForConditionalGeneration


 logger = logging.getLogger(__name__)
@@ -130,9 +130,7 @@ class PipelineDataFormat:

    SUPPORTED_FORMATS = ["json", "csv", "pipe"]

-    def __init__(
-        self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False,
-    ):
+    def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False):
        self.output_path = output_path
        self.input_path = input_path
        self.column = column.split(",") if column is not None else [""]
@@ -178,7 +176,7 @@ class PipelineDataFormat:

    @staticmethod
    def from_str(
-        format: str, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False,
+        format: str, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False
    ):
        if format == "json":
            return JsonPipelineDataFormat(output_path, input_path, column, overwrite=overwrite)
@@ -191,9 +189,7 @@ class PipelineDataFormat:


 class CsvPipelineDataFormat(PipelineDataFormat):
-    def __init__(
-        self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False,
-    ):
+    def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False):
        super().__init__(output_path, input_path, column, overwrite=overwrite)

    def __iter__(self):
@@ -214,9 +210,7 @@ class CsvPipelineDataFormat(PipelineDataFormat):


 class JsonPipelineDataFormat(PipelineDataFormat):
-    def __init__(
-        self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False,
-    ):
+    def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False):
        super().__init__(output_path, input_path, column, overwrite=overwrite)

        with open(input_path, "r") as f:
@@ -342,7 +336,6 @@ class Pipeline(_ScikitCompat):
        tokenizer: PreTrainedTokenizer,
        modelcard: Optional[ModelCard] = None,
        framework: Optional[str] = None,
-        task: str = "",
        args_parser: ArgumentHandler = None,
        device: int = -1,
        binary_output: bool = False,
@@ -363,11 +356,6 @@ class Pipeline(_ScikitCompat):
        if self.framework == "pt" and self.device.type == "cuda":
            self.model = self.model.to(self.device)

-        # Update config with task specific parameters
-        task_specific_params = self.model.config.task_specific_params
-        if task_specific_params is not None and task in task_specific_params:
-            self.model.config.update(task_specific_params.get(task))
-
    def save_pretrained(self, save_directory):
        """
        Save the pipeline's model and tokenizer to the specified save_directory
@@ -432,7 +420,7 @@ class Pipeline(_ScikitCompat):
        """
        args = ["input_ids", "attention_mask"]

-        if not isinstance(self.model.config, (DistilBertConfig, XLMConfig, RobertaConfig, BartConfig, T5Config)):
+        if not isinstance(self.model.config, (DistilBertConfig, XLMConfig, RobertaConfig, BartConfig)):
            args += ["token_type_ids"]

        # PR #1548 (CLI) There is an issue with attention_mask
@@ -444,18 +432,14 @@ class Pipeline(_ScikitCompat):
        else:
            return {k: [feature[k] for feature in features] for k in args}

-    def _parse_and_tokenize(self, *texts, pad_to_max_length=False, **kwargs):
+    def _parse_and_tokenize(self, *texts, **kwargs):
        """
        Parse arguments and tokenize
        """
        # Parse arguments
        inputs = self._args_parser(*texts, **kwargs)
        inputs = self.tokenizer.batch_encode_plus(
-            inputs,
-            add_special_tokens=True,
-            return_tensors=self.framework,
-            max_length=self.tokenizer.max_len,
-            pad_to_max_length=pad_to_max_length,
+            inputs, add_special_tokens=True, return_tensors=self.framework, max_length=self.tokenizer.max_len
        )

        # Filter out features not available on specific models
@@ -536,7 +520,6 @@ class FeatureExtractionPipeline(Pipeline):
        framework: Optional[str] = None,
        args_parser: ArgumentHandler = None,
        device: int = -1,
-        task: str = "",
    ):
        super().__init__(
            model=model,
@@ -546,7 +529,6 @@ class FeatureExtractionPipeline(Pipeline):
            args_parser=args_parser,
            device=device,
            binary_output=True,
-            task=task,
        )

    def __call__(self, *args, **kwargs):
@@ -643,7 +625,6 @@ class FillMaskPipeline(Pipeline):
        args_parser: ArgumentHandler = None,
        device: int = -1,
        topk=5,
-        task: str = "",
    ):
        super().__init__(
            model=model,
@@ -653,7 +634,6 @@ class FillMaskPipeline(Pipeline):
            args_parser=args_parser,
            device=device,
            binary_output=True,
-            task=task,
        )

        self.topk = topk
@@ -745,7 +725,6 @@ class NerPipeline(Pipeline):
        device: int = -1,
        binary_output: bool = False,
        ignore_labels=["O"],
-        task: str = "",
    ):
        super().__init__(
            model=model,
@@ -755,7 +734,6 @@ class NerPipeline(Pipeline):
            args_parser=args_parser,
            device=device,
            binary_output=binary_output,
-            task=task,
        )

        self._basic_tokenizer = BasicTokenizer(do_lower_case=False)
@@ -918,7 +896,6 @@ class QuestionAnsweringPipeline(Pipeline):
        modelcard: Optional[ModelCard] = None,
        framework: Optional[str] = None,
        device: int = -1,
-        task: str = "",
        **kwargs
    ):
        super().__init__(
@@ -928,7 +905,6 @@ class QuestionAnsweringPipeline(Pipeline):
            framework=framework,
            args_parser=QuestionAnsweringArgumentHandler(),
            device=device,
-            task=task,
            **kwargs,
        )

@@ -1126,11 +1102,7 @@ class QuestionAnsweringPipeline(Pipeline):
            chars_idx += len(word) + 1

        # Join text with spaces
-        return {
-            "answer": " ".join(words),
-            "start": max(0, char_start_idx),
-            "end": min(len(text), char_end_idx),
-        }
+        return {"answer": " ".join(words), "start": max(0, char_start_idx), "end": min(len(text), char_end_idx)}


 class SummarizationPipeline(Pipeline):
@@ -1139,16 +1111,12 @@ class SummarizationPipeline(Pipeline):

    Usage::

-        # use bart in pytorch
        summarizer = pipeline("summarization")
-        summarizer("Sam Shleifer writes the best docstring examples in the whole world.", min_length=5, max_length=20)
-
-        # use t5 in tf
-        summarizer = pipeline("summarization", model="t5-base", tokenizer="t5-base", framework="tf")
-        summarizer("Sam Shleifer writes the best docstring examples in the whole world.", min_length=5, max_length=20)
+        summarizer("Sam Shleifer writes the best docstring examples in the whole world.")

    Supported Models:
-        The models that this pipeline can use are models that have been fine-tuned on a summarization task, which is currently, '`bart-large-cnn`', '`t5-small`', '`t5-base`', '`t5-large`', '`t5-3b`', '`t5-11b`'.
+        The models that this pipeline can use are models that have been fine-tuned on a summarization task, which is
+        currently only ``BartForConditionalGeneration.from_pretrained('bart-large-cnn')``

    Arguments:
        model (:obj:`str` or :obj:`~transformers.PreTrainedModel` or :obj:`~transformers.TFPreTrainedModel`, `optional`, defaults to :obj:`None`):
@@ -1179,8 +1147,17 @@ class SummarizationPipeline(Pipeline):
            on the associated CUDA device id.
    """

+    task = "summarization"
+
    def __call__(
-        self, *documents, return_tensors=False, return_text=True, clean_up_tokenization_spaces=False, **generate_kwargs
+        self,
+        *documents,
+        return_tensors=False,
+        return_text=True,
+        max_length=142,
+        min_length=21,
+        clean_up_tokenization_spaces=False,
+        **generate_kwargs
    ):
        r"""
        Args:
@@ -1188,6 +1165,10 @@ class SummarizationPipeline(Pipeline):
            return_text: (bool, default=True) whether to add a decoded "summary_text" to each result
            return_tensors: (bool, default=False) whether to return the raw "summary_token_ids" to each result

+            max_length: (`optional`) int
+                The max length of the sequence to be generated. Does not include tokens in input_ids.
+            min_len: (`optional`) int
+            no_repeat_ngram_size:  (`optional`) int. ban ngrams of this length from being repeated in the generated text
            clean_up_tokenization_spaces: (`optional`) bool whether to include extra spaces in the output
            **generate_kwargs: extra kwargs passed to `self.model.generate`_

@@ -1199,60 +1180,19 @@ class SummarizationPipeline(Pipeline):

        """
        assert return_tensors or return_text, "You must specify return_tensors=True or return_text=True"
-        assert len(documents) > 0, "Please provide a document to summarize"
-
-        if self.framework == "tf" and "BartForConditionalGeneration" in self.model.__class__.__name__:
-            raise NotImplementedError(
-                "Tensorflow is not yet supported for Bart. Please consider using T5, e.g. `t5-base`"
-            )
-
-        prefix = self.model.config.prefix if self.model.config.prefix is not None else ""
-
-        if isinstance(documents[0], list):
-            assert (
-                self.tokenizer.pad_token_id is not None
-            ), "Please make sure that the tokenizer has a pad_token_id when using a batch input"
-
-            documents = ([prefix + document for document in documents[0]],)
-            pad_to_max_length = True
-
-        elif isinstance(documents[0], str):
-            documents = (prefix + documents[0],)
-            pad_to_max_length = False
-        else:
-            raise ValueError(
-                " `documents[0]`: {} have the wrong format. The should be either of type `str` or type `list`".format(
-                    documents[0]
-                )
-            )
-
+        if self.framework == "tf":
+            raise NotImplementedError("Tensorflow not supported")
        with self.device_placement():
-            inputs = self._parse_and_tokenize(*documents, pad_to_max_length=pad_to_max_length)
-
-            if self.framework == "pt":
-                inputs = self.ensure_tensor_on_device(**inputs)
-                input_length = inputs["input_ids"].shape[-1]
-            elif self.framework == "tf":
-                input_length = tf.shape(inputs["input_ids"])[-1].numpy()
-
-            if input_length < self.model.config.min_length // 2:
-                logger.warning(
-                    "Your min_length is set to {}, but you input_length is only {}. You might consider decreasing min_length manually, e.g. summarizer('...', min_length=10)".format(
-                        self.model.config.min_length, input_length
-                    )
-                )
-
-            if input_length < self.model.config.max_length:
-                logger.warning(
-                    "Your max_length is set to {}, but you input_length is only {}. You might consider decreasing max_length manually, e.g. summarizer('...', max_length=50)".format(
-                        self.model.config.max_length, input_length
-                    )
-                )
-
+            inputs = self._parse_and_tokenize(*documents)
+            inputs = self.ensure_tensor_on_device(**inputs)
            summaries = self.model.generate(
-                inputs["input_ids"], attention_mask=inputs["attention_mask"], **generate_kwargs,
+                inputs["input_ids"],
+                attention_mask=inputs["attention_mask"],
+                max_length=max_length,
+                min_length=min_length,
+                do_sample=False,
+                **generate_kwargs,
            )
-
            results = []
            for summary in summaries:
                record = {}
@@ -1260,115 +1200,7 @@ class SummarizationPipeline(Pipeline):
                    record["summary_token_ids"] = summary
                if return_text:
                    record["summary_text"] = self.tokenizer.decode(
-                        summary, skip_special_tokens=True, clean_up_tokenization_spaces=clean_up_tokenization_spaces,
-                    )
-                results.append(record)
-            return results
-
-
-class TranslationPipeline(Pipeline):
-    """
-    Translates from one language to another.
-
-    Usage::
-        en_fr_translator = pipeline("translation_en_to_fr")
-        en_fr_translator("How old are you?")
-
-    Supported Models: "t5-small", "t5-base", "t5-large", "t5-3b", "t5-11b"
-
-    Arguments:
-        model (:obj:`str` or :obj:`~transformers.PreTrainedModel` or :obj:`~transformers.TFPreTrainedModel`, `optional`, defaults to :obj:`None`):
-            The model that will be used by the pipeline to make predictions. This can be :obj:`None`, a string
-            checkpoint identifier or an actual pre-trained model inheriting from
-            :class:`~transformers.PreTrainedModel` for PyTorch and :class:`~transformers.TFPreTrainedModel` for
-            TensorFlow.
-            If :obj:`None`, the default of the pipeline will be loaded.
-        tokenizer (:obj:`str` or :obj:`~transformers.PreTrainedTokenizer`, `optional`, defaults to :obj:`None`):
-            The tokenizer that will be used by the pipeline to encode data for the model. This can be :obj:`None`,
-            a string checkpoint identifier or an actual pre-trained tokenizer inheriting from
-            :class:`~transformers.PreTrainedTokenizer`.
-            If :obj:`None`, the default of the pipeline will be loaded.
-        modelcard (:obj:`str` or :class:`~transformers.ModelCard`, `optional`, defaults to :obj:`None`):
-            Model card attributed to the model for this pipeline.
-        framework (:obj:`str`, `optional`, defaults to :obj:`None`):
-            The framework to use, either "pt" for PyTorch or "tf" for TensorFlow. The specified framework must be
-            installed.
-            If no framework is specified, will default to the one currently installed. If no framework is specified
-            and both frameworks are installed, will default to PyTorch.
-        args_parser (:class:`~transformers.pipelines.ArgumentHandler`, `optional`, defaults to :obj:`None`):
-            Reference to the object in charge of parsing supplied pipeline parameters.
-        device (:obj:`int`, `optional`, defaults to :obj:`-1`):
-            Device ordinal for CPU/GPU supports. Setting this to -1 will leverage CPU, >=0 will run the model
-            on the associated CUDA device id.
-    """
-
-    def __call__(
-        self, *texts, return_tensors=False, return_text=True, clean_up_tokenization_spaces=False, **generate_kwargs
-    ):
-        r"""
-        Args:
-            *texts: (list of strings) articles to be summarized
-            return_text: (bool, default=True) whether to add a decoded "translation_text" to each result
-            return_tensors: (bool, default=False) whether to return the raw "translation_token_ids" to each result
-
-            **generate_kwargs: extra kwargs passed to `self.model.generate`_
-
-        Returns:
-            list of dicts with 'translation_text' and/or 'translation_token_ids' for each text_to_translate
-        .. _`self.model.generate`:
-            https://huggingface.co/transformers/model_doc/bart.html#transformers.BartForConditionalGeneration.generate
-        """
-        assert return_tensors or return_text, "You must specify return_tensors=True or return_text=True"
-
-        prefix = self.model.config.prefix if self.model.config.prefix is not None else ""
-
-        if isinstance(texts[0], list):
-            assert (
-                self.tokenizer.pad_token_id is not None
-            ), "Please make sure that the tokenizer has a pad_token_id when using a batch input"
-            texts = ([prefix + text for text in texts[0]],)
-            pad_to_max_length = True
-
-        elif isinstance(texts[0], str):
-            texts = (prefix + texts[0],)
-            pad_to_max_length = False
-        else:
-            raise ValueError(
-                " `documents[0]`: {} have the wrong format. The should be either of type `str` or type `list`".format(
-                    texts[0]
-                )
-            )
-
-        with self.device_placement():
-            inputs = self._parse_and_tokenize(*texts, pad_to_max_length=pad_to_max_length)
-
-            if self.framework == "pt":
-                inputs = self.ensure_tensor_on_device(**inputs)
-                input_length = inputs["input_ids"].shape[-1]
-
-            elif self.framework == "tf":
-                input_length = tf.shape(inputs["input_ids"])[-1].numpy()
-
-            if input_length > 0.9 * self.model.config.max_length:
-                logger.warning(
-                    "Your input_length: {} is bigger than 0.9 * max_length: {}. You might consider increasing your max_length manually, e.g. translator('...', max_length=400)".format(
-                        input_length, self.model.config.max_length
-                    )
-                )
-
-            translations = self.model.generate(
-                inputs["input_ids"], attention_mask=inputs["attention_mask"], **generate_kwargs,
-            )
-            results = []
-            for translation in translations:
-                record = {}
-                if return_tensors:
-                    record["translation_token_ids"] = translation
-                if return_text:
-                    record["translation_text"] = self.tokenizer.decode(
-                        translation,
-                        skip_special_tokens=True,
-                        clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+                        summary, skip_special_tokens=True, clean_up_tokenization_spaces=clean_up_tokenization_spaces
                    )
                results.append(record)
            return results
@@ -1434,44 +1266,14 @@ SUPPORTED_TASKS = {
    },
    "summarization": {
        "impl": SummarizationPipeline,
-        "tf": TFAutoModelWithLMHead if is_tf_available() else None,
-        "pt": AutoModelWithLMHead if is_torch_available() else None,
+        "pt": BartForConditionalGeneration if is_torch_available() else None,
+        "tf": None,
        "default": {
            "model": {"pt": "bart-large-cnn", "tf": None},
            "config": None,
            "tokenizer": ("bart-large-cnn", {"use_fast": False}),
        },
    },
-    "translation_en_to_fr": {
-        "impl": TranslationPipeline,
-        "tf": TFAutoModelWithLMHead if is_tf_available() else None,
-        "pt": AutoModelWithLMHead if is_torch_available() else None,
-        "default": {
-            "model": {"pt": "t5-base", "tf": "t5-base"},
-            "config": None,
-            "tokenizer": ("t5-base", {"use_fast": False}),
-        },
-    },
-    "translation_en_to_de": {
-        "impl": TranslationPipeline,
-        "tf": TFAutoModelWithLMHead if is_tf_available() else None,
-        "pt": AutoModelWithLMHead if is_torch_available() else None,
-        "default": {
-            "model": {"pt": "t5-base", "tf": "t5-base"},
-            "config": None,
-            "tokenizer": ("t5-base", {"use_fast": False}),
-        },
-    },
-    "translation_en_to_ro": {
-        "impl": TranslationPipeline,
-        "tf": TFAutoModelWithLMHead if is_tf_available() else None,
-        "pt": AutoModelWithLMHead if is_torch_available() else None,
-        "default": {
-            "model": {"pt": "t5-base", "tf": "t5-base"},
-            "config": None,
-            "tokenizer": ("t5-base", {"use_fast": False}),
-        },
-    },
 }


@@ -1559,7 +1361,7 @@ def pipeline(
    framework = framework or get_framework(model)

    targeted_task = SUPPORTED_TASKS[task]
-    task_class, model_class = targeted_task["impl"], targeted_task[framework]
+    task, model_class = targeted_task["impl"], targeted_task[framework]

    # Use default model/config/tokenizer for the task if no model is provided
    if model is None:
@@ -1620,4 +1422,4 @@ def pipeline(
            )
        model = model_class.from_pretrained(model, config=config, **model_kwargs)

-    return task_class(model=model, tokenizer=tokenizer, modelcard=modelcard, framework=framework, task=task, **kwargs,)
+    return task(model=model, tokenizer=tokenizer, modelcard=modelcard, framework=framework, **kwargs)
@@ -28,12 +28,20 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
-    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
    WEIGHTS_NAME,
    AdamW,
-    AutoConfig,
-    AutoModelForQuestionAnswering,
-    AutoTokenizer,
+    BertConfig,
+    BertForQuestionAnswering,
+    BertTokenizer,
+    DistilBertConfig,
+    DistilBertForQuestionAnswering,
+    DistilBertTokenizer,
+    XLMConfig,
+    XLMForQuestionAnswering,
+    XLMTokenizer,
+    XLNetConfig,
+    XLNetForQuestionAnswering,
+    XLNetTokenizer,
    get_linear_schedule_with_warmup,
 )
 from utils_squad import (
@@ -60,10 +68,16 @@ except ImportError:

 logger = logging.getLogger(__name__)

-MODEL_CONFIG_CLASSES = list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
-MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+ALL_MODELS = sum(
+    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLNetConfig, XLMConfig)), ()
+)

-ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)
+MODEL_CLASSES = {
+    "bert": (BertConfig, BertForQuestionAnswering, BertTokenizer),
+    "xlnet": (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer),
+    "xlm": (XLMConfig, XLMForQuestionAnswering, XLMTokenizer),
+    "distilbert": (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer),
+}


 def set_seed(args):
@@ -404,7 +418,7 @@ def main():
        default=None,
        type=str,
        required=True,
-        help="Model type selected in the list: " + ", ".join(MODEL_TYPES),
+        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
    )
    parser.add_argument(
        "--model_name_or_path",
@@ -612,16 +626,17 @@ def main():
        # download model & vocab

    args.model_type = args.model_type.lower()
-    config = AutoConfig.from_pretrained(
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(
        args.config_name if args.config_name else args.model_name_or_path,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    tokenizer = AutoTokenizer.from_pretrained(
+    tokenizer = tokenizer_class.from_pretrained(
        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
        do_lower_case=args.do_lower_case,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    model = AutoModelForQuestionAnswering.from_pretrained(
+    model = model_class.from_pretrained(
        args.model_name_or_path,
        from_tf=bool(".ckpt" in args.model_name_or_path),
        config=config,
@@ -672,8 +687,8 @@ def main():
        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))

        # Load a trained model and vocabulary that you have fine-tuned
-        model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)
-        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
        model.to(args.device)

    # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory
@@ -691,7 +706,7 @@ def main():
        for checkpoint in checkpoints:
            # Reload the model
            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
-            model = AutoModelForQuestionAnswering.from_pretrained(checkpoint)
+            model = model_class.from_pretrained(checkpoint)
            model.to(args.device)

            # Evaluate
@@ -0,0 +1,122 @@
+import unittest
+
+import torch
+import os
+from tests.utils import require_torch, slow
+from transformers import BartTokenizer, BartModel, BartForConditionalGeneration
+from transformers.modeling_bart import shift_tokens_right
+from py3nvml.py3nvml import *
+
+DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+DATA_PATH = "small_test.source"
+SAVE_PREFIX =  os.getenv('SAVE_PREFIX', '')
+from durbango.logging_utils import collect_log_data
+from durbango import patch_module_with_memory_mixin
+
+def save_logs_print_mem(bart, save_path):
+    pth = SAVE_PREFIX + save_path
+    print(f'*** {pth} ***')
+    bart.save_logs(pth+'.txt')
+    bart.save_log_csv(pth+'.csv')
+    print(bart.summary)
+    print(f'*** DONE ***')
+import py3nvml
+
+class Memtest(unittest.TestCase):
+    def setUp(self):
+        if hasattr(self.model, 'reset_logs'): self.model.reset_logs()
+        self.model.log_mem('start')
+        torch.cuda.empty_cache()
+        if torch.cuda.is_available():
+            nvmlInit()
+
+
+    def tearDown(self) -> None:
+        try:
+            nvmlShutdown()
+        except Exception:
+            pass
+
+class TestHface(Memtest):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.lns = [" " + x.rstrip() for x in open(DATA_PATH).readlines()][:6]
+        tokenizer = BartTokenizer.from_pretrained('bart-large')
+        dct = tokenizer.batch_encode_plus(cls.lns, max_length=1024, return_tensors="pt", pad_to_max_length=True)
+        cls.ids = dct['input_ids'].to(DEFAULT_DEVICE)
+        cls.enc_mask = dct['attention_mask'].to(DEFAULT_DEVICE)
+        cls.prev_output_tokens = shift_tokens_right(cls.ids, 1).to(DEFAULT_DEVICE)
+        cls.model = BartForConditionalGeneration.from_pretrained('bart-large-cnn').to(DEFAULT_DEVICE)
+        patch_module_with_memory_mixin(cls.model)
+        return cls
+
+    def test_hf_fwd(self):
+        bart = self.model
+        with torch.no_grad():
+            self.model(self.ids, attention_mask=self.enc_mask, generation_mode=False)
+            self.model.log_mem('done')
+        save_logs_print_mem(self.model, 'hf_fwd')
+
+
+    def test_hf_short_generate(self):
+        self.model.generate(self.ids, attention_mask=self.enc_mask, num_beams=4,
+                            max_length=9, min_length=6,
+                            no_repeat_ngram_size=3,
+                            early_stopping=True,
+                            decoder_start_token_id=2,
+                            )
+        self.model.log_mem('done')
+        save_logs_print_mem(self.model, 'hf_short_generate')
+
+
+    @slow
+    def test_hf_generate(self):
+        self.model.generate(self.ids, attention_mask=self.enc_mask, num_beams=4, max_length=140, min_length=56,
+                            no_repeat_ngram_size=3,
+                            early_stopping=True,
+                            decoder_start_token_id=2,
+                            )
+        self.model.log_mem('done')
+        save_logs_print_mem(self.model, 'hf_generate')
+
+
+try:
+    import fairseq
+    HAS_FAIRSEQ = True
+except ImportError:
+    HAS_FAIRSEQ = False
+class TestFairseq(Memtest):
+
+    @classmethod
+    def setUpClass(cls):
+        import fairseq
+        source_path = "test.source"
+        cls.lns = [" " + x.rstrip() for x in open(source_path).readlines()][:6]
+        tokenizer = BartTokenizer.from_pretrained('bart-large')
+        dct = tokenizer.batch_encode_plus(cls.lns, max_length=1024, return_tensors="pt", pad_to_max_length=True)
+        cls.ids = dct['input_ids'].to(DEFAULT_DEVICE)
+        cls.prev_output_tokens = shift_tokens_right(cls.ids, 1).to(DEFAULT_DEVICE)
+        cls.model = torch.hub.load('pytorch/fairseq', 'bart.large.cnn').eval().to(DEFAULT_DEVICE)
+        patch_module_with_memory_mixin(cls.model)
+        return cls
+
+    def test_fs_fwd(self):
+        bart = self.model
+        with torch.no_grad():
+            bart.model(self.ids, None, self.prev_output_tokens)
+            bart.log_mem('done')
+        save_logs_print_mem(bart, 'fs_fwd')
+
+    def test_fs_short_gen(self):
+        bart = self.model
+        bart.sample(self.lns, beam=4, lenpen=2.0, max_len_b=7, min_len=5, no_repeat_ngram_size=3)
+        bart.log_mem('done')
+        save_logs_print_mem(bart, 'fs_short_generate')
+
+    @slow
+    def test_fs_gen(self):
+        bart = self.model
+        bart.sample(self.lns, beam=4, lenpen=2.0, max_len_b=140, min_len=55, no_repeat_ngram_size=3)
+        bart.log_mem('done')
+        save_logs_print_mem(bart, 'fs_generate')
@@ -279,7 +279,10 @@ class BartHeadTests(unittest.TestCase):
            bos_token_id=0,
        )
        lm_model = BartForConditionalGeneration(config).to(torch_device)
+        lm_model.apply(patch_module_with_memory_mixin)
        lm_model.eval()
+        lm_model.log_mem()
+        lm_model.model.decoder.log_mem()

        max_length = 5
        new_input_ids = lm_model.generate(
@@ -392,7 +395,7 @@ def _long_tensor(tok_lst):


 TOLERANCE = 1e-4
-
+from durbango.logging_utils import patch_module_with_memory_mixin

@require_torch
 class BartModelIntegrationTests(unittest.TestCase):
@@ -1,50 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Google AI Language Team Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-
-from transformers import is_torch_available
-
-from .utils import require_torch, slow, torch_device
-
-
-if is_torch_available():
-    import torch
-    from transformers import CamembertModel
-
-
-@require_torch
-class CamembertModelIntegrationTest(unittest.TestCase):
-    @slow
-    def test_output_embeds_base_model(self):
-        model = CamembertModel.from_pretrained("camembert-base")
-
-        input_ids = torch.tensor(
-            [[5, 121, 11, 660, 16, 730, 25543, 110, 83, 6]], device=torch_device, dtype=torch.long,
-        )  # J'aime le camembert !
-        output = model(input_ids)[0]
-        expected_shape = torch.Size((1, 10, 768))
-        self.assertEqual(output.shape, expected_shape)
-        # compare the actual values for a slice.
-        expected_slice = torch.tensor(
-            [[[-0.0254, 0.0235, 0.1027], [0.0606, -0.1811, -0.0418], [-0.1561, -0.1127, 0.2687]]],
-            device=torch_device,
-            dtype=torch.float,
-        )
-        # camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
-        # camembert.eval()
-        # expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()
-
-        self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4))
@@ -1,50 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Google AI Language Team Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-
-from transformers import is_tf_available
-
-from .utils import require_tf, slow
-
-
-if is_tf_available():
-    import tensorflow as tf
-    import numpy as np
-    from transformers import TFCamembertModel
-
-
-@require_tf
-class TFCamembertModelIntegrationTest(unittest.TestCase):
-    @slow
-    def test_output_embeds_base_model(self):
-        model = TFCamembertModel.from_pretrained("jplu/tf-camembert-base")
-
-        input_ids = tf.convert_to_tensor(
-            [[5, 121, 11, 660, 16, 730, 25543, 110, 83, 6]], dtype=tf.int32,
-        )  # J'aime le camembert !"
-
-        output = model(input_ids)[0]
-        expected_shape = tf.TensorShape((1, 10, 768))
-        self.assertEqual(output.shape, expected_shape)
-        # compare the actual values for a slice.
-        expected_slice = tf.convert_to_tensor(
-            [[[-0.0254, 0.0235, 0.1027], [0.0606, -0.1811, -0.0418], [-0.1561, -0.1127, 0.2687]]], dtype=tf.float32,
-        )
-        # camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
-        # camembert.eval()
-        # expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()
-
-        self.assertTrue(np.allclose(output[:, :3, :3].numpy(), expected_slice.numpy(), atol=1e-4))
@@ -78,15 +78,6 @@ TF_FILL_MASK_FINETUNED_MODELS = [
    (("distilroberta-base", {"use_fast": False}), "distilroberta-base", None),
 ]

-SUMMARIZATION_FINETUNED_MODELS = {("bart-large-cnn", "bart-large-cnn"), ("t5-small", "t5-small")}
-TF_SUMMARIZATION_FINETUNED_MODELS = {("t5-small", "t5-small")}
-
-TRANSLATION_FINETUNED_MODELS = {
-    ("t5-small", "t5-small", "translation_en_to_de"),
-    ("t5-small", "t5-small", "translation_en_to_ro"),
-}
-TF_TRANSLATION_FINETUNED_MODELS = {("t5-small", "t5-small", "translation_en_to_fr")}
-

 class MonoColumnInputTestCase(unittest.TestCase):
    def _test_mono_column_pipeline(
@@ -261,44 +252,10 @@ class MonoColumnInputTestCase(unittest.TestCase):
        valid_inputs = ["A string like this", ["list of strings entry 1", "list of strings v2"]]
        invalid_inputs = [4, "<mask>"]
        mandatory_keys = ["summary_text"]
-        for model, tokenizer in SUMMARIZATION_FINETUNED_MODELS:
-            nlp = pipeline(task="summarization", model=model, tokenizer=tokenizer)
-            self._test_mono_column_pipeline(
-                nlp, valid_inputs, invalid_inputs, mandatory_keys,
-            )
-
-    @require_tf
-    def test_tf_summarization(self):
-        valid_inputs = ["A string like this", ["list of strings entry 1", "list of strings v2"]]
-        invalid_inputs = [4, "<mask>"]
-        mandatory_keys = ["summary_text"]
-        for model, tokenizer in TF_SUMMARIZATION_FINETUNED_MODELS:
-            nlp = pipeline(task="summarization", model=model, tokenizer=tokenizer, framework="tf")
-            self._test_mono_column_pipeline(
-                nlp, valid_inputs, invalid_inputs, mandatory_keys,
-            )
-
-    @require_torch
-    def test_translation(self):
-        valid_inputs = ["A string like this", ["list of strings entry 1", "list of strings v2"]]
-        invalid_inputs = [4, "<mask>"]
-        mandatory_keys = ["translation_text"]
-        for model, tokenizer, task in TRANSLATION_FINETUNED_MODELS:
-            nlp = pipeline(task=task, model=model, tokenizer=tokenizer)
-            self._test_mono_column_pipeline(
-                nlp, valid_inputs, invalid_inputs, mandatory_keys,
-            )
-
-    @require_tf
-    def test_tf_translation(self):
-        valid_inputs = ["A string like this", ["list of strings entry 1", "list of strings v2"]]
-        invalid_inputs = [4, "<mask>"]
-        mandatory_keys = ["translation_text"]
-        for model, tokenizer, task in TF_TRANSLATION_FINETUNED_MODELS:
-            nlp = pipeline(task=task, model=model, tokenizer=tokenizer, framework="tf")
-            self._test_mono_column_pipeline(
-                nlp, valid_inputs, invalid_inputs, mandatory_keys,
-            )
+        nlp = pipeline(task="summarization")
+        self._test_mono_column_pipeline(
+            nlp, valid_inputs, invalid_inputs, mandatory_keys,
+        )


 class MultiColumnInputTestCase(unittest.TestCase):
Author	SHA1	Message	Date
sshleifer	29a1d7bfe4	boom boom	2020-03-24 00:27:01 -04:00
sshleifer	93d42f7b54	boom boom	2020-03-24 00:26:06 -04:00
sshleifer	2fc31af923	boom boom	2020-03-24 00:23:47 -04:00
sshleifer	e0a500d620	boom boom	2020-03-24 00:06:37 -04:00
sshleifer	8e9fa9bb33	boom boom	2020-03-24 00:03:38 -04:00
sshleifer	e9cc7f2895	call init	2020-03-23 23:54:44 -04:00
sshleifer	53f1dcbdb5	add example logging statement	2020-03-23 23:47:04 -04:00
sshleifer	76e6652391	add fairseq	2020-03-23 23:22:54 -04:00
sshleifer	17f3ae3bb8	works	2020-03-23 23:13:52 -04:00
sshleifer	e2931f3860	boom boom	2020-03-23 22:10:08 -04:00