v2.6.0 release: isort un-pinned

Add right model and tokenizer path in example
[model_cards] 🇹🇷 Add new (uncased, 128k) BERTurk model
2020-03-24 11:52:02 -04:00 · 2020-03-24 11:30:12 -04:00 · 2020-03-24 11:29:06 -04:00 · 2020-03-24 11:29:06 -04:00 · 2020-03-24 11:29:06 -04:00 · 2020-03-24 11:28:33 -04:00
30 changed files with 876 additions and 453 deletions
@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'2.5.1'
+release = u'2.6.0'


 # -- General configuration ---------------------------------------------------
@@ -31,6 +31,7 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
+    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
    WEIGHTS_NAME,
    AdamW,
    AutoConfig,
@@ -38,7 +39,6 @@ from transformers import (
    AutoTokenizer,
    get_linear_schedule_with_warmup,
 )
-from transformers.modeling_auto import MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
 from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file


@@ -52,6 +52,7 @@ logger = logging.getLogger(__name__)

 MODEL_CONFIG_CLASSES = list(MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys())
 MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+
 ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), ())

 TOKENIZER_ARGS = ["do_lower_case", "strip_accents", "keep_accents", "use_fast"]
@@ -13,16 +13,11 @@ from seqeval import metrics

 from transformers import (
    TF2_WEIGHTS_NAME,
-    BertConfig,
-    BertTokenizer,
-    DistilBertConfig,
-    DistilBertTokenizer,
+    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
+    AutoConfig,
+    AutoTokenizer,
    GradientAccumulator,
-    RobertaConfig,
-    RobertaTokenizer,
-    TFBertForTokenClassification,
-    TFDistilBertForTokenClassification,
-    TFRobertaForTokenClassification,
+    TFAutoModelForTokenClassification,
    create_optimizer,
 )
 from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file
@@ -34,22 +29,17 @@ except ImportError:
    from fastprogress.fastprogress import master_bar, progress_bar


-ALL_MODELS = sum(
-    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)), ()
-)
+MODEL_CONFIG_CLASSES = list(TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)

-MODEL_CLASSES = {
-    "bert": (BertConfig, TFBertForTokenClassification, BertTokenizer),
-    "roberta": (RobertaConfig, TFRobertaForTokenClassification, RobertaTokenizer),
-    "distilbert": (DistilBertConfig, TFDistilBertForTokenClassification, DistilBertTokenizer),
-}
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)


 flags.DEFINE_string(
    "data_dir", None, "The input data dir. Should contain the .conll files (or other data files) " "for the task."
 )

-flags.DEFINE_string("model_type", None, "Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))
+flags.DEFINE_string("model_type", None, "Model type selected in the list: " + ", ".join(MODEL_TYPES))

 flags.DEFINE_string(
    "model_name_or_path",
@@ -509,8 +499,7 @@ def main(_):
    labels = get_labels(args["labels"])
    num_labels = len(labels) + 1
    pad_token_label_id = 0
-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args["model_type"]]
-    config = config_class.from_pretrained(
+    config = AutoConfig.from_pretrained(
        args["config_name"] if args["config_name"] else args["model_name_or_path"],
        num_labels=num_labels,
        cache_dir=args["cache_dir"] if args["cache_dir"] else None,
@@ -520,14 +509,14 @@ def main(_):

    # Training
    if args["do_train"]:
-        tokenizer = tokenizer_class.from_pretrained(
+        tokenizer = AutoTokenizer.from_pretrained(
            args["tokenizer_name"] if args["tokenizer_name"] else args["model_name_or_path"],
            do_lower_case=args["do_lower_case"],
            cache_dir=args["cache_dir"] if args["cache_dir"] else None,
        )

        with strategy.scope():
-            model = model_class.from_pretrained(
+            model = TFAutoModelForTokenClassification.from_pretrained(
                args["model_name_or_path"],
                from_pt=bool(".bin" in args["model_name_or_path"]),
                config=config,
@@ -562,7 +551,7 @@ def main(_):

    # Evaluation
    if args["do_eval"]:
-        tokenizer = tokenizer_class.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
+        tokenizer = AutoTokenizer.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
        checkpoints = []
        results = []

@@ -584,7 +573,7 @@ def main(_):
            global_step = checkpoint.split("-")[-1] if re.match(".*checkpoint-[0-9]", checkpoint) else "final"

            with strategy.scope():
-                model = model_class.from_pretrained(checkpoint)
+                model = TFAutoModelForTokenClassification.from_pretrained(checkpoint)

            y_true, y_pred, eval_loss = evaluate(
                args, strategy, model, tokenizer, labels, pad_token_label_id, mode="dev"
@@ -611,8 +600,8 @@ def main(_):
                        writer.write("\n")

    if args["do_predict"]:
-        tokenizer = tokenizer_class.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
-        model = model_class.from_pretrained(args["output_dir"])
+        tokenizer = AutoTokenizer.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
+        model = TFAutoModelForTokenClassification.from_pretrained(args["output_dir"])
        eval_batch_size = args["per_device_eval_batch_size"] * args["n_device"]
        predict_dataset, _ = load_and_cache_examples(
            args, tokenizer, labels, pad_token_label_id, eval_batch_size, mode="test"
@@ -1 +0,0 @@
-python benchmarks.py --models bart-large-cnn --batch_sizes 2 --torch
@@ -30,32 +30,12 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
+    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    WEIGHTS_NAME,
    AdamW,
-    AlbertConfig,
-    AlbertForSequenceClassification,
-    AlbertTokenizer,
-    BertConfig,
-    BertForSequenceClassification,
-    BertTokenizer,
-    DistilBertConfig,
-    DistilBertForSequenceClassification,
-    DistilBertTokenizer,
-    FlaubertConfig,
-    FlaubertForSequenceClassification,
-    FlaubertTokenizer,
-    RobertaConfig,
-    RobertaForSequenceClassification,
-    RobertaTokenizer,
-    XLMConfig,
-    XLMForSequenceClassification,
-    XLMRobertaConfig,
-    XLMRobertaForSequenceClassification,
-    XLMRobertaTokenizer,
-    XLMTokenizer,
-    XLNetConfig,
-    XLNetForSequenceClassification,
-    XLNetTokenizer,
+    AutoConfig,
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
    get_linear_schedule_with_warmup,
 )
 from transformers import glue_compute_metrics as compute_metrics
@@ -72,33 +52,10 @@ except ImportError:

 logger = logging.getLogger(__name__)

-ALL_MODELS = sum(
-    (
-        tuple(conf.pretrained_config_archive_map.keys())
-        for conf in (
-            BertConfig,
-            XLNetConfig,
-            XLMConfig,
-            RobertaConfig,
-            DistilBertConfig,
-            AlbertConfig,
-            XLMRobertaConfig,
-            FlaubertConfig,
-        )
-    ),
-    (),
-)
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)

-MODEL_CLASSES = {
-    "bert": (BertConfig, BertForSequenceClassification, BertTokenizer),
-    "xlnet": (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
-    "xlm": (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
-    "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
-    "distilbert": (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer),
-    "albert": (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer),
-    "xlmroberta": (XLMRobertaConfig, XLMRobertaForSequenceClassification, XLMRobertaTokenizer),
-    "flaubert": (FlaubertConfig, FlaubertForSequenceClassification, FlaubertTokenizer),
-}
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)


 def set_seed(args):
@@ -442,7 +399,7 @@ def main():
        default=None,
        type=str,
        required=True,
-        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+        help="Model type selected in the list: " + ", ".join(MODEL_TYPES),
    )
    parser.add_argument(
        "--model_name_or_path",
@@ -622,19 +579,18 @@ def main():
        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

    args.model_type = args.model_type.lower()
-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(
+    config = AutoConfig.from_pretrained(
        args.config_name if args.config_name else args.model_name_or_path,
        num_labels=num_labels,
        finetuning_task=args.task_name,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    tokenizer = tokenizer_class.from_pretrained(
+    tokenizer = AutoTokenizer.from_pretrained(
        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
        do_lower_case=args.do_lower_case,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    model = model_class.from_pretrained(
+    model = AutoModelForSequenceClassification.from_pretrained(
        args.model_name_or_path,
        from_tf=bool(".ckpt" in args.model_name_or_path),
        config=config,
@@ -673,14 +629,14 @@ def main():
        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))

        # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
+        model = AutoModelForSequenceClassification.from_pretrained(args.output_dir)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir)
        model.to(args.device)

    # Evaluation
    results = {}
    if args.do_eval and args.local_rank in [-1, 0]:
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
        checkpoints = [args.output_dir]
        if args.eval_all_checkpoints:
            checkpoints = list(
@@ -692,7 +648,7 @@ def main():
            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
            prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""

-            model = model_class.from_pretrained(checkpoint)
+            model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
            model.to(args.device)
            result = evaluate(args, model, tokenizer, prefix=prefix)
            result = dict((k + "_{}".format(global_step), v) for k, v in result.items())
@@ -38,28 +38,15 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
+    CONFIG_MAPPING,
+    MODEL_WITH_LM_HEAD_MAPPING,
    WEIGHTS_NAME,
    AdamW,
-    BertConfig,
-    BertForMaskedLM,
-    BertTokenizer,
-    CamembertConfig,
-    CamembertForMaskedLM,
-    CamembertTokenizer,
-    DistilBertConfig,
-    DistilBertForMaskedLM,
-    DistilBertTokenizer,
-    GPT2Config,
-    GPT2LMHeadModel,
-    GPT2Tokenizer,
-    OpenAIGPTConfig,
-    OpenAIGPTLMHeadModel,
-    OpenAIGPTTokenizer,
+    AutoConfig,
+    AutoModelWithLMHead,
+    AutoTokenizer,
    PreTrainedModel,
    PreTrainedTokenizer,
-    RobertaConfig,
-    RobertaForMaskedLM,
-    RobertaTokenizer,
    get_linear_schedule_with_warmup,
 )

@@ -73,14 +60,8 @@ except ImportError:
 logger = logging.getLogger(__name__)


-MODEL_CLASSES = {
-    "gpt2": (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer),
-    "openai-gpt": (OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer),
-    "bert": (BertConfig, BertForMaskedLM, BertTokenizer),
-    "roberta": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
-    "distilbert": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
-    "camembert": (CamembertConfig, CamembertForMaskedLM, CamembertTokenizer),
-}
+MODEL_CONFIG_CLASSES = list(MODEL_WITH_LM_HEAD_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)


 class TextDataset(Dataset):
@@ -693,23 +674,21 @@ def main():
    if args.local_rank not in [-1, 0]:
        torch.distributed.barrier()  # Barrier to make sure only the first process in distributed training download model & vocab

-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-
    if args.config_name:
-        config = config_class.from_pretrained(args.config_name, cache_dir=args.cache_dir)
+        config = AutoConfig.from_pretrained(args.config_name, cache_dir=args.cache_dir)
    elif args.model_name_or_path:
-        config = config_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
+        config = AutoConfig.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
    else:
-        config = config_class()
+        config = CONFIG_MAPPING[args.model_type]()

    if args.tokenizer_name:
-        tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir)
+        tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir)
    elif args.model_name_or_path:
-        tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
+        tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
    else:
        raise ValueError(
            "You are instantiating a new {} tokenizer. This is not supported, but you can do it from another script, save it,"
-            "and load it from here, using --tokenizer_name".format(tokenizer_class.__name__)
+            "and load it from here, using --tokenizer_name".format(AutoTokenizer.__name__)
        )

    if args.block_size <= 0:
@@ -719,7 +698,7 @@ def main():
        args.block_size = min(args.block_size, tokenizer.max_len)

    if args.model_name_or_path:
-        model = model_class.from_pretrained(
+        model = AutoModelWithLMHead.from_pretrained(
            args.model_name_or_path,
            from_tf=bool(".ckpt" in args.model_name_or_path),
            config=config,
@@ -727,7 +706,7 @@ def main():
        )
    else:
        logger.info("Training new model from scratch")
-        model = model_class(config=config)
+        model = AutoModelWithLMHead(config=config)

    model.to(args.device)

@@ -768,8 +747,8 @@ def main():
        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))

        # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
+        model = AutoModelWithLMHead.from_pretrained(args.output_dir)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir)
        model.to(args.device)

    # Evaluation
@@ -786,7 +765,7 @@ def main():
            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
            prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""

-            model = model_class.from_pretrained(checkpoint)
+            model = AutoModelWithLMHead.from_pretrained(checkpoint)
            model.to(args.device)
            result = evaluate(args, model, tokenizer, prefix=prefix)
            result = dict((k + "_{}".format(global_step), v) for k, v in result.items())
@@ -30,29 +30,12 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
+    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
    WEIGHTS_NAME,
    AdamW,
-    AlbertConfig,
-    AlbertForQuestionAnswering,
-    AlbertTokenizer,
-    BertConfig,
-    BertForQuestionAnswering,
-    BertTokenizer,
-    CamembertConfig,
-    CamembertForQuestionAnswering,
-    CamembertTokenizer,
-    DistilBertConfig,
-    DistilBertForQuestionAnswering,
-    DistilBertTokenizer,
-    RobertaConfig,
-    RobertaForQuestionAnswering,
-    RobertaTokenizer,
-    XLMConfig,
-    XLMForQuestionAnswering,
-    XLMTokenizer,
-    XLNetConfig,
-    XLNetForQuestionAnswering,
-    XLNetTokenizer,
+    AutoConfig,
+    AutoModelForQuestionAnswering,
+    AutoTokenizer,
    get_linear_schedule_with_warmup,
    squad_convert_examples_to_features,
 )
@@ -72,23 +55,10 @@ except ImportError:

 logger = logging.getLogger(__name__)

-ALL_MODELS = sum(
-    (
-        tuple(conf.pretrained_config_archive_map.keys())
-        for conf in (BertConfig, CamembertConfig, RobertaConfig, XLNetConfig, XLMConfig)
-    ),
-    (),
-)
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)

-MODEL_CLASSES = {
-    "bert": (BertConfig, BertForQuestionAnswering, BertTokenizer),
-    "camembert": (CamembertConfig, CamembertForQuestionAnswering, CamembertTokenizer),
-    "roberta": (RobertaConfig, RobertaForQuestionAnswering, RobertaTokenizer),
-    "xlnet": (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer),
-    "xlm": (XLMConfig, XLMForQuestionAnswering, XLMTokenizer),
-    "distilbert": (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer),
-    "albert": (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer),
-}
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)


 def set_seed(args):
@@ -513,7 +483,7 @@ def main():
        default=None,
        type=str,
        required=True,
-        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+        help="Model type selected in the list: " + ", ".join(MODEL_TYPES),
    )
    parser.add_argument(
        "--model_name_or_path",
@@ -757,17 +727,16 @@ def main():
        torch.distributed.barrier()

    args.model_type = args.model_type.lower()
-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(
+    config = AutoConfig.from_pretrained(
        args.config_name if args.config_name else args.model_name_or_path,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    tokenizer = tokenizer_class.from_pretrained(
+    tokenizer = AutoTokenizer.from_pretrained(
        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
        do_lower_case=args.do_lower_case,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    model = model_class.from_pretrained(
+    model = AutoModelForQuestionAnswering.from_pretrained(
        args.model_name_or_path,
        from_tf=bool(".ckpt" in args.model_name_or_path),
        config=config,
@@ -817,8 +786,8 @@ def main():
        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))

        # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)  # , force_download=True)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)  # , force_download=True)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
        model.to(args.device)

    # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory
@@ -842,7 +811,7 @@ def main():
        for checkpoint in checkpoints:
            # Reload the model
            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
-            model = model_class.from_pretrained(checkpoint)  # , force_download=True)
+            model = AutoModelForQuestionAnswering.from_pretrained(checkpoint)  # , force_download=True)
            model.to(args.device)

            # Evaluate
@@ -0,0 +1,76 @@
+---
+language: turkish
+---
+
+# 🤗 + 📚 dbmdz Turkish BERT model
+
+In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
+Library open sources a cased model for Turkish 🎉
+
+# 🇹🇷 BERTurk
+
+BERTurk is a community-driven cased BERT model for Turkish.
+
+Some datasets used for pretraining and evaluation are contributed from the
+awesome Turkish NLP community, as well as the decision for the model name: BERTurk.
+
+## Stats
+
+The current version of the model is trained on a filtered and sentence
+segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/),
+a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a
+special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/).
+
+The final training corpus has a size of 35GB and 44,04,976,662 tokens.
+
+Thanks to Google's TensorFlow Research Cloud (TFRC) we could train a cased model
+on a TPU v3-8 for 2M steps.
+
+For this model we use a vocab size of 128k.
+
+## Model weights
+
+Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
+compatible weights are available. If you need access to TensorFlow checkpoints,
+please raise an issue!
+
+| Model                                | Downloads
+| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------
+| `dbmdz/bert-base-turkish-128k-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/vocab.txt)
+
+## Usage
+
+With Transformers >= 2.3 our BERTurk cased model can be loaded like:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-128k-cased")
+model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-128k-cased")
+```
+
+## Results
+
+For results on PoS tagging or NER tasks, please refer to
+[this repository](https://github.com/stefan-it/turkish-bert).
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
+
+# Contact (Bugs, Feedback, Contribution and more)
+
+For questions about our BERT models just open an issue
+[here](https://github.com/dbmdz/berts/issues/new) 🤗
+
+# Acknowledgments
+
+Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us
+additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing
+us the Turkish NER dataset for evaluation.
+
+Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
+Thanks for providing access to the TFRC ❤️
+
+Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
+it is possible to download both cased and uncased models from their S3 storage 🤗
@@ -0,0 +1,76 @@
+---
+language: turkish
+---
+
+# 🤗 + 📚 dbmdz Turkish BERT model
+
+In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
+Library open sources an uncased model for Turkish 🎉
+
+# 🇹🇷 BERTurk
+
+BERTurk is a community-driven uncased BERT model for Turkish.
+
+Some datasets used for pretraining and evaluation are contributed from the
+awesome Turkish NLP community, as well as the decision for the model name: BERTurk.
+
+## Stats
+
+The current version of the model is trained on a filtered and sentence
+segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/),
+a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a
+special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/).
+
+The final training corpus has a size of 35GB and 44,04,976,662 tokens.
+
+Thanks to Google's TensorFlow Research Cloud (TFRC) we could train an uncased model
+on a TPU v3-8 for 2M steps.
+
+For this model we use a vocab size of 128k.
+
+## Model weights
+
+Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
+compatible weights are available. If you need access to TensorFlow checkpoints,
+please raise an issue!
+
+| Model                                  | Downloads
+| -------------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| `dbmdz/bert-base-turkish-128k-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/vocab.txt)
+
+## Usage
+
+With Transformers >= 2.3 our BERTurk uncased model can be loaded like:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-128k-uncased")
+model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-128k-uncased")
+```
+
+## Results
+
+For results on PoS tagging or NER tasks, please refer to
+[this repository](https://github.com/stefan-it/turkish-bert).
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
+
+# Contact (Bugs, Feedback, Contribution and more)
+
+For questions about our BERT models just open an issue
+[here](https://github.com/dbmdz/berts/issues/new) 🤗
+
+# Acknowledgments
+
+Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us
+additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing
+us the Turkish NER dataset for evaluation.
+
+Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
+Thanks for providing access to the TFRC ❤️
+
+Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
+it is possible to download both cased and uncased models from their S3 storage 🤗
@@ -0,0 +1,74 @@
+---
+language: turkish
+---
+
+# 🤗 + 📚 dbmdz Turkish BERT model
+
+In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
+Library open sources an uncased model for Turkish 🎉
+
+# 🇹🇷 BERTurk
+
+BERTurk is a community-driven uncased BERT model for Turkish.
+
+Some datasets used for pretraining and evaluation are contributed from the
+awesome Turkish NLP community, as well as the decision for the model name: BERTurk.
+
+## Stats
+
+The current version of the model is trained on a filtered and sentence
+segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/),
+a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a
+special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/).
+
+The final training corpus has a size of 35GB and 44,04,976,662 tokens.
+
+Thanks to Google's TensorFlow Research Cloud (TFRC) we could train an uncased model
+on a TPU v3-8 for 2M steps.
+
+## Model weights
+
+Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
+compatible weights are available. If you need access to TensorFlow checkpoints,
+please raise an issue!
+
+| Model                             | Downloads
+| --------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| `dbmdz/bert-base-turkish-uncased`   | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/vocab.txt)
+
+## Usage
+
+With Transformers >= 2.3 our BERTurk uncased model can be loaded like:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-uncased")
+model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-uncased")
+```
+
+## Results
+
+For results on PoS tagging or NER tasks, please refer to
+[this repository](https://github.com/stefan-it/turkish-bert).
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
+
+# Contact (Bugs, Feedback, Contribution and more)
+
+For questions about our BERT models just open an issue
+[here](https://github.com/dbmdz/berts/issues/new) 🤗
+
+# Acknowledgments
+
+Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us
+additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing
+us the Turkish NER dataset for evaluation.
+
+Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
+Thanks for providing access to the TFRC ❤️
+
+Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
+it is possible to download both cased and uncased models from their S3 storage 🤗
@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 78.94044093451794,
+  "f1": 81.7724930324639,
+  "total": 6078,
+  "HasAns_exact": 76.28865979381443,
+  "HasAns_f1": 82.20385314478195,
+  "HasAns_total": 2910,
+  "NoAns_exact": 81.37626262626263,
+  "NoAns_f1": 81.37626262626263,
+  "NoAns_total": 3168,
+  "best_exact": 78.95689371503784,
+  "best_exact_thresh": 0.0,
+  "best_f1": 81.78894581298378,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 24,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 384,
+  "model_name_or_path": "albert-base-v2",
+  "model_type": "albert",
+  "num_train_epochs": 3,
+  "per_gpu_train_batch_size": 8,
+  "save_steps": 5000,
+  "seed": 42,
+  "train_batch_size": 8,
+  "version_2_with_negative": true,
+  "warmup_steps": 0,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
+* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
+* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
+* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)
@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 79.2694965449161,
+  "f1": 82.50844352970152,
+  "total": 6078,
+  "HasAns_exact": 74.87972508591065,
+  "HasAns_f1": 81.64478342732858,
+  "HasAns_total": 2910,
+  "NoAns_exact": 83.30176767676768,
+  "NoAns_f1": 83.30176767676768,
+  "NoAns_total": 3168,
+  "best_exact": 79.2694965449161,
+  "best_exact_thresh": 0.0,
+  "best_f1": 82.50844352970155,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 1,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 384,
+  "model_name_or_path": "albert-large-v2",
+  "model_type": "albert",
+  "num_train_epochs": 5,
+  "per_gpu_train_batch_size": 8,
+  "save_steps": 5000,
+  "seed": 42,
+  "train_batch_size": 8,
+  "version_2_with_negative": true,
+  "warmup_steps": 0,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
+* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
+* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
+* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)
@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 85.93287265547877,
+  "f1": 88.91258331187983,
+  "total": 6078,
+  "HasAns_exact": 84.36426116838489,
+  "HasAns_f1": 90.58786301361013,
+  "HasAns_total": 2910,
+  "NoAns_exact": 87.37373737373737,
+  "NoAns_f1": 87.37373737373737,
+  "NoAns_total": 3168,
+  "best_exact": 85.93287265547877,
+  "best_exact_thresh": 0.0,
+  "best_f1": 88.91258331187993,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 24,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 512,
+  "model_name_or_path": "albert-xxlarge-v1",
+  "model_type": "albert",
+  "num_train_epochs": 4,
+  "per_gpu_train_batch_size": 1,
+  "save_steps": 1000,
+  "seed": 42,
+  "train_batch_size": 1,
+  "version_2_with_negative": true,
+  "warmup_steps": 814,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
+* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
+* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
+* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)
@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 65.16946363935504,
+  "f1": 67.87348075352251,
+  "total": 6078,
+  "HasAns_exact": 69.51890034364261,
+  "HasAns_f1": 75.16667217179045,
+  "HasAns_total": 2910,
+  "NoAns_exact": 61.17424242424242,
+  "NoAns_f1": 61.17424242424242,
+  "NoAns_total": 3168,
+  "best_exact": 65.16946363935504,
+  "best_exact_thresh": 0.0,
+  "best_f1": 67.87348075352243,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 24,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 384,
+  "model_name_or_path": "distilbert-base-uncased-distilled-squad",
+  "model_type": "distilbert",
+  "num_train_epochs": 4,
+  "per_gpu_train_batch_size": 32,
+  "save_steps": 5000,
+  "seed": 42,
+  "train_batch_size": 32,
+  "version_2_with_negative": true,
+  "warmup_steps": 0,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
+* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
+* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
+* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)
@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 75.32082922013821,
+  "f1": 78.66699523704254,
+  "total": 6078,
+  "HasAns_exact": 74.84536082474227,
+  "HasAns_f1": 81.83436324767868,
+  "HasAns_total": 2910,
+  "NoAns_exact": 75.75757575757575,
+  "NoAns_f1": 75.75757575757575,
+  "NoAns_total": 3168,
+  "best_exact": 75.32082922013821,
+  "best_exact_thresh": 0.0,
+  "best_f1": 78.66699523704266,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 24,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 384,
+  "model_name_or_path": "roberta-base",
+  "model_type": "roberta",
+  "num_train_epochs": 4,
+  "per_gpu_train_batch_size": 16,
+  "save_steps": 5000,
+  "seed": 42,
+  "train_batch_size": 16,
+  "version_2_with_negative": true,
+  "warmup_steps": 0,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
+* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
+* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
+* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
@@ -2,26 +2,28 @@

 This is the model [SciBERT](https://github.com/allenai/scibert) [1] fine-tuned on the [SNLI](https://nlp.stanford.edu/projects/snli/) and the [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) datasets using the [`sentence-transformers` library](https://github.com/UKPLab/sentence-transformers/) to produce universal sentence embeddings [2].

-The model uses the original `scivocab` wordpiece vocabulary and was trained using the **average pooling strategy** and a **softmax loss**. 
+The model uses the original `scivocab` wordpiece vocabulary and was trained using the **average pooling strategy** and a **softmax loss**.

-**Base model**: `allenai/scibert-scivocab-cased` from HuggingFace AutoModel
+**Base model**: `allenai/scibert-scivocab-cased` from HuggingFace's `AutoModel`.
+
+**Training time**: ~4 hours on the NVIDIA Tesla P100 GPU provided in Kaggle Notebooks.

 **Parameters**:

-| Parameter      | Value |
-|----------------|-------|
-| Batch size     | 64    |
-| Training steps | 20000 |
-| Warmup steps   | 1450  |
+| Parameter        | Value |
+|------------------|-------|
+| Batch size       | 64    |
+| Training steps   | 20000 |
+| Warmup steps     | 1450  |
+| Lowercasing      | True  |
+| Max. Seq. Length | 128   |

 **Performances**: The performance was evaluated on the test portion of the [STS dataset](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) using Spearman rank correlation and compared to the performances of a general BERT base model obtained with the same procedure to verify their similarity.

-
-| Model                       | Score       |
-|-----------------------------|-------------|
-| `scibert-nli` (ours)        | 74.50       |
-| `bert-base-nli-mean-tokens` | 77.12       |
-
+| Model                         | Score       |
+|-------------------------------|-------------|
+| `scibert-nli` (this)          | 74.50       |
+| `bert-base-nli-mean-tokens`[3]| 77.12       |

 An example usage for similarity-based scientific paper retrieval is provided in the [Covid Papers Browser](https://github.com/gsarti/covid-papers-browser) repository.

@@ -30,3 +32,5 @@ An example usage for similarity-based scientific paper retrieval is provided in
 [1] I. Beltagy et al, [SciBERT: A Pretrained Language Model for Scientific Text](https://www.aclweb.org/anthology/D19-1371/)

 [2] A. Conneau et al., [Supervised Learning of Universal Sentence Representations from Natural Language Inference Data](https://www.aclweb.org/anthology/D17-1070/)
+
+[3] N. Reimers et I. Gurevych, [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://www.aclweb.org/anthology/D19-1410/)
@@ -93,8 +93,8 @@ from transformers import pipeline

 qa_pipeline = pipeline(
    "question-answering",
-    model="mrm8488/bert-multi-uncased-finetuned-xquadv1",
-    tokenizer="bert-multi-uncased-finetuned-xquadv1"
+    model="mrm8488/xlm-multi-finetuned-xquadv1",
+    tokenizer="mrm8488/xlm-multi-finetuned-xquadv1"
 )

 # English
@@ -114,7 +114,7 @@ qa_pipeline({

 #Output: {'answer': 'работал в репозитории hugginface /','end': 76, 'score': 0.00012340750456964894, 'start': 42}
 ```
-Try it on a Colab:
+Try it on a Colab (*Do not forget to change the model and tokenizer path in the Colab if necessary*):

 <a href="https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/Try_mrm8488_xquad_finetuned_uncased_model.ipynb" target="_parent"><img src="https://camo.githubusercontent.com/52feade06f2fecbf006889a904d221e6a730c194/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Open In Colab" data-canonical-src="https://colab.research.google.com/assets/colab-badge.svg"></a>

@@ -64,7 +64,7 @@ if stale_egg_info.exists():
 extras = {}

 extras["mecab"] = ["mecab-python3"]
-extras["sklearn"] = ["scikit-learn==0.22.1"]
+extras["sklearn"] = ["scikit-learn"]
 extras["tf"] = ["tensorflow"]
 extras["tf-cpu"] = ["tensorflow-cpu"]
 extras["torch"] = ["torch"]
@@ -76,14 +76,14 @@ extras["testing"] = ["pytest", "pytest-xdist"]
 extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme"]
 extras["quality"] = [
    "black",
-    "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
+    "isort",
    "flake8",
 ]
 extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3", "scikit-learn", "tensorflow", "torch"]

 setup(
    name="transformers",
-    version="2.5.1",
+    version="2.6.0",
    author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
    author_email="thomas@huggingface.co",
    description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.

-__version__ = "2.5.1"
+__version__ = "2.6.0"

 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -158,6 +158,12 @@ if is_torch_available():
        AutoModelWithLMHead,
        AutoModelForTokenClassification,
        ALL_PRETRAINED_MODEL_ARCHIVE_MAP,
+        MODEL_MAPPING,
+        MODEL_FOR_PRETRAINING_MAPPING,
+        MODEL_WITH_LM_HEAD_MAPPING,
+        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
+        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
    )

    from .modeling_bert import (
@@ -317,6 +323,12 @@ if is_tf_available():
        TFAutoModelWithLMHead,
        TFAutoModelForTokenClassification,
        TF_ALL_PRETRAINED_MODEL_ARCHIVE_MAP,
+        TF_MODEL_MAPPING,
+        TF_MODEL_FOR_PRETRAINING_MAPPING,
+        TF_MODEL_WITH_LM_HEAD_MAPPING,
+        TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
+        TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+        TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
    )

    from .modeling_tf_bert import (
@@ -1,67 +0,0 @@
-from transformers import *
-import torch
-DEFAULT_DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
-def runner(source_path, out_file, batch_size=8, device=DEFAULT_DEVICE, prof_generate=False):
-
-    tokenizer = BartTokenizer.from_pretrained('bart-large')
-    lns = [" " + x.rstrip() for x in open(source_path).readlines()][:batch_size]
-
-    dct = tokenizer.batch_encode_plus(lns, max_length=1024, return_tensors="pt", pad_to_max_length=True)
-    ids = dct['input_ids'].to(DEFAULT_DEVICE)
-    msk = dct['attention_mask'].to(DEFAULT_DEVICE)
-    model = BartForConditionalGeneration.from_pretrained('bart-large-cnn', output_past=prof_generate).to(DEFAULT_DEVICE)
-    model.log_mem('starting')
-    if prof_generate:
-
-        summaries = model.generate(
-            input_ids=ids,
-            attention_mask=msk,
-            num_beams=4,
-            length_penalty=2.0,
-            max_length=140 + 2,  # +2 from original because we start at step=1 and stop before max_length
-            min_length=55 + 1,  # +1 from original because we start at step=1
-            no_repeat_ngram_size=3,
-            early_stopping=True,
-            do_sample=False,
-            decoder_start_token_id=model.config.eos_token_ids[0],
-        )
-        model.log_mem('done')
-        dec = [tokenizer.decode(s) for s in summaries]
-        print(dec[0])
-    else:
-        #model.decoder.generation_mode = Fals
-        with torch.no_grad():
-            model(
-                input_ids=ids,
-                attention_mask=msk,
-            )
-
-    log_df = model.combine_logs()
-    log_df.to_csv(out_file)
-
-
-import argparse
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "output_path", type=str, help="where to save summaries",
-    )
-    parser.add_argument(
-        "--source_path", type=str, default="/home/shleifer/transformers_fork/notebooks/test.source",
-        help="like cnn_dm/test.source", required=False
-    )
-    parser.add_argument(
-        "--device", type=str, required=False, default=DEFAULT_DEVICE, help="cuda, cuda:1, cpu etc.",
-    )
-    parser.add_argument(
-        "--bs", type=int, default=8, required=False, help="batch size: how many to summarize at a time",
-    )
-    parser.add_argument(
-        "--do-generate", action='store_true', required=False, help="batch size: how many to summarize at a time",
-    )
-    args = parser.parse_args()
-    runner(args.source_path, args.output_path, batch_size=args.bs, device=args.device, prof_generate=args.do_generate)
-
-
-
@@ -19,10 +19,10 @@ from .configuration_utils import PretrainedConfig


 ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "albert-base-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json",
-    "albert-large-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json",
-    "albert-xlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json",
-    "albert-xxlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json",
+    "albert-base-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v1-config.json",
+    "albert-large-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v1-config.json",
+    "albert-xlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v1-config.json",
+    "albert-xxlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v1-config.json",
    "albert-base-v2": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-config.json",
    "albert-large-v2": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-config.json",
    "albert-xlarge-v2": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-config.json",
@@ -33,10 +33,10 @@ logger = logging.getLogger(__name__)


 ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = {
-    "albert-base-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin",
-    "albert-large-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin",
-    "albert-xlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin",
-    "albert-xxlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin",
+    "albert-base-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v1-pytorch_model.bin",
+    "albert-large-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v1-pytorch_model.bin",
+    "albert-xlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v1-pytorch_model.bin",
+    "albert-xxlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v1-pytorch_model.bin",
    "albert-base-v2": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-pytorch_model.bin",
    "albert-large-v2": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-pytorch_model.bin",
    "albert-xlarge-v2": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-pytorch_model.bin",
@@ -25,8 +25,7 @@ from .activations import ACT2FN
 from .configuration_bart import BartConfig
 from .file_utils import add_start_docstrings, add_start_docstrings_to_callable
 from .modeling_utils import PreTrainedModel, create_position_ids_from_input_ids
-from durbango.logging_utils import LoggingMixin
-from durbango.torch_utils import print_tensor_sizes, local_sizeof, get_tensor_shapes_and_pointers
+

 logger = logging.getLogger(__name__)

@@ -110,7 +109,7 @@ def _prepare_bart_decoder_inputs(
    return decoder_input_ids, decoder_attn_mask


-class PretrainedBartModel(PreTrainedModel, LoggingMixin):
+class PretrainedBartModel(PreTrainedModel):
    config_class = BartConfig
    base_model_prefix = "model"
    pretrained_model_archive_map = BART_PRETRAINED_MODEL_ARCHIVE_MAP
@@ -186,10 +185,9 @@ def make_padding_mask(input_ids, padding_idx=1):


 # Helper Modules
-from durbango.torch_utils import get_shapes


-class EncoderLayer(nn.Module, LoggingMixin):
+class EncoderLayer(nn.Module):
    def __init__(self, config: BartConfig):
        super().__init__()
        self.embed_dim = config.d_model
@@ -218,8 +216,7 @@ class EncoderLayer(nn.Module, LoggingMixin):
            encoded output of shape `(seq_len, batch, embed_dim)`
        """
        residual = x
-        x, attn_weights = self.self_attn(query=x, key=x, key_padding_mask=encoder_padding_mask, update_layer_state=False,)
-
+        x, attn_weights = self.self_attn(query=x, key=x, key_padding_mask=encoder_padding_mask,)
        x = F.dropout(x, p=self.dropout, training=self.training)
        x = residual + x
        x = self.self_attn_layer_norm(x)
@@ -233,8 +230,8 @@ class EncoderLayer(nn.Module, LoggingMixin):
        x = self.final_layer_norm(x)
        return x, attn_weights

-import gc
-class BartEncoder(nn.Module, LoggingMixin):
+
+class BartEncoder(nn.Module):
    """
    Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer
    is a :class:`EncoderLayer`.
@@ -285,19 +282,16 @@ class BartEncoder(nn.Module, LoggingMixin):
            attention_mask = attention_mask.eq(0)

        inputs_embeds = self.embed_tokens(input_ids)
-        x = inputs_embeds + self.embed_positions(input_ids)
+        embed_pos = self.embed_positions(input_ids)
+        x = inputs_embeds + embed_pos
        x = self.layernorm_embedding(x)
        x = F.dropout(x, p=self.dropout, training=self.training)
-        assert not (self.output_attentions or self.output_hidden_states)

        # B x T x C -> T x B x C
        x = x.transpose(0, 1)
-        self.log_mem('encoder: starting_loop')
-        encoder_states, all_attentions = [], []
-        #rdd_start = print_tensor_sizes()
-        #rdd_start.to_csv(f'rdd_start.csv')
-        for i, encoder_layer in enumerate(self.layers):

+        encoder_states, all_attentions = [], []
+        for encoder_layer in self.layers:
            if self.output_hidden_states:
                encoder_states.append(x)
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
@@ -305,16 +299,19 @@ class BartEncoder(nn.Module, LoggingMixin):
            if self.training and (dropout_probability < self.layerdrop):  # skip the layer
                attn = None
            else:
-                x, _ = encoder_layer(x, attention_mask)
-            assert len(encoder_states) == 0
-            assert len(all_attentions) == 0
-            #self.log_mem(f'x: {x.shape}, attn: {attn.shape}')
-            self.log_mem(f'Encoder: called layer {i}')
+                x, attn = encoder_layer(x, attention_mask)
+
+            if self.output_attentions:
+                all_attentions.append(attn)
+
+        if self.output_hidden_states:
+            encoder_states.append(x)
+
        encoder_states = [hidden_state.transpose(0, 1) for hidden_state in encoder_states]
        return x, encoder_states, all_attentions


-class DecoderLayer(nn.Module, LoggingMixin):
+class DecoderLayer(nn.Module):
    def __init__(self, config: BartConfig):
        super().__init__()
        self.embed_dim = config.d_model
@@ -377,7 +374,7 @@ class DecoderLayer(nn.Module, LoggingMixin):
        )  # just self_attn weights for now, following t5, layer_state = cache for decoding


-class BartDecoder(nn.Module, LoggingMixin):
+class BartDecoder(nn.Module):
    """
    Transformer decoder consisting of *config.decoder_layers* layers. Each layer
    is a :class:`DecoderLayer`.
@@ -438,8 +435,6 @@ class BartDecoder(nn.Module, LoggingMixin):
            encoder_padding_mask = encoder_padding_mask.eq(0)

        # embed positions
-
-        self.log_mem('decoder: embedded positions')
        positions = self.embed_positions(input_ids, generation_mode=generation_mode)

        if generation_mode:
@@ -448,7 +443,6 @@ class BartDecoder(nn.Module, LoggingMixin):
            assert input_ids.ne(self.padding_idx).any()

        x = self.embed_tokens(input_ids)
-        self.log_mem('decoder: embedded tokens')
        x += positions

        x = self.layernorm_embedding(x)
@@ -470,7 +464,6 @@ class BartDecoder(nn.Module, LoggingMixin):
            x, layer_self_attn, layer_past = decoder_layer(
                x, encoder_hidden_states, encoder_padding_mask, layer_state=layer_state, attention_mask=combined_mask,
            )
-            self.log_mem(f'decoder: called attn {i}')

            if self.output_past:
                next_decoder_cache.append(layer_past.copy())
@@ -490,7 +483,6 @@ class BartDecoder(nn.Module, LoggingMixin):
        return x, next_cache, all_hidden_states, list(all_self_attns)


-
 def _reorder_buffer(attn_cache, new_order):
    for k, input_buffer_k in attn_cache.items():
        if input_buffer_k is not None:
@@ -498,8 +490,8 @@ def _reorder_buffer(attn_cache, new_order):
    return attn_cache


-class SelfAttention(nn.Module, LoggingMixin):
-    """Multi-headed attention from "Attention Is All You Need"""
+class SelfAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""

    def __init__(
        self,
@@ -527,16 +519,11 @@ class SelfAttention(nn.Module, LoggingMixin):
    def _shape(self, tensor, dim_0, bsz):
        return tensor.contiguous().view(dim_0, bsz * self.num_heads, self.head_dim).transpose(0, 1)

-
-    def log_mem(self, msg='', verbose=False):
-        super().log_mem(msg=f'{self.cache_key}_attn:{msg}', verbose=verbose)
-
    def forward(
        self,
        query,
        key: Optional[Tensor],
        key_padding_mask: Optional[Tensor] = None,
-        update_layer_state=True,
        layer_state: Optional[Dict[str, Optional[Tensor]]] = None,
        attn_mask: Optional[Tensor] = None,
    ) -> Tuple[Tensor, Optional[Tensor]]:
@@ -557,7 +544,6 @@ class SelfAttention(nn.Module, LoggingMixin):
            layer_state = {}

        q = self.q_proj(query) * self.scaling
-        self.log_mem('\tq_proj')
        if static_kv:
            if key is None:
                k = v = None
@@ -568,39 +554,29 @@ class SelfAttention(nn.Module, LoggingMixin):
            k = self.k_proj(query)
            v = self.v_proj(query)

-
        q = self._shape(q, tgt_len, bsz)
-        self.log_mem(f'\tq_reshape -> {q.shape}')
        if k is not None:
            k = self._shape(k, -1, bsz)
-            self.log_mem(f'\t done reshaping k,v ->, {k.shape}')
        if v is not None:
            v = self._shape(v, -1, bsz)

-
        if saved_state is not None:
-            self.log_mem('\t about to use saved_state')
            k, v, key_padding_mask = self._use_saved_state(k, v, saved_state, key_padding_mask, static_kv, bsz)

        # Update cache
-        if update_layer_state:
-            layer_state[self.cache_key] = {
-                "prev_key": k.view(bsz, self.num_heads, -1, self.head_dim),
-                "prev_value": v.view(bsz, self.num_heads, -1, self.head_dim),
-                "prev_key_padding_mask": key_padding_mask if not static_kv else None,
-            }
-            self.log_mem('\t attn: done layer_state')
+        layer_state[self.cache_key] = {
+            "prev_key": k.view(bsz, self.num_heads, -1, self.head_dim),
+            "prev_value": v.view(bsz, self.num_heads, -1, self.head_dim),
+            "prev_key_padding_mask": key_padding_mask if not static_kv else None,
+        }

        assert k is not None
        src_len = k.size(1)
-        self.log_mem('\t attn: before BMM(q,k)')
        attn_weights = torch.bmm(q, k.transpose(1, 2))
-        self.log_mem('\t attn: done BMM(q,k)')
        assert attn_weights.size() == (bsz * self.num_heads, tgt_len, src_len)

        if attn_mask is not None:
            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_mask
-            self.log_mem('\t attn: done causal mask')
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        # This is part of a workaround to get around fork/join parallelism not supporting Optional types.
@@ -608,26 +584,21 @@ class SelfAttention(nn.Module, LoggingMixin):
            key_padding_mask = None
        assert key_padding_mask is None or key_padding_mask.size()[:2] == (bsz, src_len,)

-        if key_padding_mask is not None:  # shape (bsz, src_len)
+        if key_padding_mask is not None:  # don't attend to padding symbols
            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
-            attn_weights = attn_weights.masked_fill(key_padding_mask.unsqueeze(1).unsqueeze(2), float("-inf"))
-            self.log_mem('\t attn: done masked_fill')
+            reshaped = key_padding_mask.unsqueeze(1).unsqueeze(2)
+            attn_weights = attn_weights.masked_fill(reshaped, float("-inf"))
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
        attn_weights = F.softmax(attn_weights, dim=-1)
-        self.log_mem('\t attn: done softmax')
        attn_probs = F.dropout(attn_weights, p=self.dropout, training=self.training,)

-
        assert v is not None
        attn_output = torch.bmm(attn_probs, v)
-        self.log_mem('\t attn: done BMM(probs, v)')
        assert attn_output.size() == (bsz * self.num_heads, tgt_len, self.head_dim)
        attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim)
-        self.log_mem('\t attn: done view(output)')
        attn_output = self.out_proj(attn_output)
-        self.log_mem('\t attn: done out_proj')
-        #attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
-        return attn_output, None
+        attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
+        return attn_output, attn_weights

    def _use_saved_state(self, k, v, saved_state, key_padding_mask, static_kv, bsz):
        # saved states are stored with shape (bsz, num_heads, seq_len, head_dim)
@@ -684,7 +655,7 @@ class SelfAttention(nn.Module, LoggingMixin):
        return new_key_padding_mask


-class BartClassificationHead(nn.Module, LoggingMixin):
+class BartClassificationHead(nn.Module):
    """Head for sentence-level classification tasks."""

    # This can trivially be shared with RobertaClassificationHead
@@ -756,8 +727,6 @@ def _filter_out_falsey_values(tup) -> Tuple:

 # Public API

-import time
-import pandas as pd

@add_start_docstrings(
    "The bare BART Model outputting raw hidden-states without any specific head on top.", BART_START_DOCSTRING,
@@ -774,7 +743,6 @@ class BartModel(PretrainedBartModel):
        self.encoder = BartEncoder(config, self.shared)
        self.decoder = BartDecoder(config, self.shared)

-
        self.init_weights()

    @add_start_docstrings_to_callable(BART_INPUTS_DOCSTRING)
@@ -790,9 +758,6 @@ class BartModel(PretrainedBartModel):
    ):

        # make masks if user doesn't supply
-        if encoder_outputs is None:
-            encoder_outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
-        assert isinstance(encoder_outputs, tuple)
        if not generation_mode:
            decoder_input_ids, decoder_attention_mask = _prepare_bart_decoder_inputs(
                self.config,
@@ -802,6 +767,9 @@ class BartModel(PretrainedBartModel):
                mask_dtype=self.shared.weight.dtype,
            )
        assert decoder_input_ids is not None
+        if encoder_outputs is None:
+            encoder_outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
+        assert isinstance(encoder_outputs, tuple)
        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
        decoder_outputs = self.decoder(
            decoder_input_ids,
@@ -836,10 +804,10 @@ class BartForConditionalGeneration(PretrainedBartModel):

    def __init__(self, config: BartConfig):
        super().__init__(config)
-        # if base_model is Nones:
-        #self.log_mem('pre-init')
-        self.model = BartModel(config)
-        #self.lm_head = _make_linear_from_emb(self.model.shared)
+        # if base_model is None:
+        base_model = BartModel(config)
+        self.model = base_model
+        self.lm_head = _make_linear_from_emb(self.model.shared)

    def tie_weights(self):
        pass  # hack to prevent changing lm_head.out_features. The input and output embeddings are still the same.
@@ -898,7 +866,6 @@ class BartForConditionalGeneration(PretrainedBartModel):
            tokenizer.decode(predictions).split()
            # ['good', 'great', 'all', 'really', 'very']
        """
-        self.model.log_mem('before BartModel.forward')
        outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
@@ -908,10 +875,7 @@ class BartForConditionalGeneration(PretrainedBartModel):
            decoder_cached_states=decoder_cached_states,
            generation_mode=generation_mode,
        )
-        self.model.log_mem('after call, before lm_head')
-        lm_logits = F.linear(outputs[0], self.model.shared.weight)
-        #lm_logits = self.lm_head(outputs[0])
-        self.model.log_mem('after lm_head')
+        lm_logits = self.lm_head(outputs[0])
        outputs = (lm_logits,) + outputs[1:]  # Add hidden states and attention if they are here
        if lm_labels is not None:
            loss_fct = nn.CrossEntropyLoss()
@@ -921,7 +885,6 @@ class BartForConditionalGeneration(PretrainedBartModel):

        return outputs

-
    def prepare_inputs_for_generation(self, decoder_input_ids, past, attention_mask, **kwargs):
        assert past is not None, "past has to be defined for encoder_outputs"

@@ -930,8 +893,7 @@ class BartForConditionalGeneration(PretrainedBartModel):
            encoder_outputs, decoder_cached_states = past, None
        else:
            encoder_outputs, decoder_cached_states = past
-        self.log_mem(f'encoder_outputs.shape: {encoder_outputs[0].shape}')
-        self.log_mem(f'decoder_input_ids.shape: {decoder_input_ids.shape}')
+
        return {
            "input_ids": None,  # encoder_outputs is defined. input_ids not needed
            "encoder_outputs": encoder_outputs,
@@ -970,7 +932,7 @@ class BartForConditionalGeneration(PretrainedBartModel):
        return self.model.encoder

    def get_output_embeddings(self):
-        return _make_linear_from_emb(self.model.shared)  # make it on the fly
+        return self.lm_head


@add_start_docstrings(
@@ -814,10 +814,9 @@ class T5Model(T5PreTrainedModel):

        return decoder_outputs + encoder_outputs

-from durbango.logging_utils import LoggingMixin

@add_start_docstrings("""T5 Model with a `language modeling` head on top. """, T5_START_DOCSTRING, T5_INPUTS_DOCSTRING)
-class T5ForConditionalGeneration(T5PreTrainedModel, LoggingMixin):
+class T5ForConditionalGeneration(T5PreTrainedModel):
    r"""
        **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
            Labels for computing the masked language modeling loss.
@@ -896,17 +896,6 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
            effective_batch_size = batch_size
            effective_batch_mult = 1

-        if self.config.is_encoder_decoder:
-            assert bos_token_id is not None, "Encoder Decoder Models need to have a bos_token_id"
-            assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self)
-            assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder)
-
-            # get encoder and store encoder outputs
-            encoder = self.get_encoder()
-
-            encoder_outputs = encoder(input_ids, attention_mask=attention_mask)
-            assert encoder_outputs[0].shape[1] == input_ids.shape[0]
-
        # Expand input ids if num_beams > 1 or num_return_sequences > 1
        if num_return_sequences > 1 or num_beams > 1:
            input_ids_len = input_ids.shape[-1]
@@ -922,8 +911,16 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
                effective_batch_size * num_beams, input_ids_len
            )  # shape: (batch_size * num_return_sequences * num_beams, cur_len)

-
        if self.config.is_encoder_decoder:
+            assert bos_token_id is not None, "Encoder Decoder Models need to have a bos_token_id"
+            assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self)
+            assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder)
+
+            # get encoder and store encoder outputs
+            encoder = self.get_encoder()
+
+            encoder_outputs = encoder(input_ids, attention_mask=attention_mask)
+
            # create empty decoder_input_ids
            input_ids = torch.full(
                (effective_batch_size * num_beams, 1),
@@ -932,16 +929,10 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
                device=next(self.parameters()).device,
            )
            cur_len = 1
-            self.log_mem('about to rearrange')
-            assert batch_size == encoder_outputs[0].shape[1], "NEED MSG"
-            expanded_index = torch.arange(batch_size).view(-1, 1).repeat(1, num_beams* effective_batch_mult).view(-1).to(input_ids.device)
-            encoder_outputs = (encoder_outputs[0].index_select(1, expanded_index), *encoder_outputs[1:])
-            self.log_mem('done rearrange')
        else:
            encoder_outputs = None
            cur_len = input_ids.shape[-1]

-
        if num_beams > 1:
            output = self._generate_beam_search(
                input_ids,
@@ -29,10 +29,10 @@ VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}

 PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
-        "albert-base-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model",
-        "albert-large-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model",
-        "albert-xlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model",
-        "albert-xxlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model",
+        "albert-base-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v1-spiece.model",
+        "albert-large-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v1-spiece.model",
+        "albert-xlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v1-spiece.model",
+        "albert-xxlarge-v1": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v1-spiece.model",
        "albert-base-v2": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-spiece.model",
        "albert-large-v2": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-spiece.model",
        "albert-xlarge-v2": "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-spiece.model",
@@ -28,20 +28,12 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange

 from transformers import (
+    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
    WEIGHTS_NAME,
    AdamW,
-    BertConfig,
-    BertForQuestionAnswering,
-    BertTokenizer,
-    DistilBertConfig,
-    DistilBertForQuestionAnswering,
-    DistilBertTokenizer,
-    XLMConfig,
-    XLMForQuestionAnswering,
-    XLMTokenizer,
-    XLNetConfig,
-    XLNetForQuestionAnswering,
-    XLNetTokenizer,
+    AutoConfig,
+    AutoModelForQuestionAnswering,
+    AutoTokenizer,
    get_linear_schedule_with_warmup,
 )
 from utils_squad import (
@@ -68,16 +60,10 @@ except ImportError:

 logger = logging.getLogger(__name__)

-ALL_MODELS = sum(
-    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLNetConfig, XLMConfig)), ()
-)
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)

-MODEL_CLASSES = {
-    "bert": (BertConfig, BertForQuestionAnswering, BertTokenizer),
-    "xlnet": (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer),
-    "xlm": (XLMConfig, XLMForQuestionAnswering, XLMTokenizer),
-    "distilbert": (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer),
-}
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)


 def set_seed(args):
@@ -418,7 +404,7 @@ def main():
        default=None,
        type=str,
        required=True,
-        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+        help="Model type selected in the list: " + ", ".join(MODEL_TYPES),
    )
    parser.add_argument(
        "--model_name_or_path",
@@ -626,17 +612,16 @@ def main():
        # download model & vocab

    args.model_type = args.model_type.lower()
-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(
+    config = AutoConfig.from_pretrained(
        args.config_name if args.config_name else args.model_name_or_path,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    tokenizer = tokenizer_class.from_pretrained(
+    tokenizer = AutoTokenizer.from_pretrained(
        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
        do_lower_case=args.do_lower_case,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
-    model = model_class.from_pretrained(
+    model = AutoModelForQuestionAnswering.from_pretrained(
        args.model_name_or_path,
        from_tf=bool(".ckpt" in args.model_name_or_path),
        config=config,
@@ -687,8 +672,8 @@ def main():
        torch.save(args, os.path.join(args.output_dir, "training_args.bin"))

        # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
        model.to(args.device)

    # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory
@@ -706,7 +691,7 @@ def main():
        for checkpoint in checkpoints:
            # Reload the model
            global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
-            model = model_class.from_pretrained(checkpoint)
+            model = AutoModelForQuestionAnswering.from_pretrained(checkpoint)
            model.to(args.device)

            # Evaluate
@@ -1,52 +0,0 @@
-import unittest
-
-import torch
-
-from tests.utils import require_torch, slow
-from transformers import BartTokenizer, BartModel
-from transformers.modeling_bart import shift_tokens_right
-
-DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-
-@require_torch
-class TestHface(unittest.TestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        source_path = "test.source"
-        cls.lns = [" " + x.rstrip() for x in open(source_path).readlines()][:6]
-        tokenizer = BartTokenizer.from_pretrained('bart-large')
-        dct = tokenizer.batch_encode_plus(cls.lns, max_length=1024, return_tensors="pt", pad_to_max_length=True)
-        cls.ids = dct['input_ids'].to(DEFAULT_DEVICE)
-        cls.enc_mask = dct['attention_mask'].to(DEFAULT_DEVICE)
-        cls.prev_output_tokens = shift_tokens_right(cls.ids, 1).to(DEFAULT_DEVICE)
-        cls.model = BartForConditionalGeneration.from_pretrained('bart-large-cnn').half().to(DEFAULT_DEVICE).half()
-        return cls
-
-
-
-    @classmethod
-    def setUpClass(cls):
-        source_path = "test.source"
-        cls.lns = [" " + x.rstrip() for x in open(source_path).readlines()][:6]
-        tokenizer = BartTokenizer.from_pretrained('bart-large')
-        dct = tokenizer.batch_encode_plus(cls.lns, max_length=100, return_tensors="pt", pad_to_max_length=True)
-        cls.ids = dct['input_ids'].to(DEFAULT_DEVICE)
-        cls.prev_output_tokens = shift_tokens_right(cls.ids, 1).to(DEFAULT_DEVICE)
-        cls.model = BartModel.from_pretrained('bart-large').to(DEFAULT_DEVICE)
-        #cls.lns = pickle_load('/Users/shleifer/transformers_fork/lns.pkl')
-        return cls
-
-    def test_hf_fwd_batch(self):
-        bart = self.model
-        bart.reset_logs()
-        with torch.no_grad():
-            bart(self.ids)
-        try:
-            log_df = bart.combine_logs()
-            #log_df.to_csv('hf_batch_fwd_logs.csv')
-            bart.save_logs('hf_batch_fwd_logs.txt')
-            print(bart.summary)
-        except AttributeError as e:
-            print(e)
-
@@ -0,0 +1,50 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from transformers import is_torch_available
+
+from .utils import require_torch, slow, torch_device
+
+
+if is_torch_available():
+    import torch
+    from transformers import CamembertModel
+
+
+@require_torch
+class CamembertModelIntegrationTest(unittest.TestCase):
+    @slow
+    def test_output_embeds_base_model(self):
+        model = CamembertModel.from_pretrained("camembert-base")
+
+        input_ids = torch.tensor(
+            [[5, 121, 11, 660, 16, 730, 25543, 110, 83, 6]], device=torch_device, dtype=torch.long,
+        )  # J'aime le camembert !
+        output = model(input_ids)[0]
+        expected_shape = torch.Size((1, 10, 768))
+        self.assertEqual(output.shape, expected_shape)
+        # compare the actual values for a slice.
+        expected_slice = torch.tensor(
+            [[[-0.0254, 0.0235, 0.1027], [0.0606, -0.1811, -0.0418], [-0.1561, -0.1127, 0.2687]]],
+            device=torch_device,
+            dtype=torch.float,
+        )
+        # camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
+        # camembert.eval()
+        # expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()
+
+        self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4))
@@ -0,0 +1,50 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from transformers import is_tf_available
+
+from .utils import require_tf, slow
+
+
+if is_tf_available():
+    import tensorflow as tf
+    import numpy as np
+    from transformers import TFCamembertModel
+
+
+@require_tf
+class TFCamembertModelIntegrationTest(unittest.TestCase):
+    @slow
+    def test_output_embeds_base_model(self):
+        model = TFCamembertModel.from_pretrained("jplu/tf-camembert-base")
+
+        input_ids = tf.convert_to_tensor(
+            [[5, 121, 11, 660, 16, 730, 25543, 110, 83, 6]], dtype=tf.int32,
+        )  # J'aime le camembert !"
+
+        output = model(input_ids)[0]
+        expected_shape = tf.TensorShape((1, 10, 768))
+        self.assertEqual(output.shape, expected_shape)
+        # compare the actual values for a slice.
+        expected_slice = tf.convert_to_tensor(
+            [[[-0.0254, 0.0235, 0.1027], [0.0606, -0.1811, -0.0418], [-0.1561, -0.1127, 0.2687]]], dtype=tf.float32,
+        )
+        # camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
+        # camembert.eval()
+        # expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()
+
+        self.assertTrue(np.allclose(output[:, :3, :3].numpy(), expected_slice.numpy(), atol=1e-4))
Author	SHA1	Message	Date
LysandreJik	fbc5bf10cf	v2.6.0 release: isort un-pinned	2020-03-24 11:52:02 -04:00
Manuel Romero	b88bda6af3	Add right model and tokenizer path in example	2020-03-24 11:30:12 -04:00
Stefan Schweter	b31ef225cf	[model_cards] 🇹🇷 Add new (uncased, 128k) BERTurk model	2020-03-24 11:29:06 -04:00
Stefan Schweter	b4009cb001	[model_cards] 🇹🇷 Add new (cased, 128k) BERTurk model	2020-03-24 11:29:06 -04:00
Stefan Schweter	d3283490ef	[model_cards] 🇹🇷 Add new (uncased) BERTurk model	2020-03-24 11:29:06 -04:00
Mohamed El-Geish	e279a312d6	Model cards for CS224n SQuAD2.0 models (#3406 ) * Model cards for CS224n SQuAD2.0 models * consistent spacing	2020-03-24 11:28:33 -04:00
Gabriele Sarti	7372e62b2c	Added precisions in SciBERT-NLI model card (#3410 )	2020-03-24 11:01:56 -04:00
LysandreJik	471cce24b3	Release: v2.6.0	2020-03-24 10:37:32 -04:00
Patrick von Platen	e392ba6938	Add camembert integration tests (#3375 ) * add integration tests for camembert * use jplu/tf-camembert fro the moment * make style	2020-03-24 10:18:37 +01:00
Julien Chaumond	a8e3336a85	[examples] Use AutoModels in more examples	2020-03-23 20:11:14 -04:00
Julien Chaumond	ec6766a363	[deps] scikit-learn's transient issue was fixed	2020-03-23 18:38:09 -04:00
				`@@ -1 +0,0 @@`
				`python benchmarks.py --models bart-large-cnn --batch_sizes 2 --torch`