v2.6.0 release: isort un-pinned

* Model cards for CS224n SQuAD2.0 models

* consistent spacing

docs/source/conf.py

@@ -26,7 +26,7 @@ author = u'huggingface'
 # The short X.Y version
 version = u''
 # The full version, including alpha/beta/rc tags
-release = u'2.5.1'
+release = u'2.6.0'
 
 
 # -- General configuration ---------------------------------------------------

examples/ner/run_ner.py

@@ -31,6 +31,7 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
 from transformers import (
+    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     WEIGHTS_NAME,
     AdamW,
     AutoConfig,
@@ -38,7 +39,6 @@ from transformers import (
     AutoTokenizer,
     get_linear_schedule_with_warmup,
 )
-from transformers.modeling_auto import MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
 from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file
 
 
@@ -52,6 +52,7 @@ logger = logging.getLogger(__name__)
 
 MODEL_CONFIG_CLASSES = list(MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys())
 MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+
 ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), ())
 
 TOKENIZER_ARGS = ["do_lower_case", "strip_accents", "keep_accents", "use_fast"]

examples/ner/run_tf_ner.py

@@ -13,16 +13,11 @@ from seqeval import metrics
 
 from transformers import (
     TF2_WEIGHTS_NAME,
-    BertConfig,
+    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
-    BertTokenizer,
+    AutoConfig,
-    DistilBertConfig,
+    AutoTokenizer,
-    DistilBertTokenizer,
     GradientAccumulator,
-    RobertaConfig,
+    TFAutoModelForTokenClassification,
-    RobertaTokenizer,
-    TFBertForTokenClassification,
-    TFDistilBertForTokenClassification,
-    TFRobertaForTokenClassification,
     create_optimizer,
 )
 from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file
@@ -34,22 +29,17 @@ except ImportError:
     from fastprogress.fastprogress import master_bar, progress_bar
 
 
-ALL_MODELS = sum(
+MODEL_CONFIG_CLASSES = list(TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.keys())
-    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)), ()
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
-)
 
-MODEL_CLASSES = {
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)
-    "bert": (BertConfig, TFBertForTokenClassification, BertTokenizer),
-    "roberta": (RobertaConfig, TFRobertaForTokenClassification, RobertaTokenizer),
-    "distilbert": (DistilBertConfig, TFDistilBertForTokenClassification, DistilBertTokenizer),
-}
 
 
 flags.DEFINE_string(
     "data_dir", None, "The input data dir. Should contain the .conll files (or other data files) " "for the task."
 )
 
-flags.DEFINE_string("model_type", None, "Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))
+flags.DEFINE_string("model_type", None, "Model type selected in the list: " + ", ".join(MODEL_TYPES))
 
 flags.DEFINE_string(
     "model_name_or_path",
@@ -509,8 +499,7 @@ def main(_):
     labels = get_labels(args["labels"])
     num_labels = len(labels) + 1
     pad_token_label_id = 0
-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args["model_type"]]
+    config = AutoConfig.from_pretrained(
-    config = config_class.from_pretrained(
         args["config_name"] if args["config_name"] else args["model_name_or_path"],
         num_labels=num_labels,
         cache_dir=args["cache_dir"] if args["cache_dir"] else None,
@@ -520,14 +509,14 @@ def main(_):
 
     # Training
     if args["do_train"]:
-        tokenizer = tokenizer_class.from_pretrained(
+        tokenizer = AutoTokenizer.from_pretrained(
             args["tokenizer_name"] if args["tokenizer_name"] else args["model_name_or_path"],
             do_lower_case=args["do_lower_case"],
             cache_dir=args["cache_dir"] if args["cache_dir"] else None,
         )
 
         with strategy.scope():
-            model = model_class.from_pretrained(
+            model = TFAutoModelForTokenClassification.from_pretrained(
                 args["model_name_or_path"],
                 from_pt=bool(".bin" in args["model_name_or_path"]),
                 config=config,
@@ -562,7 +551,7 @@ def main(_):
 
     # Evaluation
     if args["do_eval"]:
-        tokenizer = tokenizer_class.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
+        tokenizer = AutoTokenizer.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
         checkpoints = []
         results = []
 
@@ -584,7 +573,7 @@ def main(_):
             global_step = checkpoint.split("-")[-1] if re.match(".*checkpoint-[0-9]", checkpoint) else "final"
 
             with strategy.scope():
-                model = model_class.from_pretrained(checkpoint)
+                model = TFAutoModelForTokenClassification.from_pretrained(checkpoint)
 
             y_true, y_pred, eval_loss = evaluate(
                 args, strategy, model, tokenizer, labels, pad_token_label_id, mode="dev"
@@ -611,8 +600,8 @@ def main(_):
                         writer.write("\n")
 
     if args["do_predict"]:
-        tokenizer = tokenizer_class.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
+        tokenizer = AutoTokenizer.from_pretrained(args["output_dir"], do_lower_case=args["do_lower_case"])
-        model = model_class.from_pretrained(args["output_dir"])
+        model = TFAutoModelForTokenClassification.from_pretrained(args["output_dir"])
         eval_batch_size = args["per_device_eval_batch_size"] * args["n_device"]
         predict_dataset, _ = load_and_cache_examples(
             args, tokenizer, labels, pad_token_label_id, eval_batch_size, mode="test"

examples/run_glue.py

@@ -30,32 +30,12 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
 from transformers import (
+    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
     WEIGHTS_NAME,
     AdamW,
-    AlbertConfig,
+    AutoConfig,
-    AlbertForSequenceClassification,
+    AutoModelForSequenceClassification,
-    AlbertTokenizer,
+    AutoTokenizer,
-    BertConfig,
-    BertForSequenceClassification,
-    BertTokenizer,
-    DistilBertConfig,
-    DistilBertForSequenceClassification,
-    DistilBertTokenizer,
-    FlaubertConfig,
-    FlaubertForSequenceClassification,
-    FlaubertTokenizer,
-    RobertaConfig,
-    RobertaForSequenceClassification,
-    RobertaTokenizer,
-    XLMConfig,
-    XLMForSequenceClassification,
-    XLMRobertaConfig,
-    XLMRobertaForSequenceClassification,
-    XLMRobertaTokenizer,
-    XLMTokenizer,
-    XLNetConfig,
-    XLNetForSequenceClassification,
-    XLNetTokenizer,
     get_linear_schedule_with_warmup,
 )
 from transformers import glue_compute_metrics as compute_metrics
@@ -72,33 +52,10 @@ except ImportError:
 
 logger = logging.getLogger(__name__)
 
-ALL_MODELS = sum(
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.keys())
-    (
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
-        tuple(conf.pretrained_config_archive_map.keys())
-        for conf in (
-            BertConfig,
-            XLNetConfig,
-            XLMConfig,
-            RobertaConfig,
-            DistilBertConfig,
-            AlbertConfig,
-            XLMRobertaConfig,
-            FlaubertConfig,
-        )
-    ),
-    (),
-)
 
-MODEL_CLASSES = {
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)
-    "bert": (BertConfig, BertForSequenceClassification, BertTokenizer),
-    "xlnet": (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
-    "xlm": (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
-    "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
-    "distilbert": (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer),
-    "albert": (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer),
-    "xlmroberta": (XLMRobertaConfig, XLMRobertaForSequenceClassification, XLMRobertaTokenizer),
-    "flaubert": (FlaubertConfig, FlaubertForSequenceClassification, FlaubertTokenizer),
-}
 
 
 def set_seed(args):
@@ -442,7 +399,7 @@ def main():
         default=None,
         type=str,
         required=True,
-        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+        help="Model type selected in the list: " + ", ".join(MODEL_TYPES),
     )
     parser.add_argument(
         "--model_name_or_path",
@@ -622,19 +579,18 @@ def main():
         torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
 
     args.model_type = args.model_type.lower()
-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = AutoConfig.from_pretrained(
-    config = config_class.from_pretrained(
         args.config_name if args.config_name else args.model_name_or_path,
         num_labels=num_labels,
         finetuning_task=args.task_name,
         cache_dir=args.cache_dir if args.cache_dir else None,
     )
-    tokenizer = tokenizer_class.from_pretrained(
+    tokenizer = AutoTokenizer.from_pretrained(
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,
         cache_dir=args.cache_dir if args.cache_dir else None,
     )
-    model = model_class.from_pretrained(
+    model = AutoModelForSequenceClassification.from_pretrained(
         args.model_name_or_path,
         from_tf=bool(".ckpt" in args.model_name_or_path),
         config=config,
@@ -673,14 +629,14 @@ def main():
         torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
 
         # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)
+        model = AutoModelForSequenceClassification.from_pretrained(args.output_dir)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir)
         model.to(args.device)
 
     # Evaluation
     results = {}
     if args.do_eval and args.local_rank in [-1, 0]:
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
         checkpoints = [args.output_dir]
         if args.eval_all_checkpoints:
             checkpoints = list(
@@ -692,7 +648,7 @@ def main():
             global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
             prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""
 
-            model = model_class.from_pretrained(checkpoint)
+            model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
             model.to(args.device)
             result = evaluate(args, model, tokenizer, prefix=prefix)
             result = dict((k + "_{}".format(global_step), v) for k, v in result.items())

examples/run_language_modeling.py

@@ -38,28 +38,15 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
 from transformers import (
+    CONFIG_MAPPING,
+    MODEL_WITH_LM_HEAD_MAPPING,
     WEIGHTS_NAME,
     AdamW,
-    BertConfig,
+    AutoConfig,
-    BertForMaskedLM,
+    AutoModelWithLMHead,
-    BertTokenizer,
+    AutoTokenizer,
-    CamembertConfig,
-    CamembertForMaskedLM,
-    CamembertTokenizer,
-    DistilBertConfig,
-    DistilBertForMaskedLM,
-    DistilBertTokenizer,
-    GPT2Config,
-    GPT2LMHeadModel,
-    GPT2Tokenizer,
-    OpenAIGPTConfig,
-    OpenAIGPTLMHeadModel,
-    OpenAIGPTTokenizer,
     PreTrainedModel,
     PreTrainedTokenizer,
-    RobertaConfig,
-    RobertaForMaskedLM,
-    RobertaTokenizer,
     get_linear_schedule_with_warmup,
 )
 
@@ -73,14 +60,8 @@ except ImportError:
 logger = logging.getLogger(__name__)
 
 
-MODEL_CLASSES = {
+MODEL_CONFIG_CLASSES = list(MODEL_WITH_LM_HEAD_MAPPING.keys())
-    "gpt2": (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer),
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
-    "openai-gpt": (OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer),
-    "bert": (BertConfig, BertForMaskedLM, BertTokenizer),
-    "roberta": (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
-    "distilbert": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
-    "camembert": (CamembertConfig, CamembertForMaskedLM, CamembertTokenizer),
-}
 
 
 class TextDataset(Dataset):
@@ -693,23 +674,21 @@ def main():
     if args.local_rank not in [-1, 0]:
         torch.distributed.barrier()  # Barrier to make sure only the first process in distributed training download model & vocab
 
-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-
     if args.config_name:
-        config = config_class.from_pretrained(args.config_name, cache_dir=args.cache_dir)
+        config = AutoConfig.from_pretrained(args.config_name, cache_dir=args.cache_dir)
     elif args.model_name_or_path:
-        config = config_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
+        config = AutoConfig.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
     else:
-        config = config_class()
+        config = CONFIG_MAPPING[args.model_type]()
 
     if args.tokenizer_name:
-        tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir)
+        tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir)
     elif args.model_name_or_path:
-        tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
+        tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
     else:
         raise ValueError(
             "You are instantiating a new {} tokenizer. This is not supported, but you can do it from another script, save it,"
-            "and load it from here, using --tokenizer_name".format(tokenizer_class.__name__)
+            "and load it from here, using --tokenizer_name".format(AutoTokenizer.__name__)
         )
 
     if args.block_size <= 0:
@@ -719,7 +698,7 @@ def main():
         args.block_size = min(args.block_size, tokenizer.max_len)
 
     if args.model_name_or_path:
-        model = model_class.from_pretrained(
+        model = AutoModelWithLMHead.from_pretrained(
             args.model_name_or_path,
             from_tf=bool(".ckpt" in args.model_name_or_path),
             config=config,
@@ -727,7 +706,7 @@ def main():
         )
     else:
         logger.info("Training new model from scratch")
-        model = model_class(config=config)
+        model = AutoModelWithLMHead(config=config)
 
     model.to(args.device)
 
@@ -768,8 +747,8 @@ def main():
         torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
 
         # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)
+        model = AutoModelWithLMHead.from_pretrained(args.output_dir)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir)
         model.to(args.device)
 
     # Evaluation
@@ -786,7 +765,7 @@ def main():
             global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
             prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""
 
-            model = model_class.from_pretrained(checkpoint)
+            model = AutoModelWithLMHead.from_pretrained(checkpoint)
             model.to(args.device)
             result = evaluate(args, model, tokenizer, prefix=prefix)
             result = dict((k + "_{}".format(global_step), v) for k, v in result.items())

examples/run_squad.py

@@ -30,29 +30,12 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
 from transformers import (
+    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     WEIGHTS_NAME,
     AdamW,
-    AlbertConfig,
+    AutoConfig,
-    AlbertForQuestionAnswering,
+    AutoModelForQuestionAnswering,
-    AlbertTokenizer,
+    AutoTokenizer,
-    BertConfig,
-    BertForQuestionAnswering,
-    BertTokenizer,
-    CamembertConfig,
-    CamembertForQuestionAnswering,
-    CamembertTokenizer,
-    DistilBertConfig,
-    DistilBertForQuestionAnswering,
-    DistilBertTokenizer,
-    RobertaConfig,
-    RobertaForQuestionAnswering,
-    RobertaTokenizer,
-    XLMConfig,
-    XLMForQuestionAnswering,
-    XLMTokenizer,
-    XLNetConfig,
-    XLNetForQuestionAnswering,
-    XLNetTokenizer,
     get_linear_schedule_with_warmup,
     squad_convert_examples_to_features,
 )
@@ -72,23 +55,10 @@ except ImportError:
 
 logger = logging.getLogger(__name__)
 
-ALL_MODELS = sum(
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
-    (
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
-        tuple(conf.pretrained_config_archive_map.keys())
-        for conf in (BertConfig, CamembertConfig, RobertaConfig, XLNetConfig, XLMConfig)
-    ),
-    (),
-)
 
-MODEL_CLASSES = {
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)
-    "bert": (BertConfig, BertForQuestionAnswering, BertTokenizer),
-    "camembert": (CamembertConfig, CamembertForQuestionAnswering, CamembertTokenizer),
-    "roberta": (RobertaConfig, RobertaForQuestionAnswering, RobertaTokenizer),
-    "xlnet": (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer),
-    "xlm": (XLMConfig, XLMForQuestionAnswering, XLMTokenizer),
-    "distilbert": (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer),
-    "albert": (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer),
-}
 
 
 def set_seed(args):
@@ -513,7 +483,7 @@ def main():
         default=None,
         type=str,
         required=True,
-        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+        help="Model type selected in the list: " + ", ".join(MODEL_TYPES),
     )
     parser.add_argument(
         "--model_name_or_path",
@@ -757,17 +727,16 @@ def main():
         torch.distributed.barrier()
 
     args.model_type = args.model_type.lower()
-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = AutoConfig.from_pretrained(
-    config = config_class.from_pretrained(
         args.config_name if args.config_name else args.model_name_or_path,
         cache_dir=args.cache_dir if args.cache_dir else None,
     )
-    tokenizer = tokenizer_class.from_pretrained(
+    tokenizer = AutoTokenizer.from_pretrained(
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,
         cache_dir=args.cache_dir if args.cache_dir else None,
     )
-    model = model_class.from_pretrained(
+    model = AutoModelForQuestionAnswering.from_pretrained(
         args.model_name_or_path,
         from_tf=bool(".ckpt" in args.model_name_or_path),
         config=config,
@@ -817,8 +786,8 @@ def main():
         torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
 
         # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)  # , force_download=True)
+        model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)  # , force_download=True)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
         model.to(args.device)
 
     # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory
@@ -842,7 +811,7 @@ def main():
         for checkpoint in checkpoints:
             # Reload the model
             global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
-            model = model_class.from_pretrained(checkpoint)  # , force_download=True)
+            model = AutoModelForQuestionAnswering.from_pretrained(checkpoint)  # , force_download=True)
             model.to(args.device)
 
             # Evaluate

model_cards/dbmdz/bert-base-turkish-128k-cased/README.md

@@ -0,0 +1,76 @@
+---
+language: turkish
+---
+
+# 🤗 + 📚 dbmdz Turkish BERT model
+
+In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
+Library open sources a cased model for Turkish 🎉
+
+# 🇹🇷 BERTurk
+
+BERTurk is a community-driven cased BERT model for Turkish.
+
+Some datasets used for pretraining and evaluation are contributed from the
+awesome Turkish NLP community, as well as the decision for the model name: BERTurk.
+
+## Stats
+
+The current version of the model is trained on a filtered and sentence
+segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/),
+a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a
+special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/).
+
+The final training corpus has a size of 35GB and 44,04,976,662 tokens.
+
+Thanks to Google's TensorFlow Research Cloud (TFRC) we could train a cased model
+on a TPU v3-8 for 2M steps.
+
+For this model we use a vocab size of 128k.
+
+## Model weights
+
+Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
+compatible weights are available. If you need access to TensorFlow checkpoints,
+please raise an issue!
+
+| Model                                | Downloads
+| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------
+| `dbmdz/bert-base-turkish-128k-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/vocab.txt)
+
+## Usage
+
+With Transformers >= 2.3 our BERTurk cased model can be loaded like:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-128k-cased")
+model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-128k-cased")
+```
+
+## Results
+
+For results on PoS tagging or NER tasks, please refer to
+[this repository](https://github.com/stefan-it/turkish-bert).
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
+
+# Contact (Bugs, Feedback, Contribution and more)
+
+For questions about our BERT models just open an issue
+[here](https://github.com/dbmdz/berts/issues/new) 🤗
+
+# Acknowledgments
+
+Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us
+additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing
+us the Turkish NER dataset for evaluation.
+
+Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
+Thanks for providing access to the TFRC ❤️
+
+Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
+it is possible to download both cased and uncased models from their S3 storage 🤗

model_cards/dbmdz/bert-base-turkish-128k-uncased/README.md

@@ -0,0 +1,76 @@
+---
+language: turkish
+---
+
+# 🤗 + 📚 dbmdz Turkish BERT model
+
+In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
+Library open sources an uncased model for Turkish 🎉
+
+# 🇹🇷 BERTurk
+
+BERTurk is a community-driven uncased BERT model for Turkish.
+
+Some datasets used for pretraining and evaluation are contributed from the
+awesome Turkish NLP community, as well as the decision for the model name: BERTurk.
+
+## Stats
+
+The current version of the model is trained on a filtered and sentence
+segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/),
+a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a
+special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/).
+
+The final training corpus has a size of 35GB and 44,04,976,662 tokens.
+
+Thanks to Google's TensorFlow Research Cloud (TFRC) we could train an uncased model
+on a TPU v3-8 for 2M steps.
+
+For this model we use a vocab size of 128k.
+
+## Model weights
+
+Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
+compatible weights are available. If you need access to TensorFlow checkpoints,
+please raise an issue!
+
+| Model                                  | Downloads
+| -------------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| `dbmdz/bert-base-turkish-128k-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/vocab.txt)
+
+## Usage
+
+With Transformers >= 2.3 our BERTurk uncased model can be loaded like:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-128k-uncased")
+model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-128k-uncased")
+```
+
+## Results
+
+For results on PoS tagging or NER tasks, please refer to
+[this repository](https://github.com/stefan-it/turkish-bert).
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
+
+# Contact (Bugs, Feedback, Contribution and more)
+
+For questions about our BERT models just open an issue
+[here](https://github.com/dbmdz/berts/issues/new) 🤗
+
+# Acknowledgments
+
+Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us
+additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing
+us the Turkish NER dataset for evaluation.
+
+Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
+Thanks for providing access to the TFRC ❤️
+
+Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
+it is possible to download both cased and uncased models from their S3 storage 🤗

model_cards/dbmdz/bert-base-turkish-uncased/README.md

@@ -0,0 +1,74 @@
+---
+language: turkish
+---
+
+# 🤗 + 📚 dbmdz Turkish BERT model
+
+In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State
+Library open sources an uncased model for Turkish 🎉
+
+# 🇹🇷 BERTurk
+
+BERTurk is a community-driven uncased BERT model for Turkish.
+
+Some datasets used for pretraining and evaluation are contributed from the
+awesome Turkish NLP community, as well as the decision for the model name: BERTurk.
+
+## Stats
+
+The current version of the model is trained on a filtered and sentence
+segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/),
+a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a
+special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/).
+
+The final training corpus has a size of 35GB and 44,04,976,662 tokens.
+
+Thanks to Google's TensorFlow Research Cloud (TFRC) we could train an uncased model
+on a TPU v3-8 for 2M steps.
+
+## Model weights
+
+Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers)
+compatible weights are available. If you need access to TensorFlow checkpoints,
+please raise an issue!
+
+| Model                             | Downloads
+| --------------------------------- | ---------------------------------------------------------------------------------------------------------------
+| `dbmdz/bert-base-turkish-uncased`   | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/vocab.txt)
+
+## Usage
+
+With Transformers >= 2.3 our BERTurk uncased model can be loaded like:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-uncased")
+model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-uncased")
+```
+
+## Results
+
+For results on PoS tagging or NER tasks, please refer to
+[this repository](https://github.com/stefan-it/turkish-bert).
+
+# Huggingface model hub
+
+All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz).
+
+# Contact (Bugs, Feedback, Contribution and more)
+
+For questions about our BERT models just open an issue
+[here](https://github.com/dbmdz/berts/issues/new) 🤗
+
+# Acknowledgments
+
+Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us
+additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing
+us the Turkish NER dataset for evaluation.
+
+Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC).
+Thanks for providing access to the TFRC ❤️
+
+Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team,
+it is possible to download both cased and uncased models from their S3 storage 🤗

model_cards/elgeish/cs224n-squad2.0-albert-base-v2/README.md

@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 78.94044093451794,
+  "f1": 81.7724930324639,
+  "total": 6078,
+  "HasAns_exact": 76.28865979381443,
+  "HasAns_f1": 82.20385314478195,
+  "HasAns_total": 2910,
+  "NoAns_exact": 81.37626262626263,
+  "NoAns_f1": 81.37626262626263,
+  "NoAns_total": 3168,
+  "best_exact": 78.95689371503784,
+  "best_exact_thresh": 0.0,
+  "best_f1": 81.78894581298378,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 24,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 384,
+  "model_name_or_path": "albert-base-v2",
+  "model_type": "albert",
+  "num_train_epochs": 3,
+  "per_gpu_train_batch_size": 8,
+  "save_steps": 5000,
+  "seed": 42,
+  "train_batch_size": 8,
+  "version_2_with_negative": true,
+  "warmup_steps": 0,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
+* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
+* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
+* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)

model_cards/elgeish/cs224n-squad2.0-albert-large-v2/README.md

@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 79.2694965449161,
+  "f1": 82.50844352970152,
+  "total": 6078,
+  "HasAns_exact": 74.87972508591065,
+  "HasAns_f1": 81.64478342732858,
+  "HasAns_total": 2910,
+  "NoAns_exact": 83.30176767676768,
+  "NoAns_f1": 83.30176767676768,
+  "NoAns_total": 3168,
+  "best_exact": 79.2694965449161,
+  "best_exact_thresh": 0.0,
+  "best_f1": 82.50844352970155,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 1,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 384,
+  "model_name_or_path": "albert-large-v2",
+  "model_type": "albert",
+  "num_train_epochs": 5,
+  "per_gpu_train_batch_size": 8,
+  "save_steps": 5000,
+  "seed": 42,
+  "train_batch_size": 8,
+  "version_2_with_negative": true,
+  "warmup_steps": 0,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
+* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
+* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
+* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)

model_cards/elgeish/cs224n-squad2.0-albert-xxlarge-v1/README.md

@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 85.93287265547877,
+  "f1": 88.91258331187983,
+  "total": 6078,
+  "HasAns_exact": 84.36426116838489,
+  "HasAns_f1": 90.58786301361013,
+  "HasAns_total": 2910,
+  "NoAns_exact": 87.37373737373737,
+  "NoAns_f1": 87.37373737373737,
+  "NoAns_total": 3168,
+  "best_exact": 85.93287265547877,
+  "best_exact_thresh": 0.0,
+  "best_f1": 88.91258331187993,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 24,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 512,
+  "model_name_or_path": "albert-xxlarge-v1",
+  "model_type": "albert",
+  "num_train_epochs": 4,
+  "per_gpu_train_batch_size": 1,
+  "save_steps": 1000,
+  "seed": 42,
+  "train_batch_size": 1,
+  "version_2_with_negative": true,
+  "warmup_steps": 814,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
+* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
+* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)
+* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)

model_cards/elgeish/cs224n-squad2.0-distilbert-base-uncased/README.md

@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 65.16946363935504,
+  "f1": 67.87348075352251,
+  "total": 6078,
+  "HasAns_exact": 69.51890034364261,
+  "HasAns_f1": 75.16667217179045,
+  "HasAns_total": 2910,
+  "NoAns_exact": 61.17424242424242,
+  "NoAns_f1": 61.17424242424242,
+  "NoAns_total": 3168,
+  "best_exact": 65.16946363935504,
+  "best_exact_thresh": 0.0,
+  "best_f1": 67.87348075352243,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 24,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 384,
+  "model_name_or_path": "distilbert-base-uncased-distilled-squad",
+  "model_type": "distilbert",
+  "num_train_epochs": 4,
+  "per_gpu_train_batch_size": 32,
+  "save_steps": 5000,
+  "seed": 42,
+  "train_batch_size": 32,
+  "version_2_with_negative": true,
+  "warmup_steps": 0,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
+* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
+* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
+* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base)

model_cards/elgeish/cs224n-squad2.0-roberta-base/README.md

@@ -0,0 +1,74 @@
+## CS224n SQuAD2.0 Project Dataset
+The goal of this model is to save CS224n students GPU time when establising
+baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf).
+The training set used to fine-tune this model is the same as
+the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however,
+evaluation and model selection were performed using roughly half of the official
+dev set, 6078 examples, picked at random. The data files can be found at
+<https://github.com/elgeish/squad/tree/master/data> — this is the Winter 2020
+version. Given that the official SQuAD2.0 dev set contains the project's test
+set, students must make sure not to use the official SQuAD2.0 dev set in any way
+— including the use of models fine-tuned on the official SQuAD2.0, since they
+used the official SQuAD2.0 dev set for model selection.
+
+## Results
+```json
+{
+  "exact": 75.32082922013821,
+  "f1": 78.66699523704254,
+  "total": 6078,
+  "HasAns_exact": 74.84536082474227,
+  "HasAns_f1": 81.83436324767868,
+  "HasAns_total": 2910,
+  "NoAns_exact": 75.75757575757575,
+  "NoAns_f1": 75.75757575757575,
+  "NoAns_total": 3168,
+  "best_exact": 75.32082922013821,
+  "best_exact_thresh": 0.0,
+  "best_f1": 78.66699523704266,
+  "best_f1_thresh": 0.0
+}
+```
+
+## Notable Arguments
+```json
+{
+  "do_lower_case": true,
+  "doc_stride": 128,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "gradient_accumulation_steps": 24,
+  "learning_rate": 3e-05,
+  "max_answer_length": 30,
+  "max_grad_norm": 1,
+  "max_query_length": 64,
+  "max_seq_length": 384,
+  "model_name_or_path": "roberta-base",
+  "model_type": "roberta",
+  "num_train_epochs": 4,
+  "per_gpu_train_batch_size": 16,
+  "save_steps": 5000,
+  "seed": 42,
+  "train_batch_size": 16,
+  "version_2_with_negative": true,
+  "warmup_steps": 0,
+  "weight_decay": 0
+}
+```
+
+## Environment Setup
+```json
+{
+  "transformers": "2.5.1",
+  "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0",
+  "python": "3.6.5=hc3d631a_2",
+  "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux",
+  "gpu": "Tesla V100-SXM2-16GB"
+}
+```
+
+## Related Models
+* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2)
+* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2)
+* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1)
+* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased)

model_cards/gsarti/scibert-nli/README.md

@@ -2,26 +2,28 @@
 
 This is the model [SciBERT](https://github.com/allenai/scibert) [1] fine-tuned on the [SNLI](https://nlp.stanford.edu/projects/snli/) and the [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) datasets using the [`sentence-transformers` library](https://github.com/UKPLab/sentence-transformers/) to produce universal sentence embeddings [2].
 
-The model uses the original `scivocab` wordpiece vocabulary and was trained using the **average pooling strategy** and a **softmax loss**. 
+The model uses the original `scivocab` wordpiece vocabulary and was trained using the **average pooling strategy** and a **softmax loss**.
 
-**Base model**: `allenai/scibert-scivocab-cased` from HuggingFace AutoModel
+**Base model**: `allenai/scibert-scivocab-cased` from HuggingFace's `AutoModel`.
+
+**Training time**: ~4 hours on the NVIDIA Tesla P100 GPU provided in Kaggle Notebooks.
 
 **Parameters**:
 
-| Parameter      | Value |
+| Parameter        | Value |
-|----------------|-------|
+|------------------|-------|
-| Batch size     | 64    |
+| Batch size       | 64    |
-| Training steps | 20000 |
+| Training steps   | 20000 |
-| Warmup steps   | 1450  |
+| Warmup steps     | 1450  |
+| Lowercasing      | True  |
+| Max. Seq. Length | 128   |
 
 **Performances**: The performance was evaluated on the test portion of the [STS dataset](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) using Spearman rank correlation and compared to the performances of a general BERT base model obtained with the same procedure to verify their similarity.
 
-
+| Model                         | Score       |
-| Model                       | Score       |
+|-------------------------------|-------------|
-|-----------------------------|-------------|
+| `scibert-nli` (this)          | 74.50       |
-| `scibert-nli` (ours)        | 74.50       |
+| `bert-base-nli-mean-tokens`[3]| 77.12       |
-| `bert-base-nli-mean-tokens` | 77.12       |
-
 
 An example usage for similarity-based scientific paper retrieval is provided in the [Covid Papers Browser](https://github.com/gsarti/covid-papers-browser) repository.
 
@@ -30,3 +32,5 @@ An example usage for similarity-based scientific paper retrieval is provided in
 [1] I. Beltagy et al, [SciBERT: A Pretrained Language Model for Scientific Text](https://www.aclweb.org/anthology/D19-1371/)
 
 [2] A. Conneau et al., [Supervised Learning of Universal Sentence Representations from Natural Language Inference Data](https://www.aclweb.org/anthology/D17-1070/)
+
+[3] N. Reimers et I. Gurevych, [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://www.aclweb.org/anthology/D19-1410/)

model_cards/mrm8488/xlm-multi-finetuned-xquadv1/README.md

@@ -93,8 +93,8 @@ from transformers import pipeline
 
 qa_pipeline = pipeline(
     "question-answering",
-    model="mrm8488/bert-multi-uncased-finetuned-xquadv1",
+    model="mrm8488/xlm-multi-finetuned-xquadv1",
-    tokenizer="bert-multi-uncased-finetuned-xquadv1"
+    tokenizer="mrm8488/xlm-multi-finetuned-xquadv1"
 )
 
 # English
@@ -114,7 +114,7 @@ qa_pipeline({
 
 #Output: {'answer': 'работал в репозитории hugginface /','end': 76, 'score': 0.00012340750456964894, 'start': 42}
 ```
-Try it on a Colab:
+Try it on a Colab (*Do not forget to change the model and tokenizer path in the Colab if necessary*):
 
 <a href="https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/Try_mrm8488_xquad_finetuned_uncased_model.ipynb" target="_parent"><img src="https://camo.githubusercontent.com/52feade06f2fecbf006889a904d221e6a730c194/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Open In Colab" data-canonical-src="https://colab.research.google.com/assets/colab-badge.svg"></a>
 

setup.py

@@ -64,7 +64,7 @@ if stale_egg_info.exists():
 extras = {}
 
 extras["mecab"] = ["mecab-python3"]
-extras["sklearn"] = ["scikit-learn==0.22.1"]
+extras["sklearn"] = ["scikit-learn"]
 extras["tf"] = ["tensorflow"]
 extras["tf-cpu"] = ["tensorflow-cpu"]
 extras["torch"] = ["torch"]
@@ -76,14 +76,14 @@ extras["testing"] = ["pytest", "pytest-xdist"]
 extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme"]
 extras["quality"] = [
     "black",
-    "isort @ git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort",
+    "isort",
     "flake8",
 ]
 extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3", "scikit-learn", "tensorflow", "torch"]
 
 setup(
     name="transformers",
-    version="2.5.1",
+    version="2.6.0",
     author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
     author_email="thomas@huggingface.co",
     description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",

src/transformers/__init__.py

@@ -2,7 +2,7 @@
 # There's no way to ignore "F401 '...' imported but unused" warnings in this
 # module, but to preserve other warnings. So, don't check this module at all.
 
-__version__ = "2.5.1"
+__version__ = "2.6.0"
 
 # Work around to update TensorFlow's absl.logging threshold which alters the
 # default Python logging output behavior when present.
@@ -158,6 +158,12 @@ if is_torch_available():
         AutoModelWithLMHead,
         AutoModelForTokenClassification,
         ALL_PRETRAINED_MODEL_ARCHIVE_MAP,
+        MODEL_MAPPING,
+        MODEL_FOR_PRETRAINING_MAPPING,
+        MODEL_WITH_LM_HEAD_MAPPING,
+        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
+        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     )
 
     from .modeling_bert import (
@@ -317,6 +323,12 @@ if is_tf_available():
         TFAutoModelWithLMHead,
         TFAutoModelForTokenClassification,
         TF_ALL_PRETRAINED_MODEL_ARCHIVE_MAP,
+        TF_MODEL_MAPPING,
+        TF_MODEL_FOR_PRETRAINING_MAPPING,
+        TF_MODEL_WITH_LM_HEAD_MAPPING,
+        TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
+        TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
+        TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     )
 
     from .modeling_tf_bert import (

templates/adding_a_new_example_script/run_xxx.py

@@ -28,20 +28,12 @@ from torch.utils.data.distributed import DistributedSampler
 from tqdm import tqdm, trange
 
 from transformers import (
+    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     WEIGHTS_NAME,
     AdamW,
-    BertConfig,
+    AutoConfig,
-    BertForQuestionAnswering,
+    AutoModelForQuestionAnswering,
-    BertTokenizer,
+    AutoTokenizer,
-    DistilBertConfig,
-    DistilBertForQuestionAnswering,
-    DistilBertTokenizer,
-    XLMConfig,
-    XLMForQuestionAnswering,
-    XLMTokenizer,
-    XLNetConfig,
-    XLNetForQuestionAnswering,
-    XLNetTokenizer,
     get_linear_schedule_with_warmup,
 )
 from utils_squad import (
@@ -68,16 +60,10 @@ except ImportError:
 
 logger = logging.getLogger(__name__)
 
-ALL_MODELS = sum(
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
-    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLNetConfig, XLMConfig)), ()
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
-)
 
-MODEL_CLASSES = {
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in MODEL_CONFIG_CLASSES), (),)
-    "bert": (BertConfig, BertForQuestionAnswering, BertTokenizer),
-    "xlnet": (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer),
-    "xlm": (XLMConfig, XLMForQuestionAnswering, XLMTokenizer),
-    "distilbert": (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer),
-}
 
 
 def set_seed(args):
@@ -418,7 +404,7 @@ def main():
         default=None,
         type=str,
         required=True,
-        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
+        help="Model type selected in the list: " + ", ".join(MODEL_TYPES),
     )
     parser.add_argument(
         "--model_name_or_path",
@@ -626,17 +612,16 @@ def main():
         # download model & vocab
 
     args.model_type = args.model_type.lower()
-    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = AutoConfig.from_pretrained(
-    config = config_class.from_pretrained(
         args.config_name if args.config_name else args.model_name_or_path,
         cache_dir=args.cache_dir if args.cache_dir else None,
     )
-    tokenizer = tokenizer_class.from_pretrained(
+    tokenizer = AutoTokenizer.from_pretrained(
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,
         cache_dir=args.cache_dir if args.cache_dir else None,
     )
-    model = model_class.from_pretrained(
+    model = AutoModelForQuestionAnswering.from_pretrained(
         args.model_name_or_path,
         from_tf=bool(".ckpt" in args.model_name_or_path),
         config=config,
@@ -687,8 +672,8 @@ def main():
         torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
 
         # Load a trained model and vocabulary that you have fine-tuned
-        model = model_class.from_pretrained(args.output_dir)
+        model = AutoModelForQuestionAnswering.from_pretrained(args.output_dir)
-        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        tokenizer = AutoTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
         model.to(args.device)
 
     # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory
@@ -706,7 +691,7 @@ def main():
         for checkpoint in checkpoints:
             # Reload the model
             global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
-            model = model_class.from_pretrained(checkpoint)
+            model = AutoModelForQuestionAnswering.from_pretrained(checkpoint)
             model.to(args.device)
 
             # Evaluate

tests/test_modeling_camembert.py

@@ -0,0 +1,50 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from transformers import is_torch_available
+
+from .utils import require_torch, slow, torch_device
+
+
+if is_torch_available():
+    import torch
+    from transformers import CamembertModel
+
+
+@require_torch
+class CamembertModelIntegrationTest(unittest.TestCase):
+    @slow
+    def test_output_embeds_base_model(self):
+        model = CamembertModel.from_pretrained("camembert-base")
+
+        input_ids = torch.tensor(
+            [[5, 121, 11, 660, 16, 730, 25543, 110, 83, 6]], device=torch_device, dtype=torch.long,
+        )  # J'aime le camembert !
+        output = model(input_ids)[0]
+        expected_shape = torch.Size((1, 10, 768))
+        self.assertEqual(output.shape, expected_shape)
+        # compare the actual values for a slice.
+        expected_slice = torch.tensor(
+            [[[-0.0254, 0.0235, 0.1027], [0.0606, -0.1811, -0.0418], [-0.1561, -0.1127, 0.2687]]],
+            device=torch_device,
+            dtype=torch.float,
+        )
+        # camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
+        # camembert.eval()
+        # expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()
+
+        self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4))

tests/test_modeling_tf_camembert.py

@@ -0,0 +1,50 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from transformers import is_tf_available
+
+from .utils import require_tf, slow
+
+
+if is_tf_available():
+    import tensorflow as tf
+    import numpy as np
+    from transformers import TFCamembertModel
+
+
+@require_tf
+class TFCamembertModelIntegrationTest(unittest.TestCase):
+    @slow
+    def test_output_embeds_base_model(self):
+        model = TFCamembertModel.from_pretrained("jplu/tf-camembert-base")
+
+        input_ids = tf.convert_to_tensor(
+            [[5, 121, 11, 660, 16, 730, 25543, 110, 83, 6]], dtype=tf.int32,
+        )  # J'aime le camembert !"
+
+        output = model(input_ids)[0]
+        expected_shape = tf.TensorShape((1, 10, 768))
+        self.assertEqual(output.shape, expected_shape)
+        # compare the actual values for a slice.
+        expected_slice = tf.convert_to_tensor(
+            [[[-0.0254, 0.0235, 0.1027], [0.0606, -0.1811, -0.0418], [-0.1561, -0.1127, 0.2687]]], dtype=tf.float32,
+        )
+        # camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
+        # camembert.eval()
+        # expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()
+
+        self.assertTrue(np.allclose(output[:, :3, :3].numpy(), expected_slice.numpy(), atol=1e-4))