fix ==

circleci
2020-03-04 11:38:18 -05:00 · 2020-03-04 11:36:03 -05:00
60 changed files with 9646 additions and 2868 deletions
@@ -14,7 +14,6 @@ jobs:
            - run: sudo pip install codecov pytest-cov
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
            - run: codecov
-
    run_tests_torch:
        working_directory: ~/transformers
        docker:
@@ -29,6 +28,21 @@ jobs:
            - run: sudo pip install codecov pytest-cov
            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
            - run: codecov
+    run_tests_legacy_torch:
+        working_directory: ~/transformers
+        docker:
+            - image: circleci/python:3.7
+        environment:
+            OMP_NUM_THREADS: 1
+        resource_class: xlarge
+        parallelism: 1
+        steps:
+            - checkout
+            - run: sudo pip install torch==1.0.0
+            - run: sudo pip install .[sklearn,testing]
+            - run: sudo pip install codecov pytest-cov
+            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
+            - run: codecov
    run_tests_tf:
        working_directory: ~/transformers
        docker:
@@ -115,6 +129,6 @@ workflows:
            - run_examples_torch
            - run_tests_custom_tokenizers
            - run_tests_torch_and_tf
-            - run_tests_torch
+            - run_tests_legacy_torch
            - run_tests_tf
            - deploy_doc: *workflow_filters
@@ -0,0 +1,7 @@
+FROM pytorch/pytorch:latest
+
+RUN git clone https://github.com/NVIDIA/apex.git && cd apex && python setup.py install --cuda_ext --cpp_ext
+
+RUN pip install transformers
+
+WORKDIR /workspace
@@ -1,26 +0,0 @@
-FROM ubuntu:18.04
-LABEL maintainer="Hugging Face"
-LABEL repository="transformers"
-
-RUN apt update && \
-    apt install -y bash \
-                   build-essential \
-                   git \
-                   curl \
-                   ca-certificates \
-                   python3 \
-                   python3-pip && \
-    rm -rf /var/lib/apt/lists
-
-RUN python3 -m pip install --no-cache-dir --upgrade pip && \
-    python3 -m pip install --no-cache-dir \
-    jupyter \
-    tensorflow-cpu \
-    torch
-
-WORKDIR /workspace
-COPY . transformers/
-RUN cd transformers/ && \
-    python3 -m pip install --no-cache-dir .
-
-CMD ["/bin/bash"]
@@ -1,26 +0,0 @@
-FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04
-LABEL maintainer="Hugging Face"
-LABEL repository="transformers"
-
-RUN apt update && \
-    apt install -y bash \
-                   build-essential \
-                   git \
-                   curl \
-                   ca-certificates \
-                   python3 \
-                   python3-pip && \
-    rm -rf /var/lib/apt/lists
-
-RUN python3 -m pip install --no-cache-dir --upgrade pip && \
-    python3 -m pip install --no-cache-dir \
-    jupyter \
-    tensorflow \
-    torch
-
-WORKDIR /workspace
-COPY . transformers/
-RUN cd transformers/ && \
-    python3 -m pip install --no-cache-dir .
-
-CMD ["/bin/bash"]
@@ -1,25 +0,0 @@
-FROM ubuntu:18.04
-LABEL maintainer="Hugging Face"
-LABEL repository="transformers"
-
-RUN apt update && \
-    apt install -y bash \
-                   build-essential \
-                   git \
-                   curl \
-                   ca-certificates \
-                   python3 \
-                   python3-pip && \
-    rm -rf /var/lib/apt/lists
-
-RUN python3 -m pip install --no-cache-dir --upgrade pip && \
-    python3 -m pip install --no-cache-dir \
-    jupyter \
-    torch
-
-WORKDIR /workspace
-COPY . transformers/
-RUN cd transformers/ && \
-    python3 -m pip install --no-cache-dir .
-
-CMD ["/bin/bash"]
@@ -1,25 +0,0 @@
-FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04
-LABEL maintainer="Hugging Face"
-LABEL repository="transformers"
-
-RUN apt update && \
-    apt install -y bash \
-                   build-essential \
-                   git \
-                   curl \
-                   ca-certificates \
-                   python3 \
-                   python3-pip && \
-    rm -rf /var/lib/apt/lists
-
-RUN python3 -m pip install --no-cache-dir --upgrade pip && \
-    python3 -m pip install --no-cache-dir \
-    mkl \
-    torch
-
-WORKDIR /workspace
-COPY . transformers/
-RUN cd transformers/ && \
-    python3 -m pip install --no-cache-dir .
-
-CMD ["/bin/bash"]
@@ -1,25 +0,0 @@
-FROM ubuntu:18.04
-LABEL maintainer="Hugging Face"
-LABEL repository="transformers"
-
-RUN apt update && \
-    apt install -y bash \
-                   build-essential \
-                   git \
-                   curl \
-                   ca-certificates \
-                   python3 \
-                   python3-pip && \
-    rm -rf /var/lib/apt/lists
-
-RUN python3 -m pip install --no-cache-dir --upgrade pip && \
-    python3 -m pip install --no-cache-dir \
-    mkl \
-    tensorflow-cpu
-
-WORKDIR /workspace
-COPY . transformers/
-RUN cd transformers/ && \
-    python3 -m pip install --no-cache-dir .
-
-CMD ["/bin/bash"]
@@ -1,25 +0,0 @@
-FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04
-LABEL maintainer="Hugging Face"
-LABEL repository="transformers"
-
-RUN apt update && \
-    apt install -y bash \
-                   build-essential \
-                   git \
-                   curl \
-                   ca-certificates \
-                   python3 \
-                   python3-pip && \
-    rm -rf /var/lib/apt/lists
-
-RUN python3 -m pip install --no-cache-dir --upgrade pip && \
-    python3 -m pip install --no-cache-dir \
-    mkl \
-    tensorflow
-
-WORKDIR /workspace
-COPY . transformers/
-RUN cd transformers/ && \
-    python3 -m pip install --no-cache-dir .
-
-CMD ["/bin/bash"]
@@ -7,7 +7,7 @@ file a `Github Issue <https://github.com/huggingface/transformers/issues/new?ass
 Paper
 ~~~~~
 The Bart model was `proposed <https://arxiv.org/abs/1910.13461>`_ by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer on 29 Oct, 2019.
-According to the abstract,
+According to the abstract:

 - Bart uses a standard seq2seq/machine translation architecture with a bidirectional encoder (like BERT) and a left-to-right decoder (like GPT).
 - The pretraining task involves randomly shuffling the order of the original sentences and a novel in-filling scheme, where spans of text are replaced with a single mask token.
@@ -18,28 +18,26 @@ The Authors' code can be found `here <https://github.com/pytorch/fairseq/tree/ma

 Implementation Notes
 ~~~~~~~~~~~~~~~~~~~~
- Bart doesn't use :obj:`token_type_ids` for sequence classification. Use BartTokenizer.encode to get the proper splitting.
- The forward pass of ``BartModel`` will create decoder inputs (using the helper function ``transformers.modeling_bart._prepare_bart_decoder_inputs``)  if they are not passed. This is different than some other modeling APIs.
- Model predictions are intended to be identical to the original implementation. This only works, however, if the string you pass to ``fairseq.encode`` starts with a space.
- ``BartForConditionalGeneration.generate`` should be used for conditional generation tasks like summarization, see the example in that docstrings
- Models that load the ``"bart-large-cnn"`` weights will not have a ``mask_token_id``, or be able to perform mask filling tasks.
-
+- Bart doesn't use :obj:`token_type_ids`, for sequence classification just use BartTokenizer.encode to get the proper splitting.
+- Inputs to the decoder are created by BartModel.forward if they are not passed. This is different than some other model APIs.
+- Model predictions are intended to be identical to the original implementation. This only works, however, if the string you pass to fairseq.encode starts with a space.
+- Decoder inputs are created automatically by the helper function ``transformers.modeling_bart._prepare_bart_decoder_inputs``
+BartModel
+- ``MaskedLM.generate`` should be used for summarization, see the example in that docstrings


 BartModel
-~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~

 .. autoclass:: transformers.BartModel
    :members: forward

-.. autofunction:: transformers.modeling_bart._prepare_bart_decoder_inputs

+BartForMaskedLM
+~~~~~~~~~~~~~~~~~~~~~~~~~~

-BartForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.BartForConditionalGeneration
-    :members: generate, forward
+.. autoclass:: transformers.BartForMaskedLM
+    :members: forward, generate


 BartForSequenceClassification
@@ -54,3 +52,8 @@ BartConfig
 .. autoclass:: transformers.BartConfig
    :members:

+Automatic Creation of Decoder Inputs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+This is enabled by default
+
+.. autofunction:: transformers.modeling_bart._prepare_bart_decoder_inputs
@@ -47,7 +47,6 @@ The different languages this model/tokenizer handles, as well as the ids of thes

 .. code-block::

-    # Continuation of the previous script
    print(tokenizer.lang2id)  # {'en': 0, 'fr': 1}


@@ -55,7 +54,6 @@ These ids should be used when passing a language parameter during a model pass.

 .. code-block::

-    # Continuation of the previous script
    input_ids = torch.tensor([tokenizer.encode("Wikipedia was used to")]) # batch size of 1


@@ -64,7 +62,6 @@ filled with the appropriate language ids, of the same size as input_ids. For eng

 .. code-block::

-    # Continuation of the previous script
    language_id = tokenizer.lang2id['en']  # 0
    langs = torch.tensor([language_id] * input_ids.shape[1])  # torch.tensor([0, 0, 0, ..., 0])

@@ -76,7 +73,6 @@ You can then feed it all as input to your model:

 .. code-block::

-    # Continuation of the previous script
    outputs = model(input_ids, langs=langs)


@@ -4,7 +4,7 @@ from pathlib import Path
 import torch
 from tqdm import tqdm

-from transformers import BartForConditionalGeneration, BartTokenizer
+from transformers import BartForMaskedLM, BartTokenizer


 DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
@@ -18,7 +18,7 @@ def chunks(lst, n):

 def generate_summaries(lns, out_file, batch_size=8, device=DEFAULT_DEVICE):
    fout = Path(out_file).open("w")
-    model = BartForConditionalGeneration.from_pretrained("bart-large-cnn", output_past=True,)
+    model = BartForMaskedLM.from_pretrained("bart-large-cnn", output_past=True,)
    tokenizer = BartTokenizer.from_pretrained("bart-large")
    for batch in tqdm(list(chunks(lns, batch_size))):
        dct = tokenizer.batch_encode_plus(batch, max_length=1024, return_tensors="pt", pad_to_max_length=True)
@@ -1,18 +0,0 @@
---
-language:
- bulgarian
- czech
- polish
- russian
---
-
-# bert-base-bg-cs-pl-ru-cased
-
-SlavicBERT\[1\] \(Slavic \(bg, cs, pl, ru\), cased, 12-layer, 768-hidden, 12-heads, 180M parameters\) was trained
-on Russian News and four Wikipedias: Bulgarian, Czech, Polish, and Russian.
-Subtoken vocabulary was built using this data. Multilingual BERT was used as an initialization for SlavicBERT.
-
-
-\[1\]: Arkhipov M., Trofimova M., Kuratov Y., Sorokin A. \(2019\).
-[Tuning Multilingual Transformers for Language-Specific Named Entity Recognition](https://www.aclweb.org/anthology/W19-3712/).
-ACL anthology W19-3712.
@@ -1,23 +0,0 @@
---
-language:
- english
---
-
-# bert-base-cased-conversational
-
-Conversational BERT \(English, cased, 12-layer, 768-hidden, 12-heads, 110M parameters\) was trained
-on the English part of Twitter, Reddit, DailyDialogues\[1\], OpenSubtitles\[2\], Debates\[3\], Blogs\[4\],
-Facebook News Comments. We used this training data to build the vocabulary of English subtokens and took
-English cased version of BERT-base as an initialization for English Conversational BERT.
-
-
-\[1\]: Yanran Li, Hui Su, Xiaoyu Shen, Wenjie Li, Ziqiang Cao, and Shuzi Niu. DailyDialog: A Manually Labelled
-Multi-turn Dialogue Dataset. IJCNLP 2017.
-
-\[2\]: P. Lison and J. Tiedemann, 2016, OpenSubtitles2016: Extracting Large Parallel Corpora from Movie and TV Subtitles.
-In Proceedings of the 10th International Conference on Language Resources and Evaluation \(LREC 2016\)
-
-\[3\]: Justine Zhang, Ravi Kumar, Sujith Ravi, Cristian Danescu-Niculescu-Mizil. Proceedings of NAACL, 2016.
-
-\[4\]: J. Schler, M. Koppel, S. Argamon and J. Pennebaker \(2006\). Effects of Age and Gender on Blogging
-in Proceedings of 2006 AAAI Spring Symposium on Computational Approaches for Analyzing Weblogs.
@@ -1,22 +0,0 @@
---
-language:
- multilingual
---
-
-# bert-base-multilingual-cased-sentence
-
-Sentence Multilingual BERT \(101 languages, cased, 12-layer, 768-hidden, 12-heads, 180M parameters\)
-is a representation-based sentence encoder for 101 languages of Multilingual BERT.
-It is initialized with Multilingual BERT and then fine-tuned on english MultiNLI\[1\] and on dev set
-of multilingual XNLI\[2\].
-Sentence representations are mean pooled token embeddings in the same manner as in Sentence-BERT\[3\].
-
-
-\[1\]: Williams A., Nangia N. & Bowman S. \(2017\) A Broad-Coverage Challenge Corpus for Sentence Understanding
-through Inference. arXiv preprint [arXiv:1704.05426](https://arxiv.org/abs/1704.05426)
-
-\[2\]: Williams A., Bowman S. \(2018\) XNLI: Evaluating Cross-lingual Sentence Representations.
-arXiv preprint [arXiv:1809.05053](https://arxiv.org/abs/1809.05053)
-
-\[3\]: N. Reimers, I. Gurevych \(2019\) Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks.
-arXiv preprint [arXiv:1908.10084](https://arxiv.org/abs/1908.10084)
@@ -1,18 +0,0 @@
---
-language:
- russian
---
-
-# rubert-base-cased-conversational
-
-Conversational RuBERT \(Russian, cased, 12-layer, 768-hidden, 12-heads, 180M parameters\) was trained
-on OpenSubtitles\[1\], [Dirty](https://d3.ru/), [Pikabu](https://pikabu.ru/),
-and a Social Media segment of Taiga corpus\[2\]. We assembled a new vocabulary for Conversational RuBERT model
-on this data and initialized the model with [RuBERT](../rubert-base-cased).
-
-
-\[1\]: P. Lison and J. Tiedemann, 2016, OpenSubtitles2016: Extracting Large Parallel Corpora from Movie and TV Subtitles.
-In Proceedings of the 10th International Conference on Language Resources and Evaluation \(LREC 2016\)
-
-\[2\]: Shavrina T., Shapovalova O. \(2017\) TO THE METHODOLOGY OF CORPUS CONSTRUCTION FOR MACHINE LEARNING:
-«TAIGA» SYNTAX TREE CORPUS AND PARSER. in proc. of “CORPORA2017”, international conference , Saint-Petersbourg, 2017.
@@ -1,21 +0,0 @@
---
-language:
- russian
---
-
-# rubert-base-cased-sentence
-
-Sentence RuBERT \(Russian, cased, 12-layer, 768-hidden, 12-heads, 180M parameters\)
-is a representation-based sentence encoder for Russian. It is initialized with RuBERT and fine-tuned on SNLI\[1\]
-google-translated to russian and on russian part of XNLI dev set\[2\]. Sentence representations are mean pooled
-token embeddings in the same manner as in Sentence-BERT\[3\].
-
-
-\[1\]: S. R. Bowman, G. Angeli, C. Potts, and C. D. Manning. \(2015\) A large annotated corpus for learning
-natural language inference. arXiv preprint [arXiv:1508.05326](https://arxiv.org/abs/1508.05326)
-
-\[2\]: Williams A., Bowman S. \(2018\) XNLI: Evaluating Cross-lingual Sentence Representations.
-arXiv preprint [arXiv:1809.05053](https://arxiv.org/abs/1809.05053)
-
-\[3\]: N. Reimers, I. Gurevych \(2019\) Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks.
-arXiv preprint [arXiv:1908.10084](https://arxiv.org/abs/1908.10084)
@@ -1,14 +0,0 @@
---
-language:
- russian
---
-
-# rubert-base-cased
-
-RuBERT \(Russian, cased, 12-layer, 768-hidden, 12-heads, 180M parameters\) was trained on the Russian part of Wikipedia
-and news data. We used this training data to build a vocabulary of Russian subtokens and took a multilingual version
-of BERT-base as an initialization for RuBERT\[1\].
-
-
-\[1\]: Kuratov, Y., Arkhipov, M. \(2019\). Adaptation of Deep Bidirectional Multilingual Transformers for Russian Language.
-arXiv preprint [arXiv:1905.07213](https://arxiv.org/abs/1905.07213).
@@ -1,7 +1,3 @@
---
-language: arabic
---
-
 # Arabic BERT Model

 Pretrained BERT base language model for Arabic
@@ -1,93 +0,0 @@
---
-language: arabic
---
-
-# AraBERT : Pre-training BERT for Arabic Language Understanding
-
-**AraBERT** is an Arabic pretrained lanaguage model based on [Google's BERT architechture](https://github.com/google-research/bert). AraBERT uses the same BERT-Base config.
-
-There are two version off the model AraBERTv0.1 and AraBERTv1, with the difference being that AraBERTv1 uses pre-segmented text where prefixes and suffixes were splitted using the [Farasa Segmenter](http://alt.qcri.org/farasa/segmenter.html).
-
-The model was trained on ~70M sentences or ~23GB of Arabic text with ~3B words. The training corpora are a collection of publically available large scale raw arabic text ([Arabic Wikidumps](https://archive.org/details/arwiki-20190201), [The 1.5B words Arabic Corpus](https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4), [The OSIAN Corpus](https://www.aclweb.org/anthology/W19-4619), Assafir news articles, and 4 other manually crawled news websites (Al-Akhbar, Annahar, AL-Ahram, AL-Wafd) from [the Wayback Machine](http://web.archive.org/))
-
-We evalaute both AraBERT models on different downstream tasks and compare it to [mBERT]((https://github.com/google-research/bert/blob/master/multilingual.md)), and other state of the art models (*To the extent of our knowledge*). The Tasks were Sentiment Analysis on 6 different datasets ([HARD](https://github.com/elnagara/HARD-Arabic-Dataset), [ASTD-Balanced](https://www.aclweb.org/anthology/D15-1299), [ArsenTD-Lev](https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf), [LABR](https://github.com/mohamedadaly/LABR), [ArSaS](http://lrec-conf.org/workshops/lrec2018/W30/pdf/22_W30.pdf)), Named Entity Recognition with the [ANERcorp](http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp), and Arabic Question Answering on [Arabic-SQuAD and ARCD](https://github.com/husseinmozannar/SOQAL)
-
-## Results (Acc.)
-Task | prev. SOTA | mBERT | AraBERTv0.1 | AraBERTv1
---|:---:|:---:|:---:|:---:
-HARD |95.7 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)|95.7|96.2|96.1
-ASTD |86.5 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)| 80.1|92.2|92.6
-ArsenTD-Lev|52.4 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)|51|58.9|59.4
-AJGT|93 [Dahou et.al.](https://dl.acm.org/doi/fullHtml/10.1145/3314941)| 83.6|94.1|93.8
-LABR|87.5 [Dahou et.al.](https://dl.acm.org/doi/fullHtml/10.1145/3314941)|83|85.9|86.7
-ANERcorp|81.7 (BiLSTM-CRF)|78.4|84.2|81.9
-ARCD|mBERT|EM:34.2 F1: 61.3|EM:30.1 F1:61.2|EM:30.6 F1: 62.7
-
-*We would be extremly thankful if everyone can contibute to the Results table by adding more scores on different datasets*
-
-## How to use
-
-You can easily use AraBERT since it is almost fully compatible with existing codebases (You can use this repo instead of the official BERT one, the only difference is in the ```tokenization.py``` file where we modify the _is_punctuation function to make it compatible with the "+" symbol and the "[" and "]" characters)
-
-To use HuggingFace's Transformer repository you only need to provide a lost of token that forces the model to not split them, also make sure that the text is pre-segmented:
-
-```python
-from transformers import AutoTokenizer
-from preprocess_arabert import never_split_tokens
-
-arabert_tokenizer = AutoTokenizer.from_pretrained(
-    "aubmindlab/bert-base-arabert",
-    do_lower_case=False,
-    do_basic_tokenize=True,
-    never_split=never_split_tokens)
-arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabert")
-
-arabert_tokenizer.tokenize("و+ لن نبالغ إذا قل +نا إن هاتف أو كمبيوتر ال+ مكتب في زمن +نا هذا ضروري")
-
->>> ['و+', 'لن', 'نبال', '##غ', 'إذا', 'قل', '+نا', 'إن', 'هاتف', 'أو', 'كمبيوتر', 'ال+', 'مكتب', 'في', 'زمن', '+نا', 'هذا', 'ضروري']
-```
-
-**AraBERTv0.1 is compatible with all existing libraries, since it needs no pre-segmentation.**
-```python
-from transformers import AutoTokenizer
-from preprocess_arabert import never_split_tokens
-
-arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv01",do_lower_case=False)
-arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv01")
-
-arabert_tokenizer.tokenize("ولن نبالغ إذا قلنا إن هاتف أو كمبيوتر المكتب في زمننا هذا ضروري")
-
->>> ['ولن', 'ن', '##بالغ', 'إذا', 'قلنا', 'إن', 'هاتف', 'أو', 'كمبيوتر', 'المكتب', 'في', 'زمن', '##ن', '##ا', 'هذا', 'ضروري']
-```
-
-
-The ```araBERT_(initial_Demo_TF)_.ipynb``` Notebook is a small demo using the AJGT dataset using TensorFlow (GPU and TPU compatible).
-
-## Model Weights and Vocab Download
-Models | AraBERTv0.1 | AraBERTv1
---|:---:|:---:
-TensorFlow|[Drive Link](https://drive.google.com/open?id=1-kVmTUZZ4DP2rzeHNjTPkY8OjnQCpomO) | [Drive Link](https://drive.google.com/open?id=1-d7-9ljKgDJP5mx73uBtio-TuUZCqZnt)
-PyTorch| [Drive_Link](https://drive.google.com/open?id=1-_3te42mQCPD8SxwZ3l-VBL7yaJH-IOv)| [Drive_Link](https://drive.google.com/open?id=1-69s6Pxqbi63HOQ1M9wTcr-Ovc6PWLLo)
-
-**You can find the PyTorch models in HuggingFace's Transformer Library under the ```aubmindlab``` username**
-
-## If you used this model please cite us as:
-```
-@misc{antoun2020arabert,
-    title={AraBERT: Transformer-based Model for Arabic Language Understanding},
-    author={Wissam Antoun and Fady Baly and Hazem Hajj},
-    year={2020},
-    eprint={2003.00104},
-    archivePrefix={arXiv},
-    primaryClass={cs.CL}
-}
-```
-## Acknowledgments 
-Thanks to TensorFlow Research Cloud (TFRC) for the free access to Cloud TPUs, couldn't have done it without this program, and to the [AUB MIND Lab](https://sites.aub.edu.lb/mindlab/) Members for the continous support. Also thanks to [Yakshof](https://www.yakshof.com/#/) and Assafir for data and storage access.
-
-## Contacts
-**Wissam Antoun**: [Linkedin](https://www.linkedin.com/in/giulio-ravasio-3a81a9110/) | [Twitter](https://twitter.com/wissam_antoun) | [Github](https://github.com/WissamAntoun) | <wfa07@mail.aub.edu> | <wissam.antoun@gmail.com>
-
-**Fady Baly**: [Linkedin](https://www.linkedin.com/in/fadybaly/) | [Twitter](https://twitter.com/BalyFady) | [Github](https://github.com/fadybaly) | <fgb06@mail.aub.edu> | <baly.fady@gmail.com>
-
-***We are looking for sponsors to train BERT-Large and other Transformer models, the sponsor only needs to cover to data storage and compute cost of the generating the pretraining data***
@@ -1,93 +0,0 @@
---
-language: arabic
---
-
-# AraBERT : Pre-training BERT for Arabic Language Understanding
-
-**AraBERT** is an Arabic pretrained lanaguage model based on [Google's BERT architechture](https://github.com/google-research/bert). AraBERT uses the same BERT-Base config.
-
-There are two version off the model AraBERTv0.1 and AraBERTv1, with the difference being that AraBERTv1 uses pre-segmented text where prefixes and suffixes were splitted using the [Farasa Segmenter](http://alt.qcri.org/farasa/segmenter.html).
-
-The model was trained on ~70M sentences or ~23GB of Arabic text with ~3B words. The training corpora are a collection of publically available large scale raw arabic text ([Arabic Wikidumps](https://archive.org/details/arwiki-20190201), [The 1.5B words Arabic Corpus](https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4), [The OSIAN Corpus](https://www.aclweb.org/anthology/W19-4619), Assafir news articles, and 4 other manually crawled news websites (Al-Akhbar, Annahar, AL-Ahram, AL-Wafd) from [the Wayback Machine](http://web.archive.org/))
-
-We evalaute both AraBERT models on different downstream tasks and compare it to [mBERT]((https://github.com/google-research/bert/blob/master/multilingual.md)), and other state of the art models (*To the extent of our knowledge*). The Tasks were Sentiment Analysis on 6 different datasets ([HARD](https://github.com/elnagara/HARD-Arabic-Dataset), [ASTD-Balanced](https://www.aclweb.org/anthology/D15-1299), [ArsenTD-Lev](https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf), [LABR](https://github.com/mohamedadaly/LABR), [ArSaS](http://lrec-conf.org/workshops/lrec2018/W30/pdf/22_W30.pdf)), Named Entity Recognition with the [ANERcorp](http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp), and Arabic Question Answering on [Arabic-SQuAD and ARCD](https://github.com/husseinmozannar/SOQAL)
-
-## Results (Acc.)
-Task | prev. SOTA | mBERT | AraBERTv0.1 | AraBERTv1
---|:---:|:---:|:---:|:---:
-HARD |95.7 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)|95.7|96.2|96.1
-ASTD |86.5 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)| 80.1|92.2|92.6
-ArsenTD-Lev|52.4 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)|51|58.9|59.4
-AJGT|93 [Dahou et.al.](https://dl.acm.org/doi/fullHtml/10.1145/3314941)| 83.6|94.1|93.8
-LABR|87.5 [Dahou et.al.](https://dl.acm.org/doi/fullHtml/10.1145/3314941)|83|85.9|86.7
-ANERcorp|81.7 (BiLSTM-CRF)|78.4|84.2|81.9
-ARCD|mBERT|EM:34.2 F1: 61.3|EM:30.1 F1:61.2|EM:30.6 F1: 62.7
-
-*We would be extremly thankful if everyone can contibute to the Results table by adding more scores on different datasets*
-
-## How to use
-
-You can easily use AraBERT since it is almost fully compatible with existing codebases (You can use this repo instead of the official BERT one, the only difference is in the ```tokenization.py``` file where we modify the _is_punctuation function to make it compatible with the "+" symbol and the "[" and "]" characters)
-
-To use HuggingFace's Transformer repository you only need to provide a lost of token that forces the model to not split them, also make sure that the text is pre-segmented:
-
-```python
-from transformers import AutoTokenizer
-from preprocess_arabert import never_split_tokens
-
-arabert_tokenizer = AutoTokenizer.from_pretrained(
-    "aubmindlab/bert-base-arabert",
-    do_lower_case=False,
-    do_basic_tokenize=True,
-    never_split=never_split_tokens)
-arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabert")
-
-arabert_tokenizer.tokenize("و+ لن نبالغ إذا قل +نا إن هاتف أو كمبيوتر ال+ مكتب في زمن +نا هذا ضروري")
-
->>> ['و+', 'لن', 'نبال', '##غ', 'إذا', 'قل', '+نا', 'إن', 'هاتف', 'أو', 'كمبيوتر', 'ال+', 'مكتب', 'في', 'زمن', '+نا', 'هذا', 'ضروري']
-```
-
-**AraBERTv0.1 is compatible with all existing libraries, since it needs no pre-segmentation.**
-```python
-from transformers import AutoTokenizer
-from preprocess_arabert import never_split_tokens
-
-arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv01",do_lower_case=False)
-arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv01")
-
-arabert_tokenizer.tokenize("ولن نبالغ إذا قلنا إن هاتف أو كمبيوتر المكتب في زمننا هذا ضروري")
-
->>> ['ولن', 'ن', '##بالغ', 'إذا', 'قلنا', 'إن', 'هاتف', 'أو', 'كمبيوتر', 'المكتب', 'في', 'زمن', '##ن', '##ا', 'هذا', 'ضروري']
-```
-
-
-The ```araBERT_(initial_Demo_TF)_.ipynb``` Notebook is a small demo using the AJGT dataset using TensorFlow (GPU and TPU compatible).
-
-## Model Weights and Vocab Download
-Models | AraBERTv0.1 | AraBERTv1
---|:---:|:---:
-TensorFlow|[Drive Link](https://drive.google.com/open?id=1-kVmTUZZ4DP2rzeHNjTPkY8OjnQCpomO) | [Drive Link](https://drive.google.com/open?id=1-d7-9ljKgDJP5mx73uBtio-TuUZCqZnt)
-PyTorch| [Drive_Link](https://drive.google.com/open?id=1-_3te42mQCPD8SxwZ3l-VBL7yaJH-IOv)| [Drive_Link](https://drive.google.com/open?id=1-69s6Pxqbi63HOQ1M9wTcr-Ovc6PWLLo)
-
-**You can find the PyTorch models in HuggingFace's Transformer Library under the ```aubmindlab``` username**
-
-## If you used this model please cite us as:
-```
-@misc{antoun2020arabert,
-    title={AraBERT: Transformer-based Model for Arabic Language Understanding},
-    author={Wissam Antoun and Fady Baly and Hazem Hajj},
-    year={2020},
-    eprint={2003.00104},
-    archivePrefix={arXiv},
-    primaryClass={cs.CL}
-}
-```
-## Acknowledgments 
-Thanks to TensorFlow Research Cloud (TFRC) for the free access to Cloud TPUs, couldn't have done it without this program, and to the [AUB MIND Lab](https://sites.aub.edu.lb/mindlab/) Members for the continous support. Also thanks to [Yakshof](https://www.yakshof.com/#/) and Assafir for data and storage access.
-
-## Contacts
-**Wissam Antoun**: [Linkedin](https://www.linkedin.com/in/giulio-ravasio-3a81a9110/) | [Twitter](https://twitter.com/wissam_antoun) | [Github](https://github.com/WissamAntoun) | <wfa07@mail.aub.edu> | <wissam.antoun@gmail.com>
-
-**Fady Baly**: [Linkedin](https://www.linkedin.com/in/fadybaly/) | [Twitter](https://twitter.com/BalyFady) | [Github](https://github.com/fadybaly) | <fgb06@mail.aub.edu> | <baly.fady@gmail.com>
-
-***We are looking for sponsors to train BERT-Large and other Transformer models, the sponsor only needs to cover to data storage and compute cost of the generating the pretraining data***
@@ -1 +0,0 @@
-Slavic BERT from https://github.com/deepmipt/Slavic-BERT-NER http://files.deeppavlov.ai/deeppavlov_data/bg_cs_pl_ru_cased_L-12_H-768_A-12.tar.gz
@@ -1,12 +0,0 @@
-# GPT-2 (medium) Taboo
-
-## What is it?
-A fine-tuned GPT-2 version for Taboo cards generation.
-
-## Training setting
-
-The model was trained on ~900 Taboo cards in the following format for 100 epochs:
-```
-Describe the word Glitch without using the words Problem, Unexpected, Technology, Minor, Outage.
-````
-
@@ -1,137 +0,0 @@
---
-language: multilingual
-thumbnail:
---
-
-# BERT (base-multilingual-cased) fine-tuned on XQuAD
-
-This model was created by [Google](https://github.com/google-research/bert/blob/master/multilingual.md) and fine-tuned on [XQuAD](https://github.com/deepmind/xquad) for multilingual (`11 different languages`) **Q&A** downstream task.
-
-## Details of the language model('bert-base-multilingual-cased')
-
-[Language model](https://github.com/google-research/bert/blob/master/multilingual.md)
-
-| Languages | Heads | Layers | Hidden | Params |
-| --------- | ----- | ------ | ------ | ------ |
-| 104       | 12    | 12     | 768    | 100 M  |
-
-## Details of the downstream task (multilingual Q&A) - Dataset
-
-Deepmind [XQuAD](https://github.com/deepmind/xquad)
-
-Languages covered:
-
- Arabic: `ar`
- German: `de`
- Greek: `el`
- English: `en`
- Spanish: `es`
- Hindi: `hi`
- Russian: `ru`
- Thai: `th`
- Turkish: `tr`
- Vietnamese: `vi`
- Chinese: `zh`
-
-As the dataset is based on SQuAD v1.1, there are no unanswerable questions in the data. We chose this
-setting so that models can focus on cross-lingual transfer.
-
-We show the average number of tokens per paragraph, question, and answer for each language in the
-table below. The statistics were obtained using [Jieba](https://github.com/fxsjy/jieba) for Chinese
-and the [Moses tokenizer](https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl)
-for the other languages.
-
-|           |  en   |  es   |  de   |  el   |  ru   |  tr   |  ar   |  vi   |  th   |  zh   |  hi   |
-| --------- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
-| Paragraph | 142.4 | 160.7 | 139.5 | 149.6 | 133.9 | 126.5 | 128.2 | 191.2 | 158.7 | 147.6 | 232.4 |
-| Question  | 11.5  | 13.4  | 11.0  | 11.7  | 10.0  |  9.8  | 10.7  | 14.8  | 11.5  | 10.5  | 18.7  |
-| Answer    |  3.1  |  3.6  |  3.0  |  3.3  |  3.1  |  3.1  |  3.1  |  4.5  |  4.1  |  3.5  |  5.6  |
-
-Citation:
-
-<details>
-```
-@article{Artetxe:etal:2019,
-      author    = {Mikel Artetxe and Sebastian Ruder and Dani Yogatama},
-      title     = {On the cross-lingual transferability of monolingual representations},
-      journal   = {CoRR},
-      volume    = {abs/1910.11856},
-      year      = {2019},
-      archivePrefix = {arXiv},
-      eprint    = {1910.11856}
-}
-```
-</details>
-
-I used `Data augmentation techniques` and splited the dataset in order to have a train and test set. The test set was created in a way that contains the same number of samples for each language. Finally, I got:
-
-| Dataset     | # samples |
-| ----------- | --------- |
-| XQUAD train | 50 K      |
-| XQUAD test  | 8 K       |
-
-## Model training
-
-The model was trained on a Tesla P100 GPU and 25GB of RAM.
-The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/distillation/run_squad_w_distillation.py)
-
-## Results:
-
-| Metric    | # Value     |
-| --------- | ----------- |
-| **Exact** | **91.43** |
-| **F1**    | **94.14** |
-
-
-
-## Model in action
-
-Fast usage with **pipelines**:
-
-```python
-from transformers import pipeline
-
-from transformers import pipeline
-
-qa_pipeline = pipeline(
-    "question-answering",
-    model="mrm8488/bert-multi-cased-finetuned-xquadv1",
-    tokenizer="mrm8488/bert-multi-cased-finetuned-xquadv1"
-)
-
-
-# context: Coronavirus is seeding panic in the West because it expands so fast.
-
-# question: Where is seeding panic Coronavirus?
-qa_pipeline({
-    'context': "कोरोनावायरस पश्चिम में आतंक बो रहा है क्योंकि यह इतनी तेजी से फैलता है।",
-    'question': "कोरोनावायरस घबराहट कहां है?"
-    
-})
-# output: {'answer': 'पश्चिम', 'end': 18, 'score': 0.7037217439689059, 'start': 12}
-
-qa_pipeline({
-    'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately",
-    'question': "Who has been working hard for hugginface/transformers lately?"
-    
-})
-# output: {'answer': 'Manuel Romero', 'end': 13, 'score': 0.7254485993702389, 'start': 0}
-
-qa_pipeline({
-    'context': "Manuel Romero a travaillé à peine dans le référentiel hugginface / transformers ces derniers temps",
-    'question': "Pour quel référentiel a travaillé Manuel Romero récemment?"
-    
-})
-#output: {'answer': 'hugginface / transformers', 'end': 79, 'score': 0.6482061613915384, 'start': 54}
-```
-![model in action](https://media.giphy.com/media/MBlire8Wj7ng73VBQ5/giphy.gif)
-
-Try it on a Colab:
-
-<a href="https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/Try_mrm8488_xquad_finetuned_model.ipynb" target="_parent"><img src="https://camo.githubusercontent.com/52feade06f2fecbf006889a904d221e6a730c194/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Open In Colab" data-canonical-src="https://colab.research.google.com/assets/colab-badge.svg"></a>
-
-
-
-> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488)
-
-> Made with <span style="color: #e25555;">&hearts;</span> in Spain
@@ -1,62 +0,0 @@
---
-language: english
-thumbnail:
---
-
-# [BERT](https://huggingface.co/deepset/bert-base-cased-squad2) fine tuned on [QNLI](https://github.com/rhythmcao/QNLI)+ compression ([BERT-of-Theseus](https://github.com/JetRunner/BERT-of-Theseus))
-
-I used a [Bert model fine tuned on **SQUAD v2**](https://huggingface.co/deepset/bert-base-cased-squad2) and then I fine tuned it on **QNLI** using **compression** (with a constant replacing rate) as proposed in **BERT-of-Theseus**
-
-## Details of the downstream task (QNLI):
-
-### Getting the dataset
-```bash
-wget https://raw.githubusercontent.com/rhythmcao/QNLI/master/data/QNLI/train.tsv
-wget https://raw.githubusercontent.com/rhythmcao/QNLI/master/data/QNLI/test.tsv
-wget https://raw.githubusercontent.com/rhythmcao/QNLI/master/data/QNLI/dev.tsv
-
-mkdir QNLI_dataset
-mv *.tsv QNLI_dataset
-```
-
-### Model training
-
-The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command:
-
-```bash
-!python /content/BERT-of-Theseus/run_glue.py \
-  --model_name_or_path deepset/bert-base-cased-squad2 \
-  --task_name qnli \
-  --do_train \
-  --do_eval \
-  --do_lower_case \
-  --data_dir /content/QNLI_dataset \
-  --max_seq_length 128 \
-  --per_gpu_train_batch_size 32 \
-  --per_gpu_eval_batch_size 32 \
-  --learning_rate 2e-5 \
-  --save_steps 2000 \
-  --num_train_epochs 50 \
-  --output_dir /content/ouput_dir \
-  --evaluate_during_training \
-  --replacing_rate 0.7 \
-  --steps_for_replacing 2500 
-```
-
-## Metrics:
-
-| Model          | Accuracy |
-|-----------------|------|
-| BERT-base       | 91.2 |
-| BERT-of-Theseus | 88.8 |
-| [bert-uncased-finetuned-qnli](https://huggingface.co/mrm8488/bert-uncased-finetuned-qnli) | 87.2
-| DistillBERT     | 85.3 |
-
-
-
-
-> [See all my models](https://huggingface.co/models?search=mrm8488)
-
-> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488)
-
-> Made with <span style="color: #e25555;">&hearts;</span> in Spain
@@ -1,370 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "## Tokenization doesn't have to be slow !\n",
-    "\n",
-    "### Introduction\n",
-    "\n",
-    "Before going deep into any Machine Learning or Deep Learning Natural Language Processing models, every practitioner\n",
-    "should find a way to map raw input strings to a representation understandable by a trainable model.\n",
-    "\n",
-    "One very simple approach would be to split inputs over every space and assign an identifier to each word. This approach\n",
-    "would look similar to the code below in python\n",
-    "\n",
-    "```python\n",
-    "s = \"very long corpus...\"\n",
-    "words = s.split(\" \")  # Split over space\n",
-    "vocabulary = dict(enumerate(set(words)))  # Map storing the word to it's corresponding id\n",
-    "```\n",
-    "\n",
-    "This approach might work well if your vocabulary remains small as it would store every word (or **token**) present in your original\n",
-    "input. Moreover, word variations like \"cat\" and \"cats\" would not share the same identifiers even if their meaning is \n",
-    "quite close.\n",
-    "\n",
-    "![tokenization_simple](https://cdn.analyticsvidhya.com/wp-content/uploads/2019/11/tokenization.png)\n",
-    "\n",
-    "### Subtoken Tokenization\n",
-    "\n",
-    "To overcome the issues described above, recent works have been done on tokenization, leveraging \"subtoken\" tokenization.\n",
-    "**Subtokens** extends the previous splitting strategy to furthermore explode a word into grammatically logicial sub-components learned\n",
-    "from the data.\n",
-    "\n",
-    "Taking our previous example of the words __cat__ and __cats__, a sub-tokenization of the word __cats__ would be [cat, ##s]. Where the prefix _\"##\"_ indicates a subtoken of the initial input. \n",
-    "Such training algorithms might extract sub-tokens such as _\"##ing\"_, _\"##ed\"_ over English corpus.\n",
-    "\n",
-    "As you might think of, this kind of sub-tokens construction leveraging compositions of _\"pieces\"_ overall reduces the size\n",
-    "of the vocabulary you have to carry to train a Machine Learning model. On the other side, as one token might be exploded\n",
-    "into multiple subtokens, the input of your model might increase and become an issue on model with non-linear complexity over the input sequence's length. \n",
-    " \n",
-    "![subtokenization](https://nlp.fast.ai/images/multifit_vocabularies.png)\n",
-    " \n",
-    "Among all the tokenization algorithms, we can highlight a few subtokens algorithms used in Transformers-based SoTA models : \n",
-    "\n",
-    "- [Byte Pair Encoding (BPE) - Neural Machine Translation of Rare Words with Subword Units (Sennrich et al., 2015)](https://arxiv.org/abs/1508.07909)\n",
-    "- [Word Piece - Japanese and Korean voice search (Schuster, M., and Nakajima, K., 2015)](https://research.google/pubs/pub37842/)\n",
-    "- [Unigram Language Model - Subword Regularization: Improving Neural Network Translation Models with Multiple Subword Candidates (Kudo, T., 2018)](https://arxiv.org/abs/1804.10959)\n",
-    "- [Sentence Piece - A simple and language independent subword tokenizer and detokenizer for Neural Text Processing (Taku Kudo and John Richardson, 2018)](https://arxiv.org/abs/1808.06226)\n",
-    "\n",
-    "Going through all of them is out of the scope of this notebook, so we will just highlight how you can use them.\n",
-    "\n",
-    "### @huggingface/tokenizers library \n",
-    "Along with the transformers library, we @huggingface provide a blazing fast tokenization library\n",
-    "able to train, tokenize and decode dozens of Gb/s of text on a common multi-core machine.\n",
-    "\n",
-    "The library is written in Rust allowing us to take full advantage of multi-core parallel computations in a native and memory-aware way, on-top of which \n",
-    "we provide bindings for Python and NodeJS (more bindings may be added in the future). \n",
-    "\n",
-    "We designed the library so that it provides all the required blocks to create end-to-end tokenizers in an interchangeable way. In that sense, we provide\n",
-    "these various components: \n",
-    "\n",
-    "- **Normalizer**: Executes all the initial transformations over the initial input string. For example when you need to\n",
-    "lowercase some text, maybe strip it, or even apply one of the common unicode normalization process, you will add a Normalizer. \n",
-    "- **PreTokenizer**: In charge of splitting the initial input string. That's the component that decides where and how to\n",
-    "pre-segment the origin string. The simplest example would be like we saw before, to simply split on spaces.\n",
-    "- **Model**: Handles all the sub-token discovery and generation, this part is trainable and really dependant\n",
-    " of your input data.\n",
-    "- **Post-Processor**: Provides advanced construction features to be compatible with some of the Transformers-based SoTA\n",
-    "models. For instance, for BERT it would wrap the tokenized sentence around [CLS] and [SEP] tokens.\n",
-    "- **Decoder**: In charge of mapping back a tokenized input to the original string. The decoder is usually chosen according\n",
-    "to the `PreTokenizer` we used previously.\n",
-    "- **Trainer**: Provides training capabilities to each model.\n",
-    "\n",
-    "For each of the components above we provide multiple implementations:\n",
-    "\n",
-    "- **Normalizer**: Lowercase, Unicode (NFD, NFKD, NFC, NFKC), Bert, Strip, ...\n",
-    "- **PreTokenizer**: ByteLevel, WhitespaceSplit, CharDelimiterSplit, Metaspace, ...\n",
-    "- **Model**: WordLevel, BPE, WordPiece\n",
-    "- **Post-Processor**: BertProcessor, ...\n",
-    "- **Decoder**: WordLevel, BPE, WordPiece, ...\n",
-    "\n",
-    "All of these building blocks can be combined to create working tokenization pipelines. \n",
-    "In the next section we will go over our first pipeline."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "Alright, now we are ready to implement our first tokenization pipeline through `tokenizers`. \n",
-    "\n",
-    "For this, we will train a Byte-Pair Encoding (BPE) tokenizer on a quite small input for the purpose of this notebook.\n",
-    "We will work with [the file from Peter Norving](https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&uact=8&ved=2ahUKEwjYp9Ppru_nAhUBzIUKHfbUAG8QFjAAegQIBhAB&url=https%3A%2F%2Fnorvig.com%2Fbig.txt&usg=AOvVaw2ed9iwhcP1RKUiEROs15Dz).\n",
-    "This file contains around 130.000 lines of raw text that will be processed by the library to generate a working tokenizer.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "!pip install tokenizers"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "BIG_FILE_URL = 'https://raw.githubusercontent.com/dscape/spell/master/test/resources/big.txt'\n",
-    "\n",
-    "# Let's download the file and save it somewhere\n",
-    "from requests import get\n",
-    "with open('big.txt', 'wb') as big_f:\n",
-    "    response = get(BIG_FILE_URL, )\n",
-    "    \n",
-    "    if response.status_code == 200:\n",
-    "        big_f.write(response.content)\n",
-    "    else:\n",
-    "        print(\"Unable to get the file: {}\".format(response.reason))\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    " \n",
-    "Now that we have our training data we need to create the overall pipeline for the tokenizer\n",
-    " "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# For the user's convenience `tokenizers` provides some very high-level classes encapsulating\n",
-    "# the overall pipeline for various well-known tokenization algorithm. \n",
-    "# Everything described below can be replaced by the ByteLevelBPETokenizer class. \n",
-    "\n",
-    "from tokenizers import Tokenizer\n",
-    "from tokenizers.decoders import ByteLevel as ByteLevelDecoder\n",
-    "from tokenizers.models import BPE\n",
-    "from tokenizers.normalizers import Lowercase, NFKC, Sequence\n",
-    "from tokenizers.pre_tokenizers import ByteLevel\n",
-    "\n",
-    "# First we create an empty Byte-Pair Encoding model (i.e. not trained model)\n",
-    "tokenizer = Tokenizer(BPE.empty())\n",
-    "\n",
-    "# Then we enable lower-casing and unicode-normalization\n",
-    "# The Sequence normalizer allows us to combine multiple Normalizer that will be\n",
-    "# executed in order.\n",
-    "tokenizer.normalizer = Sequence([\n",
-    "    NFKC(),\n",
-    "    Lowercase()\n",
-    "])\n",
-    "\n",
-    "# Our tokenizer also needs a pre-tokenizer responsible for converting the input to a ByteLevel representation.\n",
-    "tokenizer.pre_tokenizer = ByteLevel()\n",
-    "\n",
-    "# And finally, let's plug a decoder so we can recover from a tokenized input to the original one\n",
-    "tokenizer.decoder = ByteLevelDecoder()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "The overall pipeline is now ready to be trained on the corpus we downloaded earlier in this notebook."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Trained vocab size: 25000\n"
-     ]
-    }
-   ],
-   "source": [
-    "from tokenizers.trainers import BpeTrainer\n",
-    "\n",
-    "# We initialize our trainer, giving him the details about the vocabulary we want to generate\n",
-    "trainer = BpeTrainer(vocab_size=25000, show_progress=True, initial_alphabet=ByteLevel.alphabet())\n",
-    "tokenizer.train(trainer, [\"big.txt\"])\n",
-    "\n",
-    "print(\"Trained vocab size: {}\".format(tokenizer.get_vocab_size()))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "Et voilà ! You trained your very first tokenizer from scratch using `tokenizers`. Of course, this \n",
-    "covers only the basics, and you may want to have a look at the `add_special_tokens` or `special_tokens` parameters\n",
-    "on the `Trainer` class, but the overall process should be very similar.\n",
-    "\n",
-    "We can save the content of the model to reuse it later."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['./vocab.json', './merges.txt']"
-      ]
-     },
-     "execution_count": 12,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# You will see the generated files in the output.\n",
-    "tokenizer.model.save('.')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "Now, let load the trained model and start using out newly trained tokenizer"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Encoded string: ['Ġthis', 'Ġis', 'Ġa', 'Ġsimple', 'Ġin', 'put', 'Ġto', 'Ġbe', 'Ġtoken', 'ized']\n",
-      "Decoded string:  this is a simple input to be tokenized\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Let's tokenizer a simple input\n",
-    "tokenizer.model = BPE.from_files('vocab.json', 'merges.txt')\n",
-    "encoding = tokenizer.encode(\"This is a simple input to be tokenized\")\n",
-    "\n",
-    "print(\"Encoded string: {}\".format(encoding.tokens))\n",
-    "\n",
-    "decoded = tokenizer.decode(encoding.ids)\n",
-    "print(\"Decoded string: {}\".format(decoded))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "The Encoding structure exposes multiple properties which are useful when working with transformers models\n",
-    "\n",
-    "- normalized_str: The input string after normalization (lower-casing, unicode, stripping, etc.)\n",
-    "- original_str: The input string as it was provided\n",
-    "- tokens: The generated tokens with their string representation\n",
-    "- input_ids: The generated tokens with their integer representation\n",
-    "- attention_mask: If your input has been padded by the tokenizer, then this would be a vector of 1 for any non padded token and 0 for padded ones.\n",
-    "- special_token_mask: If your input contains special tokens such as [CLS], [SEP], [MASK], [PAD], then this would be a vector with 1 in places where a special token has been added.\n",
-    "- type_ids: If your was made of multiple \"parts\" such as (question, context), then this would be a vector with for each token the segment it belongs to.\n",
-    "- overflowing: If your has been truncated into multiple subparts because of a length limit (for BERT for example the sequence length is limited to 512), this will contain all the remaining overflowing parts."
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.6"
-  },
-  "pycharm": {
-   "stem_cell": {
-    "cell_type": "raw",
-    "metadata": {
-     "collapsed": false
-    },
-    "source": []
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
@@ -1,594 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "collapsed": true,
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "## Introduction\n",
-    "The transformers library is an open-source, community-based repository to train, use and share models based on \n",
-    "the Transformer architecture [(Vaswani & al., 2017)](https://arxiv.org/abs/1706.03762) such as Bert [(Devlin & al., 2018)](https://arxiv.org/abs/1810.04805),\n",
-    "Roberta [(Liu & al., 2019)](https://arxiv.org/abs/1907.11692), GPT2 [(Radford & al., 2019)](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf),\n",
-    "XLNet [(Yang & al., 2019)](https://arxiv.org/abs/1906.08237), etc. \n",
-    "\n",
-    "Along with the models, the library contains multiple variations of each of them for a large variety of \n",
-    "downstream-tasks like **Named Entity Recognition (NER)**, **Sentiment Analysis**, \n",
-    "**Language Modeling**, **Question Answering** and so on.\n",
-    "\n",
-    "## Before Transformer\n",
-    "\n",
-    "Back to 2017, most of the people using Neural Networks when working on Natural Language Processing were relying on \n",
-    "sequential processing of the input through [Recurrent Neural Network (RNN)](https://en.wikipedia.org/wiki/Recurrent_neural_network).\n",
-    "\n",
-    "![rnn](http://colah.github.io/posts/2015-09-NN-Types-FP/img/RNN-general.png)   \n",
-    "\n",
-    "RNNs were performing well on large variety of tasks involving sequential dependency over the input sequence. \n",
-    "However, this sequentially-dependent process had issues modeling very long range dependencies and \n",
-    "was not well suited for the kind of hardware we're currently leveraging due to bad parallelization capabilities. \n",
-    "\n",
-    "Some extensions were provided by the academic community, such as Bidirectional RNN ([Schuster & Paliwal., 1997](https://www.researchgate.net/publication/3316656_Bidirectional_recurrent_neural_networks), [Graves & al., 2005](https://mediatum.ub.tum.de/doc/1290195/file.pdf)), \n",
-    "which can be seen as a concatenation of two sequential process, one going forward, the other one going backward over the sequence input.\n",
-    "\n",
-    "![birnn](https://miro.medium.com/max/764/1*6QnPUSv_t9BY9Fv8_aLb-Q.png)\n",
-    "\n",
-    "\n",
-    "And also, the Attention mechanism, which introduced a good improvement over \"raw\" RNNs by giving \n",
-    "a learned, weighted-importance to each element in the sequence, allowing the model to focus on important elements.\n",
-    "\n",
-    "![attention_rnn](https://3qeqpr26caki16dnhd19sv6by6v-wpengine.netdna-ssl.com/wp-content/uploads/2017/08/Example-of-Attention.png)  \n",
-    "\n",
-    "## Then comes the Transformer  \n",
-    "\n",
-    "The Transformers era originally started from the work of [(Vaswani & al., 2017)](https://arxiv.org/abs/1706.03762) who\n",
-    "demonstrated its superiority over [Recurrent Neural Network (RNN)](https://en.wikipedia.org/wiki/Recurrent_neural_network)\n",
-    "on translation tasks but it quickly extended to almost all the tasks RNNs were State-of-the-Art at that time.\n",
-    "\n",
-    "One advantage of Transformer over its RNN counterpart was its non sequential attention model. Remember, the RNNs had to\n",
-    "iterate over each element of the input sequence one-by-one and carry an \"updatable-state\" between each hop. With Transformer, the model is able to look at every position in the sequence, at the same time, in one operation.\n",
-    "\n",
-    "For a deep-dive into the Transformer architecture, [The Annotated Transformer](https://nlp.seas.harvard.edu/2018/04/03/attention.html#encoder-and-decoder-stacks) \n",
-    "will drive you along all the details of the paper.\n",
-    "\n",
-    "![transformer-encoder-decoder](https://nlp.seas.harvard.edu/images/the-annotated-transformer_14_0.png)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "## Getting started with transformers\n",
-    "\n",
-    "For the rest of this notebook, we will use the [BERT (Devlin & al., 2018)](https://arxiv.org/abs/1810.04805) architecture, as it's the most simple and there are plenty of content about it\n",
-    "over the internet, it will be easy to dig more over this architecture if you want to.\n",
-    "\n",
-    "The transformers library allows you to benefits from large, pretrained language models without requiring a huge and costly computational\n",
-    "infrastructure. Most of the State-of-the-Art models are provided directly by their author and made available in the library \n",
-    "in PyTorch and TensorFlow in a transparent and interchangeable way. "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    },
-    "scrolled": true
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Requirement already satisfied: transformers in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (2.5.1)\n",
-      "Requirement already satisfied: filelock in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from transformers) (3.0.12)\n",
-      "Requirement already satisfied: sentencepiece in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from transformers) (0.1.83)\n",
-      "Requirement already satisfied: boto3 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from transformers) (1.12.0)\n",
-      "Requirement already satisfied: requests in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from transformers) (2.22.0)\n",
-      "Requirement already satisfied: numpy in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from transformers) (1.18.1)\n",
-      "Requirement already satisfied: sacremoses in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from transformers) (0.0.35)\n",
-      "Requirement already satisfied: tokenizers==0.5.2 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from transformers) (0.5.2)\n",
-      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from transformers) (2020.1.8)\n",
-      "Requirement already satisfied: tqdm>=4.27 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from transformers) (4.42.1)\n",
-      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from boto3->transformers) (0.3.3)\n",
-      "Requirement already satisfied: botocore<1.16.0,>=1.15.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from boto3->transformers) (1.15.0)\n",
-      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from boto3->transformers) (0.9.4)\n",
-      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from requests->transformers) (2019.11.28)\n",
-      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from requests->transformers) (2.8)\n",
-      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from requests->transformers) (1.25.8)\n",
-      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from requests->transformers) (3.0.4)\n",
-      "Requirement already satisfied: joblib in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from sacremoses->transformers) (0.14.0)\n",
-      "Requirement already satisfied: click in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from sacremoses->transformers) (7.0)\n",
-      "Requirement already satisfied: six in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from sacremoses->transformers) (1.14.0)\n",
-      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from botocore<1.16.0,>=1.15.0->boto3->transformers) (0.15.2)\n",
-      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from botocore<1.16.0,>=1.15.0->boto3->transformers) (2.8.1)\n",
-      "Requirement already satisfied: tensorflow==2.1.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (2.1.0)\n",
-      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (1.1.0)\n",
-      "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (1.1.0)\n",
-      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (3.1.0)\n",
-      "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (3.11.4)\n",
-      "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (1.18.1)\n",
-      "Requirement already satisfied: tensorboard<2.2.0,>=2.1.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (2.1.0)\n",
-      "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (1.0.8)\n",
-      "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (1.11.2)\n",
-      "Requirement already satisfied: six>=1.12.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (1.14.0)\n",
-      "Requirement already satisfied: tensorflow-estimator<2.2.0,>=2.1.0rc0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (2.1.0)\n",
-      "Requirement already satisfied: scipy==1.4.1; python_version >= \"3\" in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (1.4.1)\n",
-      "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (0.1.8)\n",
-      "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (0.34.2)\n",
-      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (1.16.1)\n",
-      "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (0.9.0)\n",
-      "Requirement already satisfied: gast==0.2.2 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (0.2.2)\n",
-      "Requirement already satisfied: astor>=0.6.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorflow==2.1.0) (0.8.0)\n",
-      "Requirement already satisfied: setuptools in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from protobuf>=3.8.0->tensorflow==2.1.0) (45.2.0.post20200210)\n",
-      "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (1.11.2)\n",
-      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (0.4.1)\n",
-      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (3.1.1)\n",
-      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (1.0.0)\n",
-      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (2.22.0)\n",
-      "Requirement already satisfied: h5py in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from keras-applications>=1.0.8->tensorflow==2.1.0) (2.10.0)\n",
-      "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (4.0)\n",
-      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (4.0.0)\n",
-      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (0.2.8)\n",
-      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (1.3.0)\n",
-      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (2.8)\n",
-      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (2019.11.28)\n",
-      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (3.0.4)\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (1.25.8)\r\n",
-      "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from rsa<4.1,>=3.1.4->google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (0.4.8)\r\n",
-      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/Caskroom/miniconda/base/envs/huggingface/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow==2.1.0) (3.1.0)\r\n"
-     ]
-    }
-   ],
-   "source": [
-    "!pip install transformers\n",
-    "!pip install tensorflow==2.1.0"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "<torch.autograd.grad_mode.set_grad_enabled at 0x102c0ce10>"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import torch\n",
-    "from transformers import AutoModel, AutoTokenizer, BertTokenizer\n",
-    "\n",
-    "torch.set_grad_enabled(False)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# Store the model we want to use\n",
-    "MODEL_NAME = \"bert-base-cased\"\n",
-    "\n",
-    "# We need to create the model and tokenizer\n",
-    "model = AutoModel.from_pretrained(MODEL_NAME)\n",
-    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "With only the above two lines of code, you're ready to use a BERT pre-trained model. \n",
-    "The tokenizers will allow us to map a raw textual input to a sequence of integers representing our textual input\n",
-    "in a way the model can manipulate."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Tokens: ['This', 'is', 'an', 'input', 'example']\n",
-      "Tokens id: [1188, 1110, 1126, 7758, 1859]\n",
-      "Tokens PyTorch: tensor([[ 101, 1188, 1110, 1126, 7758, 1859,  102]])\n",
-      "Tokenw   ise output: torch.Size([1, 7, 768]), Pooled output: torch.Size([1, 768])\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Tokens comes from a process that splits the input into sub-entities with interesting linguistic properties. \n",
-    "tokens = tokenizer.tokenize(\"This is an input example\")\n",
-    "print(\"Tokens: {}\".format(tokens))\n",
-    "\n",
-    "# This is not sufficient for the model, as it requires integers as input, \n",
-    "# not a problem, let's convert tokens to ids.\n",
-    "tokens_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
-    "print(\"Tokens id: {}\".format(tokens_ids))\n",
-    "\n",
-    "# Add the required special tokens\n",
-    "tokens_ids = tokenizer.build_inputs_with_special_tokens(tokens_ids)\n",
-    "\n",
-    "# We need to convert to a Deep Learning framework specific format, let's use PyTorch for now.\n",
-    "tokens_pt = torch.tensor([tokens_ids])\n",
-    "print(\"Tokens PyTorch: {}\".format(tokens_pt))\n",
-    "\n",
-    "# Now we're ready to go through BERT with out input\n",
-    "outputs, pooled = model(tokens_pt)\n",
-    "print(\"Tokenw   ise output: {}, Pooled output: {}\".format(outputs.shape, pooled.shape))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "As you can see, BERT outputs two tensors:\n",
-    " - One with the generated representation for every token in the input `(1, NB_TOKENS, REPRESENTATION_SIZE)`\n",
-    " - One with an aggregated representation for the whole input `(1, REPRESENTATION_SIZE)`\n",
-    " \n",
-    "The first, token-based, representation can be leveraged if your task requires to keep the sequence representation and you\n",
-    "want to operate at a token-level. This is particularly useful for Named Entity Recognition and Question-Answering.\n",
-    "\n",
-    "The second, aggregated, representation is especially useful if you need to extract the overall context of the sequence and don't\n",
-    "require a fine-grained token-leven. This is the case for Sentiment-Analysis of the sequence or Information Retrieval."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "The code you saw in the previous section introduced all the steps required to do simple model invocation.\n",
-    "For more day-to-day usage, transformers provides you higher-level methods which will makes your NLP journey easier\n",
-    "Let's improve our previous example"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "input_ids:\n",
-      "\ttensor([[ 101, 1188, 1110, 1126, 7758, 1859,  102]])\n",
-      "token_type_ids:\n",
-      "\ttensor([[0, 0, 0, 0, 0, 0, 0]])\n",
-      "attention_mask:\n",
-      "\ttensor([[1, 1, 1, 1, 1, 1, 1]])\n",
-      "Difference with previous code: (0.0, 0.0)\n"
-     ]
-    }
-   ],
-   "source": [
-    "# tokens = tokenizer.tokenize(\"This is an input example\")\n",
-    "# tokens_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
-    "# tokens_pt = torch.tensor([tokens_ids])\n",
-    "\n",
-    "# This code can be factored into one-line as follow\n",
-    "tokens_pt2 = tokenizer.encode_plus(\"This is an input example\", return_tensors=\"pt\")\n",
-    "\n",
-    "for key, value in tokens_pt2.items():\n",
-    "    print(\"{}:\\n\\t{}\".format(key, value))\n",
-    "\n",
-    "outputs2, pooled2 = model(**tokens_pt2)\n",
-    "print(\"Difference with previous code: ({}, {})\".format((outputs2 - outputs).sum(), (pooled2 - pooled).sum()))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "As you can see above, the methode `encode_plus` provides a convenient way to generate all the required parameters\n",
-    "that will go through the model. \n",
-    "\n",
-    "Moreover, you might have noticed it generated some additional tensors: \n",
-    "\n",
-    "- token_type_ids: This tensor will map every tokens to their corresponding segment (see below).\n",
-    "- attention_mask: This tensor is used to \"mask\" padded values in a batch of sequence with different lengths (see below)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Single segment token (str): ['[CLS]', 'This', 'is', 'a', 'sample', 'input', '[SEP]']\n",
-      "Single segment token (int): [101, 1188, 1110, 170, 6876, 7758, 102]\n",
-      "Single segment type       : [0, 0, 0, 0, 0, 0, 0]\n",
-      "\n",
-      "Multi segment token (str): ['[CLS]', 'This', 'is', 'segment', 'A', '[SEP]', 'This', 'is', 'segment', 'B', '[SEP]']\n",
-      "Multi segment token (int): [101, 1188, 1110, 6441, 138, 102, 1188, 1110, 6441, 139, 102]\n",
-      "Multi segment type       : [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1]\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Single segment input\n",
-    "single_seg_input = tokenizer.encode_plus(\"This is a sample input\")\n",
-    "\n",
-    "# Multiple segment input\n",
-    "multi_seg_input = tokenizer.encode_plus(\"This is segment A\", \"This is segment B\")\n",
-    "\n",
-    "print(\"Single segment token (str): {}\".format(tokenizer.convert_ids_to_tokens(single_seg_input['input_ids'])))\n",
-    "print(\"Single segment token (int): {}\".format(single_seg_input['input_ids']))\n",
-    "print(\"Single segment type       : {}\".format(single_seg_input['token_type_ids']))\n",
-    "\n",
-    "# Segments are concatened in the input to the model, with \n",
-    "print()\n",
-    "print(\"Multi segment token (str): {}\".format(tokenizer.convert_ids_to_tokens(multi_seg_input['input_ids'])))\n",
-    "print(\"Multi segment token (int): {}\".format(multi_seg_input['input_ids']))\n",
-    "print(\"Multi segment type       : {}\".format(multi_seg_input['token_type_ids']))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Tokens (int)      : [101, 1188, 1110, 170, 6876, 102, 0, 0]\n",
-      "Tokens (str)      : ['[CLS]', 'This', 'is', 'a', 'sample', '[SEP]', '[PAD]', '[PAD]']\n",
-      "Tokens (attn_mask): [1, 1, 1, 1, 1, 1, 0, 0]\n",
-      "\n",
-      "Tokens (int)      : [101, 1188, 1110, 1330, 2039, 6876, 3087, 102]\n",
-      "Tokens (str)      : ['[CLS]', 'This', 'is', 'another', 'longer', 'sample', 'text', '[SEP]']\n",
-      "Tokens (attn_mask): [1, 1, 1, 1, 1, 1, 1, 1]\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Padding highlight\n",
-    "tokens = tokenizer.batch_encode_plus(\n",
-    "    [\"This is a sample\", \"This is another longer sample text\"], \n",
-    "    pad_to_max_length=True  # First sentence will have some PADDED tokens to match second sequence length\n",
-    ")\n",
-    "\n",
-    "for i in range(2):\n",
-    "    print(\"Tokens (int)      : {}\".format(tokens['input_ids'][i]))\n",
-    "    print(\"Tokens (str)      : {}\".format([tokenizer.convert_ids_to_tokens(s) for s in tokens['input_ids'][i]]))\n",
-    "    print(\"Tokens (attn_mask): {}\".format(tokens['attention_mask'][i]))\n",
-    "    print()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Frameworks interoperability\n",
-    "\n",
-    "One of the most powerfull feature of transformers is its ability to seamlessly move from PyTorch to Tensorflow\n",
-    "without pain for the user.\n",
-    "\n",
-    "We provide some convenient methods to load TensorFlow pretrained weight insinde a PyTorch model and opposite."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "from transformers import TFBertModel, BertModel\n",
-    "\n",
-    "# Let's load a BERT model for TensorFlow and PyTorch\n",
-    "model_tf = TFBertModel.from_pretrained('bert-base-cased')\n",
-    "model_pt = BertModel.from_pretrained('bert-base-cased')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# transformers generates a ready to use dictionary with all the required parameters for the specific framework.\n",
-    "input_tf = tokenizer.encode_plus(\"This is a sample input\", return_tensors=\"tf\")\n",
-    "input_pt = tokenizer.encode_plus(\"This is a sample input\", return_tensors=\"pt\")\n",
-    "\n",
-    "# Let's compare the outputs\n",
-    "output_tf, output_pt = model_tf(input_tf), model_pt(**input_pt)\n",
-    "\n",
-    "# Models outputs 2 values (The value for each tokens, the pooled representation of the input sentence)\n",
-    "# Here we compare the output differences between PyTorch and TensorFlow.\n",
-    "for name, o_tf, o_pt in zip([\"output\", \"pooled\"], output_tf, output_pt):\n",
-    "    print(\"{} differences: {}\".format(name, (o_tf.numpy() - o_pt.numpy()).sum()))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "## Want it lighter? Faster? Let's talk distillation! \n",
-    "\n",
-    "One of the main concerns when using these Transformer based models is the computational power they require. All over this notebook we are using BERT model as it can be run on common machines but that's not the case for all of the models.\n",
-    "\n",
-    "For example, Google released a few months ago **T5** an Encoder/Decoder architecture based on Transformer and available in `transformers` with no more than 11 billions parameters. Microsoft also recently entered the game with **Turing-NLG** using 17 billions parameters. This kind of model requires tens of gigabytes to store the weights and a tremendous compute infrastructure to run such models which makes it impracticable for the common man !\n",
-    "\n",
-    "![transformers-parameters](https://lh5.googleusercontent.com/NRdXzEcgZV3ooykjIaTm9uvbr9QnSjDQHHAHb2kk_Lm9lIF0AhS-PJdXGzpcBDztax922XAp386hyNmWZYsZC1lUN2r4Ip5p9v-PHO19-jevRGg4iQFxgv5Olq4DWaqSA_8ptep7)\n",
-    "\n",
-    "With the goal of making Transformer-based NLP accessible to everyone we @huggingface developed models that take advantage of a training process called **Distillation** which allows us to drastically reduce the resources needed to run such models with almost zero drop in performances.\n",
-    "\n",
-    "Going over the whole Distillation process is out of the scope of this notebook, but if you want more information on the subject you may refer to [this Medium article written by my colleague Victor SANH, author of DistilBERT paper](https://medium.com/huggingface/distilbert-8cf3380435b5), you might also want to directly have a look at the paper [(Sanh & al., 2019)](https://arxiv.org/abs/1910.01108)\n",
-    "\n",
-    "Of course, in `transformers` we have distilled some models and made them available directly in the library ! "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "from transformers import DistilBertModel\n",
-    "\n",
-    "bert_distil = DistilBertModel.from_pretrained('distilbert-base-cased')\n",
-    "input_pt = tokenizer.encode_plus(\n",
-    "    'This is a sample input to demonstrate performance of distiled models especially inference time', \n",
-    "    return_tensors=\"pt\"\n",
-    ")\n",
-    "\n",
-    "\n",
-    "%time _ = bert_distil(input_pt['input_ids'])\n",
-    "%time _ = model_pt(input_pt['input_ids'])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Community provided models\n",
-    "\n",
-    "Last but not least, earlier in this notebook we introduced Hugging Face `transformers` as a repository for the NLP community to exchange pretrained models. We wanted to highlight this features and all the possibilities it offers for the end-user.\n",
-    "\n",
-    "To leverage community pretrained models, just provide the organisation name and name of the model to `from_pretrained` and it will do all the magic for you ! \n",
-    "\n",
-    "\n",
-    "We currently have more 50 models provided by the community and more are added every day, don't hesitate to give it a try !"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# Let's load German BERT from the Bavarian State Library\n",
-    "de_bert = BertModel.from_pretrained(\"dbmdz/bert-base-german-cased\")\n",
-    "de_tokenizer = BertTokenizer.from_pretrained(\"dbmdz/bert-base-german-cased\")\n",
-    "\n",
-    "de_input = de_tokenizer.encode_plus(\n",
-    "    \"Hugging Face ist einen französische Firma Mitarbeitern in New-York.\",\n",
-    "    return_tensors=\"pt\"\n",
-    ")\n",
-    "output_de, pooled_de = de_bert(**de_input)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.6"
-  },
-  "pycharm": {
-   "stem_cell": {
-    "cell_type": "raw",
-    "metadata": {
-     "collapsed": false
-    },
-    "source": []
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
@@ -1,511 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "## How can I leverage State-of-the-Art Natural Language Models with only one line of code ?"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "Newly introduced in transformers v2.3.0, **pipelines** provides a high-level, easy to use,\n",
-    "API for doing inference over a variety of downstream-tasks, including: \n",
-    "\n",
-    "- Sentence Classification (Sentiment Analysis): Indicate if the overall sentence is either positive or negative. _(Binary Classification task or Logitic Regression task)_\n",
-    "- Token Classification (Named Entity Recognition, Part-of-Speech tagging): For each sub-entities _(**tokens**)_ in the input, assign them a label _(Classification task)_.\n",
-    "- Question-Answering: Provided a tuple (question, context) the model should find the span of text in **content** answering the **question**.\n",
-    "- Mask-Filling: Suggests possible word(s) to fill the masked input with respect to the provided **context**.\n",
-    "- Feature Extraction: Maps the input to a higher, multi-dimensional space learned from the data.\n",
-    "\n",
-    "Pipelines encapsulate the overall process of every NLP process:\n",
-    " \n",
-    " 1. Tokenization: Split the initial input into multiple sub-entities with ... properties (i.e. tokens).\n",
-    " 2. Inference: Maps every tokens into a more meaningful representation. \n",
-    " 3. Decoding: Use the above representation to generate and/or extract the final output for the underlying task.\n",
-    "\n",
-    "The overall API is exposed to the end-user through the `pipeline()` method with the following \n",
-    "structure:\n",
-    "\n",
-    "```python\n",
-    "from transformers import pipeline\n",
-    "\n",
-    "# Using default model and tokenizer for the task\n",
-    "pipeline(\"<task-name>\")\n",
-    "\n",
-    "# Using a user-specified model\n",
-    "pipeline(\"<task-name>\", model=\"<model_name>\")\n",
-    "\n",
-    "# Using custom model/tokenizer as str\n",
-    "pipeline('<task-name>', model='<model name>', tokenizer='<tokenizer_name>')\n",
-    "```"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "!pip install transformers"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% code\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code \n"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "from __future__ import print_function\n",
-    "import ipywidgets as widgets\n",
-    "from transformers import pipeline"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "## 1. Sentence Classification - Sentiment Analysis"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "c9db53f30b9446c0af03268633a966c0"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "text": [
-      "\n"
-     ],
-     "output_type": "stream"
-    },
-    {
-     "data": {
-      "text/plain": "[{'label': 'POSITIVE', 'score': 0.9997656}]"
-     },
-     "metadata": {},
-     "output_type": "execute_result",
-     "execution_count": 8
-    }
-   ],
-   "source": [
-    "nlp_sentence_classif = pipeline('sentiment-analysis')\n",
-    "nlp_sentence_classif('Such a nice weather outside !')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "## 2. Token Classification - Named Entity Recognition"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "1e300789e22644f1aed66a5ed60e75c4"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "text": [
-      "\n"
-     ],
-     "output_type": "stream"
-    },
-    {
-     "data": {
-      "text/plain": "[{'word': 'Hu', 'score': 0.9970937967300415, 'entity': 'I-ORG'},\n {'word': '##gging', 'score': 0.9345750212669373, 'entity': 'I-ORG'},\n {'word': 'Face', 'score': 0.9787060022354126, 'entity': 'I-ORG'},\n {'word': 'French', 'score': 0.9981995820999146, 'entity': 'I-MISC'},\n {'word': 'New', 'score': 0.9983047246932983, 'entity': 'I-LOC'},\n {'word': '-', 'score': 0.8913455009460449, 'entity': 'I-LOC'},\n {'word': 'York', 'score': 0.9979523420333862, 'entity': 'I-LOC'}]"
-     },
-     "metadata": {},
-     "output_type": "execute_result",
-     "execution_count": 9
-    }
-   ],
-   "source": [
-    "nlp_token_class = pipeline('ner')\n",
-    "nlp_token_class('Hugging Face is a French company based in New-York.')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 3. Question Answering"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "82aca58f1ea24b4cb37f16402e8a5923"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "text": [
-      "\n"
-     ],
-     "output_type": "stream"
-    },
-    {
-     "name": "stderr",
-     "text": [
-      "convert squad examples to features: 100%|██████████| 1/1 [00:00<00:00, 225.51it/s]\n",
-      "add example index and unique id: 100%|██████████| 1/1 [00:00<00:00, 2158.67it/s]\n"
-     ],
-     "output_type": "stream"
-    },
-    {
-     "data": {
-      "text/plain": "{'score': 0.9632966867654424, 'start': 42, 'end': 50, 'answer': 'New-York.'}"
-     },
-     "metadata": {},
-     "output_type": "execute_result",
-     "execution_count": 10
-    }
-   ],
-   "source": [
-    "nlp_qa = pipeline('question-answering')\n",
-    "nlp_qa(context='Hugging Face is a French company based in New-York.', question='Where is based Hugging Face ?')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 4. Text Generation - Mask Filling"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "49df2227b4fa4eb28dcdcfc3d9261d0f"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "text": [
-      "\n"
-     ],
-     "output_type": "stream"
-    },
-    {
-     "data": {
-      "text/plain": "[{'sequence': '<s> Hugging Face is a French company based in Paris</s>',\n  'score': 0.23106691241264343,\n  'token': 2201},\n {'sequence': '<s> Hugging Face is a French company based in Lyon</s>',\n  'score': 0.0819825753569603,\n  'token': 12790},\n {'sequence': '<s> Hugging Face is a French company based in Geneva</s>',\n  'score': 0.04769463092088699,\n  'token': 11559},\n {'sequence': '<s> Hugging Face is a French company based in Brussels</s>',\n  'score': 0.047622501850128174,\n  'token': 6497},\n {'sequence': '<s> Hugging Face is a French company based in France</s>',\n  'score': 0.04130595177412033,\n  'token': 1470}]"
-     },
-     "metadata": {},
-     "output_type": "execute_result",
-     "execution_count": 11
-    }
-   ],
-   "source": [
-    "nlp_fill = pipeline('fill-mask')\n",
-    "nlp_fill('Hugging Face is a French company based in <mask>')"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## 5. Projection - Features Extraction "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "2af4cfb19e3243dda014d0f56b48f4b2"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "text": [
-      "\n"
-     ],
-     "output_type": "stream"
-    },
-    {
-     "data": {
-      "text/plain": "(1, 12, 768)"
-     },
-     "metadata": {},
-     "output_type": "execute_result",
-     "execution_count": 12
-    }
-   ],
-   "source": [
-    "import numpy as np\n",
-    "nlp_features = pipeline('feature-extraction')\n",
-    "output = nlp_features('Hugging Face is a French company based in Paris')\n",
-    "np.array(output).shape   # (Samples, Tokens, Vector Size)\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "Alright ! Now you have a nice picture of what is possible through transformers' pipelines, and there is more\n",
-    "to come in future releases. \n",
-    "\n",
-    "In the meantime, you can try the different pipelines with your own inputs"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% code\n"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "Dropdown(description='Task:', index=1, options=('sentiment-analysis', 'ner', 'fill_mask'), value='ner')",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "10bac065d46f4e4d9a8498dcc8104ecd"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": "Text(value='', description='Your input:', placeholder='Enter something')",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "2c5f1411f7a94714bc00f01b0e3b27b2"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "task = widgets.Dropdown(\n",
-    "    options=['sentiment-analysis', 'ner', 'fill_mask'],\n",
-    "    value='ner',\n",
-    "    description='Task:',\n",
-    "    disabled=False\n",
-    ")\n",
-    "\n",
-    "input = widgets.Text(\n",
-    "    value='',\n",
-    "    placeholder='Enter something',\n",
-    "    description='Your input:',\n",
-    "    disabled=False\n",
-    ")\n",
-    "\n",
-    "def forward(_):\n",
-    "    if len(input.value) > 0: \n",
-    "        if task.value == 'ner':\n",
-    "            output = nlp_token_class(input.value)\n",
-    "        elif task.value == 'sentiment-analysis':\n",
-    "            output = nlp_sentence_classif(input.value)\n",
-    "        else:\n",
-    "            if input.value.find('<mask>') == -1:\n",
-    "                output = nlp_fill(input.value + ' <mask>')\n",
-    "            else:\n",
-    "                output = nlp_fill(input.value)                \n",
-    "        print(output)\n",
-    "\n",
-    "input.on_submit(forward)\n",
-    "display(task, input)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {
-    "pycharm": {
-     "is_executing": false,
-     "name": "#%% Question Answering\n"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "Textarea(value='Einstein is famous for the general theory of relativity', description='Context:', placeholder=…",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "019fde2343634e94b6f32d04f6350ec1"
-      }
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "context = widgets.Textarea(\n",
-    "    value='Einstein is famous for the general theory of relativity',\n",
-    "    placeholder='Enter something',\n",
-    "    description='Context:',\n",
-    "    disabled=False\n",
-    ")\n",
-    "\n",
-    "query = widgets.Text(\n",
-    "    value='Why is Einstein famous for ?',\n",
-    "    placeholder='Enter something',\n",
-    "    description='Question:',\n",
-    "    disabled=False\n",
-    ")\n",
-    "\n",
-    "def forward(_):\n",
-    "    if len(context.value) > 0 and len(query.value) > 0: \n",
-    "        output = nlp_qa(question=query.value, context=context.value)            \n",
-    "        print(output)\n",
-    "\n",
-    "query.on_submit(forward)\n",
-    "display(context, query)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.6"
-  },
-  "pycharm": {
-   "stem_cell": {
-    "cell_type": "raw",
-    "source": [],
-    "metadata": {
-     "collapsed": false
-    }
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 1
-}
@@ -1,17 +0,0 @@
-# Transformers Notebooks
-
-You can find here a list of the official notebooks provided by Hugging Face.
-
-Also, we would like to list here interesting content created by the community. 
-If you wrote some notebook(s) leveraging transformers and would like be listed here, please open a 
-Pull Request and we'll review it so it can be included here. 
-
-
-## Hugging Face's notebooks :hugs:
-
-| Notebook     |      Description      |   |
-|:----------|:-------------:|------:|
-| [Getting Started Tokenizers](01-training-tokenizers.ipynb)  | How to train and use your very own tokenizer  |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/transformers/blob/master/notebooks/01-training-tokenizers.ipynb) |
-| [Getting Started Transformers](02-transformers.ipynb)   | How to easily start using transformers  | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/transformers/blob/master/notebooks/02-transformers.ipynb) |
-| [How to use Pipelines](03-pipelines.ipynb)  | Simple and efficient way to use State-of-the-Art models on downstream tasks through transformers | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/transformers/blob/master/notebooks/03-pipelines.ipynb) |
-| [How to train a language model](https://github.com/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb)| Highlight all the steps to effectively train Transformer model on custom data | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/01_how_to_train.ipynb)|
@@ -206,11 +206,7 @@ if is_torch_available():
        XLMForQuestionAnsweringSimple,
        XLM_PRETRAINED_MODEL_ARCHIVE_MAP,
    )
-    from .modeling_bart import (
-        BartForSequenceClassification,
-        BartModel,
-        BartForConditionalGeneration,
-    )
+    from .modeling_bart import BartForSequenceClassification, BartModel, BartForMaskedLM
    from .modeling_roberta import (
        RobertaForMaskedLM,
        RobertaModel,
@@ -135,8 +135,6 @@ class GPT2Config(PretrainedConfig):
        summary_activation=None,
        summary_proj_to_labels=True,
        summary_first_dropout=0.1,
-        bos_token_id=50256,
-        eos_token_id=50256,
        **kwargs
    ):
        super().__init__(**kwargs)
@@ -158,9 +156,6 @@ class GPT2Config(PretrainedConfig):
        self.summary_first_dropout = summary_first_dropout
        self.summary_proj_to_labels = summary_proj_to_labels

-        self.bos_token_id = bos_token_id
-        self.eos_token_ids = [eos_token_id]
-
    @property
    def max_position_embeddings(self):
        return self.n_positions
@@ -149,7 +149,6 @@ class TransfoXLConfig(PretrainedConfig):
        proj_init_std=0.01,
        init_std=0.02,
        layer_norm_epsilon=1e-5,
-        eos_token_id=0,
        **kwargs
    ):
        super().__init__(**kwargs)
@@ -187,8 +186,6 @@ class TransfoXLConfig(PretrainedConfig):
        self.init_std = init_std
        self.layer_norm_epsilon = layer_norm_epsilon

-        self.eos_token_ids = [eos_token_id]
-
    @property
    def max_position_embeddings(self):
        return self.tgt_len + self.ext_len + self.mem_len
@@ -98,18 +98,6 @@ class PretrainedConfig(object):
                logger.error("Can't set {} with value {} for {}".format(key, value, self))
                raise err

-    @property
-    def num_labels(self):
-        return self._num_labels
-
-    @num_labels.setter
-    def num_labels(self, num_labels):
-        self._num_labels = num_labels
-        self.id2label = {i: "LABEL_{}".format(i) for i in range(self.num_labels)}
-        self.id2label = dict((int(key), value) for key, value in self.id2label.items())
-        self.label2id = dict(zip(self.id2label.values(), self.id2label.keys()))
-        self.label2id = dict((key, int(value)) for key, value in self.label2id.items())
-
    def save_pretrained(self, save_directory):
        """
        Save a configuration object to the directory `save_directory`, so that it
@@ -194,8 +194,6 @@ class XLMConfig(PretrainedConfig):
        end_n_top=5,
        mask_token_id=0,
        lang_id=0,
-        bos_token_id=0,
-        pad_token_id=2,
        **kwargs
    ):
        """Constructs XLMConfig.
@@ -236,9 +234,6 @@ class XLMConfig(PretrainedConfig):
        if "n_words" in kwargs:
            self.n_words = kwargs["n_words"]

-        self.bos_token_id = bos_token_id
-        self.pad_token_id = pad_token_id
-
    @property
    def n_words(self):  # For backward compatibility
        return self.vocab_size
@@ -155,9 +155,6 @@ class XLNetConfig(PretrainedConfig):
        summary_last_dropout=0.1,
        start_n_top=5,
        end_n_top=5,
-        bos_token_id=1,
-        pad_token_id=5,
-        eos_token_id=2,
        **kwargs
    ):
        """Constructs XLNetConfig.
@@ -191,10 +188,6 @@ class XLNetConfig(PretrainedConfig):
        self.start_n_top = start_n_top
        self.end_n_top = end_n_top

-        self.bos_token_id = bos_token_id
-        self.pad_token_id = pad_token_id
-        self.eos_token_ids = [eos_token_id]
-
    @property
    def max_position_embeddings(self):
        return -1
@@ -23,13 +23,7 @@ import fairseq
 import torch
 from packaging import version

-from transformers import (
-    BartConfig,
-    BartForConditionalGeneration,
-    BartForSequenceClassification,
-    BartModel,
-    BartTokenizer,
-)
+from transformers import BartConfig, BartForMaskedLM, BartForSequenceClassification, BartModel, BartTokenizer


 FAIRSEQ_MODELS = ["bart.large", "bart.large.mnli", "bart.large.cnn"]
@@ -92,14 +86,14 @@ def convert_bart_checkpoint(checkpoint_path, pytorch_dump_folder_path):
    model.eval()
    # Check results

-    if checkpoint_path == "bart.large.cnn":
-        model = BartForConditionalGeneration(config, base_model=model)
+    if checkpoint_path == "bart.large.cnn":  # generate doesnt work yet
+        model = BartForMaskedLM(config, base_model=model)
        assert "lm_head.weight" in model.state_dict()
        assert model.lm_head.out_features == config.max_position_embeddings
        model.eval()
-        our_outputs = model.model(tokens)[0]
+        our_outputs = model.model.forward(tokens)[0]
    else:
-        our_outputs = model(tokens)[0]
+        our_outputs = model.forward(tokens)[0]
    assert their_output.shape == our_outputs.shape
    assert (their_output == our_outputs).all().item()
    Path(pytorch_dump_folder_path).mkdir(exist_ok=True)
@@ -45,12 +45,7 @@ from .modeling_albert import (
    AlbertForTokenClassification,
    AlbertModel,
 )
-from .modeling_bart import (
-    BART_PRETRAINED_MODEL_ARCHIVE_MAP,
-    BartForConditionalGeneration,
-    BartForSequenceClassification,
-    BartModel,
-)
+from .modeling_bart import BART_PRETRAINED_MODEL_ARCHIVE_MAP, BartForMaskedLM, BartForSequenceClassification, BartModel
 from .modeling_bert import (
    BERT_PRETRAINED_MODEL_ARCHIVE_MAP,
    BertForMaskedLM,
@@ -171,7 +166,7 @@ MODEL_FOR_PRETRAINING_MAPPING = OrderedDict(
        (AlbertConfig, AlbertForMaskedLM),
        (CamembertConfig, CamembertForMaskedLM),
        (XLMRobertaConfig, XLMRobertaForMaskedLM),
-        (BartConfig, BartForConditionalGeneration),
+        (BartConfig, BartForMaskedLM),
        (RobertaConfig, RobertaForMaskedLM),
        (BertConfig, BertForPreTraining),
        (OpenAIGPTConfig, OpenAIGPTLMHeadModel),
@@ -191,7 +186,7 @@ MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
        (AlbertConfig, AlbertForMaskedLM),
        (CamembertConfig, CamembertForMaskedLM),
        (XLMRobertaConfig, XLMRobertaForMaskedLM),
-        (BartConfig, BartForConditionalGeneration),
+        (BartConfig, BartForMaskedLM),
        (RobertaConfig, RobertaForMaskedLM),
        (BertConfig, BertForMaskedLM),
        (OpenAIGPTConfig, OpenAIGPTLMHeadModel),
@@ -640,9 +640,9 @@ class SelfAttention(nn.Module):
            reshaped = key_padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool)
            attn_weights = attn_weights.masked_fill(reshaped, float("-inf"))
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
-        attn_weights = F.softmax(attn_weights, dim=-1)
-        attn_probs = F.dropout(attn_weights, p=self.dropout, training=self.training,)
-
+        attn_weights_float = F.softmax(attn_weights, dim=-1, dtype=torch.float32)
+        attn_weights = attn_weights_float.type_as(attn_weights)
+        attn_probs = F.dropout(attn_weights_float, p=self.dropout, training=self.training,)
        assert v is not None
        attn_output = torch.bmm(attn_probs, v)
        assert attn_output.size() == (bsz * self.num_heads, tgt_len, self.head_dim)
@@ -696,7 +696,7 @@ class SelfAttention(nn.Module):
        elif prev_key_padding_mask is not None:
            filler = torch.zeros(batch_size, src_len - prev_key_padding_mask.size(1))
            if prev_key_padding_mask.is_cuda:
-                filler = filler.to(prev_key_padding_mask.device)
+                filler = filler.cuda()
            new_key_padding_mask = torch.cat([prev_key_padding_mask.float(), filler.float()], dim=1)
        elif key_padding_mask is not None:
            filler = torch.zeros(batch_size, src_len - key_padding_mask.size(1))
@@ -778,6 +778,21 @@ def _filter_out_falsey_values(tup) -> Tuple:
    return tuple(x for x in tup if isinstance(x, torch.Tensor) or x)


+RET_DOCSTRING = r"""
+    Return:
+        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
+        last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
+            of shape :obj:`(batch_size, sequence_length, hidden_size)`.
+            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
+        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
+            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
+            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+"""
 # Public API


@@ -848,9 +863,10 @@ class BartModel(PretrainedBartModel):


@add_start_docstrings(
-    "The BART Model with a language modeling head. Can be used for summarization.", BART_START_DOCSTRING,
+    "The bare BART Model with a language modeling head. This is the model used for summarization.",
+    BART_START_DOCSTRING,
 )
-class BartForConditionalGeneration(PretrainedBartModel):
+class BartForMaskedLM(PretrainedBartModel):
    base_model_prefix = "model"

    def __init__(self, config: BartConfig):
@@ -903,18 +919,11 @@ class BartForConditionalGeneration(PretrainedBartModel):

    Examples::

-            # Mask filling only works for bart-large
-            from transformers import BartTokenizer, BartForConditionalGeneration
-            tokenizer = AutoTokenizer.from_pretrained('bart-large')
-            TXT = "My friends are <mask> but they eat too many carbs."
-            model = BartForConditionalGeneration.from_pretrained('bart-large')
-            input_ids = tokenizer.batch_encode_plus([TXT], return_tensors='pt')['input_ids']
-            logits = model(input_ids)[0]
-            masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
-            probs = logits[0, masked_index].softmax(dim=0)
-            values, predictions = probs.topk(5)
-            tokenizer.decode(predictions).split()
-            # ['good', 'great', 'all', 'really', 'very']
+            tokenizer = BartTokenizer.from_pretrained('bart-large')
+            model = BartForMaskedLM.from_pretrained('bart-large')
+            input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0)  # Batch size 1
+            outputs = model(input_ids=input_ids, lm_labels=input_ids)
+            loss, prediction_scores = outputs[:2]
        """
        outputs = self.model(
            input_ids,
@@ -983,7 +992,8 @@ class BartForConditionalGeneration(PretrainedBartModel):
        min_len=0,
        no_repeat_ngram_size=0,
    ):
-        r""" Generates summaries using the lm-head and greedy beam search
+        r""" Generates sequences for models with a LM head. The method currently supports greedy or penalized greedy decoding, sampling with top-k or nucleus sampling
+        and beam-search.

        Adapted in part from Facebook's `XLM beam search code`_ and `Fairseq beam search code`_.

@@ -1021,16 +1031,16 @@ class BartForConditionalGeneration(PretrainedBartModel):
                sequence_length is <= max_length (examples can finish early)

        Examples::
-            from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig
-            # see ``examples/summarization/bart/evaluate_cnn.py`` for a longer example
-            config = BartConfig(vocab_size=50264, output_past=True) # no mask_token_id
-            model = BartForConditionalGeneration.from_pretrained('bart-large-cnn', config=config)
-            tokenizer = BartTokenizer.from_pretrained('bart-large-cnn')
+
+            config = BartConfig(vocab_size=50264, output_past=True)
+            model = AutoModelWithLMHead.from_pretrained('bart-large-cnn', config=config)
+            tokenizer = AutoTokenizer.from_pretrained('bart-large-cnn')
            ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
            inputs = tokenizer.batch_encode_plus([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
            # Generate Summary
-            summary_ids = model.generate(inputs['input_ids'], attention_mask=inputs['attention_mask'], num_beams=4, max_length=5)
-            print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
+            generated_ids = model.generate(inputs['input_ids'], attention_mask=inputs['attention_mask'], num_beams=4, max_length=5)
+            print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in generated_ids])
+
        """
        bos_token_id = self.config.bos_token_id
        pad_token_id = self.config.pad_token_id
@@ -148,12 +148,9 @@ class FlaubertModel(XLMModel):

    Examples::

-        from transformers import FlaubertTokenizer, FlaubertModel
-        import torch
-
        tokenizer = FlaubertTokenizer.from_pretrained('flaubert-base-cased')
        model = FlaubertModel.from_pretrained('flaubert-base-cased')
-        input_ids = torch.tensor(tokenizer.encode("Le chat mange une pomme.", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        input_ids = torch.tensor(tokenizer.encode("Le chat manges une pomme.", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
        outputs = model(input_ids)
        last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple

@@ -480,7 +480,7 @@ class TFAlbertMLMHead(tf.keras.layers.Layer):

 class TFAlbertMainLayer(tf.keras.layers.Layer):
    def __init__(self, config, **kwargs):
-        super().__init__(**kwargs)
+        super().__init__(config, **kwargs)
        self.num_hidden_layers = config.num_hidden_layers

        self.embeddings = TFAlbertEmbeddings(config, name="embeddings")
@@ -104,10 +104,10 @@ class TFMultiHeadAttention(tf.keras.layers.Layer):
        k = self.split_into_heads(k, batch_size)
        v = self.split_into_heads(v, batch_size)
        if layer_past is not None:
-            past_key, past_value = tf.unstack(layer_past, axis=0)
+            past_key, past_value = tf.unstack(layer_past, axis=1)
            k = tf.concat((past_key, k), axis=-2)
            v = tf.concat((past_value, v), axis=-2)
-        present = tf.stack((k, v), axis=0)
+        present = tf.stack((k, v), axis=1)

        output = scaled_dot_product_attention(q, k, v, mask, attention_mask, head_mask)
        scaled_attention = tf.transpose(output[0], perm=[0, 2, 1, 3])
@@ -139,10 +139,10 @@ class TFAttention(tf.keras.layers.Layer):
        key = self.split_heads(key)
        value = self.split_heads(value)
        if layer_past is not None:
-            past_key, past_value = tf.unstack(layer_past, axis=0)
+            past_key, past_value = tf.unstack(layer_past, axis=1)
            key = tf.concat([past_key, key], axis=-2)
            value = tf.concat([past_value, value], axis=-2)
-        present = tf.stack([key, value], axis=0)
+        present = tf.stack([key, value], axis=1)

        attn_outputs = self._attn([query, key, value, attention_mask, head_mask], training=training)
        a = attn_outputs[0]
@@ -199,7 +199,7 @@ class TFBlock(tf.keras.layers.Layer):

 class TFOpenAIGPTMainLayer(tf.keras.layers.Layer):
    def __init__(self, config, *inputs, **kwargs):
-        super().__init__(*inputs, **kwargs)
+        super().__init__(config, *inputs, **kwargs)
        self.output_hidden_states = config.output_hidden_states
        self.output_attentions = config.output_attentions
        self.num_hidden_layers = config.n_layer
@@ -142,7 +142,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
        # # initialize all new embeddings (in particular added tokens)
        # self._init_weights(new_embeddings)

-        # # Copy token embeddings from the previous weights
+        # # Copy word embeddings from the previous weights
        # num_tokens_to_copy = min(old_num_tokens, new_num_tokens)
        # new_embeddings.weight.data[:num_tokens_to_copy, :] = old_embeddings.weight.data[:num_tokens_to_copy, :]

@@ -557,19 +557,6 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
        else:
            assert len(shape_list(input_ids)) == 2, "Input prompt should be of shape (batch_size, sequence length)."

-        if do_sample is False:
-            if num_beams == 1:
-                # no_beam_search greedy generation conditions
-                assert (
-                    num_return_sequences == 1
-                ), "Greedy decoding will always produce the same output for num_beams == 1 and num_return_sequences > 1. Please set num_return_sequences = 1"
-
-            else:
-                # beam_search greedy generation conditions
-                assert (
-                    num_beams >= num_return_sequences
-                ), "Greedy beam search decoding cannot return more sequences than it has beams. Please set num_beams >= num_return_sequences"
-
        if pad_token_id is None and eos_token_ids is not None:
            logger.warning(
                "Setting `pad_token_id` to {} (first `eos_token_id`) to generate sequence".format(eos_token_ids[0])
@@ -580,7 +567,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
        cur_len = shape_list(input_ids)[1]
        vocab_size = self.config.vocab_size

-        if num_return_sequences != 1 and do_sample:
+        if num_return_sequences != 1:
            # Expand input to num return sequences
            input_ids = tf.broadcast_to(tf.expand_dims(input_ids, 1), (batch_size, num_return_sequences, cur_len))
            effective_batch_size = batch_size * num_return_sequences
@@ -601,7 +588,6 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
                pad_token_id,
                eos_token_ids,
                effective_batch_size,
-                num_return_sequences,
                length_penalty,
                num_beams,
                vocab_size,
@@ -641,7 +627,19 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
            All returned sequence are generated independantly.
        """

-        # length of generated sentences / unfinished sentences
+        def _create_next_token_logits_penalties(input_ids, logits):
+            # create logit penalties for already seen input_ids
+            token_penalties = np.ones(shape_list(logits))
+            prev_input_ids = [np.unique(input_id) for input_id in input_ids.numpy()]
+            for i, prev_input_id in enumerate(prev_input_ids):
+                logit_penalized = logits[i].numpy()[prev_input_id]
+                # if previous logit score is < 0 then multiply repetition penalty else divide
+                logit_penalized[logit_penalized < 0] = repetition_penalty
+                logit_penalized[logit_penalized > 0] = 1 / repetition_penalty
+                np.put(token_penalties[i], prev_input_id, logit_penalized)
+            return tf.convert_to_tensor(token_penalties, dtype=tf.float32)
+
+        # current position / max lengths / length of generated sentences / unfinished sentences
        unfinished_sents = tf.ones_like(input_ids[:, 0])
        sent_lengths = tf.ones_like(input_ids[:, 0]) * max_length

@@ -658,9 +656,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):

            # repetition penalty from CTRL paper (https://arxiv.org/abs/1909.05858)
            if repetition_penalty != 1.0:
-                next_token_logits_penalties = _create_next_token_logits_penalties(
-                    input_ids, next_token_logits, repetition_penalty
-                )
+                next_token_logits_penalties = _create_next_token_logits_penalties(input_ids, next_token_logits)
                next_token_logits = tf.math.multiply(next_token_logits, next_token_logits_penalties)

            if do_sample:
@@ -742,249 +738,11 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin):
        pad_token_id,
        eos_token_ids,
        batch_size,
-        num_return_sequences,
        length_penalty,
        num_beams,
        vocab_size,
    ):
-        """ Generate sequences for each example with beam search.
-        """
-
-        # Expand input to num beams
-        input_ids = tf.broadcast_to(tf.expand_dims(input_ids, 1), (batch_size, num_beams, cur_len))
-        input_ids = tf.reshape(input_ids, (batch_size * num_beams, cur_len))  # (batch_size * num_beams, cur_len)
-
-        # generated hypotheses
-        generated_hyps = [
-            BeamHypotheses(num_beams, max_length, length_penalty, early_stopping=False) for _ in range(batch_size)
-        ]
-
-        # scores for each sentence in the beam
-        if do_sample is False:
-            beam_scores_begin = tf.zeros((batch_size, 1), dtype=tf.float32)
-            beam_scores_end = tf.zeros((batch_size, num_beams - 1), dtype=tf.float32) * 1e-9
-            beam_scores = tf.concat([beam_scores_begin, beam_scores_end], -1)
-        else:
-            beam_scores = tf.zeros((batch_size, num_beams), dtype=tf.float32)
-
-        beam_scores = tf.reshape(beam_scores, (batch_size * num_beams,))
-
-        # cache compute states
-        past = None
-
-        # done sentences
-        done = [False for _ in range(batch_size)]
-
-        while cur_len < max_length:
-            model_inputs = self.prepare_inputs_for_generation(input_ids, past=past)
-            outputs = self(**model_inputs)  # (batch_size * num_beams, cur_len, vocab_size)
-            next_token_logits = outputs[0][:, -1, :]  # (batch_size * num_beams, vocab_size)
-
-            # if model has past, then set the past variable to speed up decoding
-            if self._do_output_past(outputs):
-                past = outputs[1]
-
-            # repetition penalty (from CTRL paper https://arxiv.org/abs/1909.05858)
-            if repetition_penalty != 1.0:
-                next_token_logits_penalties = _create_next_token_logits_penalties(
-                    input_ids, next_token_logits, repetition_penalty
-                )
-                next_token_logits = tf.math.multiply(next_token_logits, next_token_logits_penalties)
-
-            if do_sample:
-                # Temperature (higher temperature => more likely to sample low probability tokens)
-                if temperature != 1.0:
-                    next_token_logits = next_token_logits / temperature
-
-                scores = tf.nn.log_softmax(next_token_logits, axis=-1)  # (batch_size * num_beams, vocab_size)
-                _scores = scores + tf.broadcast_to(
-                    beam_scores[:, None], (batch_size * num_beams, vocab_size)
-                )  # (batch_size * num_beams, vocab_size)
-
-                # Top-p/top-k filtering
-                _scores = tf_top_k_top_p_filtering(
-                    _scores, top_k=top_k, top_p=top_p, min_tokens_to_keep=2
-                )  # (batch_size * num_beams, vocab_size)
-                # Sample 2 next tokens for each beam (so we have some spare tokens and match output of greedy beam search)
-                _scores = tf.reshape(_scores, (batch_size, num_beams * vocab_size))
-
-                next_tokens = tf.random.categorical(
-                    _scores, dtype=tf.int32, num_samples=2 * num_beams
-                )  # (batch_size, 2 * num_beams)
-                # Compute next scores
-                next_scores = tf.gather(_scores, next_tokens, batch_dims=1)  # (batch_size, 2 * num_beams)
-            else:
-                # do greedy beam search
-                scores = tf.nn.log_softmax(next_token_logits, axis=-1)  # (batch_size * num_beams, vocab_size)
-                assert shape_list(scores) == [batch_size * num_beams, vocab_size]
-                # Add the log prob of the new beams to the log prob of the beginning of the sequence (sum of logs == log of the product)
-                next_scores = scores + tf.broadcast_to(
-                    beam_scores[:, None], (batch_size * num_beams, vocab_size)
-                )  # (batch_size * num_beams, vocab_size)
-
-                # re-organize to group the beam together (we are keeping top hypothesis accross beams)
-                next_scores = tf.reshape(
-                    next_scores, (batch_size, num_beams * vocab_size)
-                )  # (batch_size, num_beams * vocab_size)
-                next_scores, next_tokens = tf.math.top_k(next_scores, 2 * num_beams, sorted=True)
-
-            assert shape_list(next_scores) == shape_list(next_tokens) == [batch_size, 2 * num_beams]
-
-            # next batch beam content
-            # list of (batch_size * num_beams) tuple(next hypothesis score, next token, current position in the batch)
-            next_batch_beam = []
-
-            # for each sentence
-            for batch_idx in range(batch_size):
-
-                # if we are done with this sentence
-                done[batch_idx] = done[batch_idx] or generated_hyps[batch_idx].is_done(
-                    tf.reduce_max(next_scores[batch_idx]).numpy()
-                )
-                if done[batch_idx]:
-                    assert (
-                        len(generated_hyps[batch_idx]) >= num_beams
-                    ), "Batch can only be done if at least {} beams have been generated".format(num_beams)
-                    assert (
-                        eos_token_ids is not None and pad_token_id is not None
-                    ), "generated beams >= num_beams -> eos_token_id and pad_token have to be defined"
-                    next_batch_beam.extend([(0, pad_token_id, 0)] * num_beams)  # pad the batch
-                    continue
-
-                # next sentence beam content
-                next_sent_beam = []
-
-                # next tokens for this sentence
-                for idx, score in zip(next_tokens[batch_idx], next_scores[batch_idx]):
-
-                    # get beam and token IDs
-                    beam_id = idx // vocab_size
-                    token_id = idx % vocab_size
-
-                    # add to generated hypotheses if end of sentence or last iteration
-                    if eos_token_ids is not None and token_id.numpy() in eos_token_ids:
-                        generated_hyps[batch_idx].add(
-                            tf.identity(input_ids[batch_idx * num_beams + beam_id, :cur_len]), score.numpy()
-                        )
-                    else:
-                        # add next predicted token if it is not eos_token
-                        next_sent_beam.append((score, token_id, batch_idx * num_beams + beam_id))
-
-                    # the beam for next step is full
-                    if len(next_sent_beam) == num_beams:
-                        break
-
-                # update next beam content
-                assert len(next_sent_beam) == num_beams, "Beam should always be full"
-                next_batch_beam.extend(next_sent_beam)
-                assert len(next_batch_beam) == num_beams * (batch_idx + 1)
-
-            # sanity check / prepare next batch
-            assert len(next_batch_beam) == batch_size * num_beams
-            beam_scores = tf.convert_to_tensor([x[0] for x in next_batch_beam], dtype=tf.float32)
-            beam_tokens = tf.convert_to_tensor([x[1] for x in next_batch_beam], dtype=tf.int32)
-            beam_idx = tf.convert_to_tensor([x[2] for x in next_batch_beam], dtype=tf.int32)
-
-            # re-order batch
-            input_ids = tf.stack([tf.identity(input_ids[x, :]) for x in beam_idx])
-            input_ids = tf.concat([input_ids, tf.expand_dims(beam_tokens, 1)], axis=-1)
-
-            # re-order internal states
-            if past:
-                past = self._reorder_cache(past, beam_idx)
-
-            # update current length
-            cur_len = cur_len + 1
-
-            # stop when we are done with each sentence
-            if all(done):
-                break
-
-        for batch_idx in range(batch_size):
-            # Add all open beam hypothesis to generated_hyps
-            if not done[batch_idx]:
-                for idx, score in zip(next_tokens[batch_idx], next_scores[batch_idx]):
-
-                    # get beam and token IDs
-                    beam_id = idx // vocab_size
-                    token_id = idx % vocab_size
-                    generated_hyps[batch_idx].add(
-                        tf.identity(input_ids[batch_idx * num_beams + beam_id, :cur_len]), score.numpy()
-                    )
-
-        # depending on whether greedy generation is wanted or not define different output_batch_size and output_num_return_sequences_per_batch
-        output_batch_size = batch_size if do_sample else batch_size * num_return_sequences
-        output_num_return_sequences_per_batch = 1 if do_sample else num_return_sequences
-
-        # select the best hypotheses
-        sent_lengths_list = []
-        best = []
-
-        # retrieve best hypotheses
-        for i, hypotheses in enumerate(generated_hyps):
-            sorted_hyps = sorted(hypotheses.beams, key=lambda x: x[0])
-            for j in range(output_num_return_sequences_per_batch):
-                best_hyp = sorted_hyps.pop()[1]
-                sent_lengths_list.append(len(best_hyp))
-                best.append(best_hyp)
-        assert output_batch_size == len(best), "Output batch size {} must match output beam hypotheses {}".format(
-            output_batch_size, len(best)
-        )
-
-        sent_lengths = tf.convert_to_tensor(sent_lengths_list, dtype=tf.int32)
-
-        # shorter batches are filled with pad_token
-        if tf.reduce_min(sent_lengths).numpy() != tf.reduce_max(sent_lengths).numpy():
-            assert pad_token_id is not None, "`Pad_token_id` has to be defined"
-            sent_max_len = min(tf.reduce_max(sent_lengths).numpy() + 1, max_length)
-            decoded_list = []
-
-            # fill with hypothesis and eos_token_id if necessary
-            for i, hypo in enumerate(best):
-                padding = tf.ones((sent_max_len - shape_list(hypo)[0],), dtype=tf.int32) * pad_token_id
-                decoded_hypo = tf.concat([hypo, padding], axis=0)
-
-                if sent_lengths[i] < max_length:
-                    decoded_hypo = tf.where(
-                        tf.range(max_length) == sent_lengths[i],
-                        eos_token_ids[0] * tf.ones((sent_max_len,), dtype=tf.int32),
-                        decoded_hypo,
-                    )
-                decoded_list.append(decoded_hypo)
-            decoded = tf.stack(decoded_list)
-        else:
-            # none of the hypotheses have an eos_token
-            assert (len(hypo) == max_length for hypo in best)
-            decoded = tf.stack(best)
-
-        return decoded
-
-    @staticmethod
-    def _reorder_cache(past, beam_idx):
-        reordered_past = []
-        for layer_past in past:
-            # get the correct batch idx from layer past batch dim
-            # batch dim of `past` and `mems` is at 2nd position
-            reordered_layer_past = [tf.identity(tf.expand_dims(layer_past[:, i], 1)) for i in beam_idx]
-            reordered_layer_past = tf.concat(reordered_layer_past, axis=1)
-            # check that shape matches
-            assert shape_list(reordered_layer_past) == shape_list(layer_past)
-            reordered_past.append(reordered_layer_past)
-        past = tuple(reordered_past)
-        return past
-
-
-def _create_next_token_logits_penalties(input_ids, logits, repetition_penalty):
-    # create logit penalties for already seen input_ids
-    token_penalties = np.ones(shape_list(logits))
-    prev_input_ids = [np.unique(input_id) for input_id in input_ids.numpy()]
-    for i, prev_input_id in enumerate(prev_input_ids):
-        logit_penalized = logits[i].numpy()[prev_input_id]
-        # if previous logit score is < 0 then multiply repetition penalty else divide
-        logit_penalized[logit_penalized < 0] = repetition_penalty
-        logit_penalized[logit_penalized > 0] = 1 / repetition_penalty
-        np.put(token_penalties[i], prev_input_id, logit_penalized)
-    return tf.convert_to_tensor(token_penalties, dtype=tf.float32)
+        pass


 def tf_top_k_top_p_filtering(logits, top_k=0, top_p=1.0, filter_value=-float("Inf"), min_tokens_to_keep=1):
@@ -1053,56 +811,6 @@ def set_tensor_by_indices_to_value(tensor, indices, value):
    return tf.where(indices, value_tensor, tensor)


-class BeamHypotheses(object):
-    def __init__(self, num_beams, max_length, length_penalty, early_stopping):
-        """
-        Initialize n-best list of hypotheses.
-        """
-        self.max_length = max_length - 1  # ignoring bos_token
-        self.length_penalty = length_penalty
-        self.early_stopping = early_stopping
-        self.num_beams = num_beams
-        self.beams = []
-        self.worst_score = 1e9
-
-    def __len__(self):
-        """
-        Number of hypotheses in the list.
-        """
-        return len(self.beams)
-
-    def add(self, hyp, sum_logprobs):
-        """
-        Add a new hypothesis to the list.
-        """
-        score = sum_logprobs / len(hyp) ** self.length_penalty
-        if len(self) < self.num_beams or score > self.worst_score:
-            self.beams.append((score, hyp))
-            if len(self) > self.num_beams:
-                sorted_scores = sorted([(s, idx) for idx, (s, _) in enumerate(self.beams)])
-                del self.beams[sorted_scores[0][1]]
-                self.worst_score = sorted_scores[1][0]
-            else:
-                self.worst_score = min(score, self.worst_score)
-
-    def is_done(self, best_sum_logprobs, cur_len=None):
-        """
-        If there are enough hypotheses and that none of the hypotheses being generated
-        can become better than the worst one in the heap, then we are done with this sentence.
-        """
-
-        if len(self) < self.num_beams:
-            return False
-        elif self.early_stopping:
-            return True
-        else:
-            if cur_len is None:
-                cur_len = self.max_length
-            cur_score = best_sum_logprobs / cur_len ** self.length_penalty
-            ret = self.worst_score >= cur_score
-            return ret
-
-
 class TFConv1D(tf.keras.layers.Layer):
    def __init__(self, nf, nx, initializer_range=0.02, **kwargs):
        """ TFConv1D layer as defined by Radford et al. for OpenAI GPT (and also used in GPT-2)
@@ -1141,7 +849,7 @@ class TFSharedEmbeddings(tf.keras.layers.Layer):
        self.initializer_range = hidden_size ** -0.5 if initializer_range is None else initializer_range

    def build(self, input_shape):
-        """Build shared token embedding layer
+        """Build shared word embedding layer
        Shared weights logic adapted from
            https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
        """
@@ -15,6 +15,7 @@
 # limitations under the License.
 """PyTorch BERT model."""

+
 import logging
 import os
 import typing
@@ -241,7 +242,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
        # initialize all new embeddings (in particular added tokens)
        self._init_weights(new_embeddings)

-        # Copy token embeddings from the previous weights
+        # Copy word embeddings from the previous weights
        num_tokens_to_copy = min(old_num_tokens, new_num_tokens)
        new_embeddings.weight.data[:num_tokens_to_copy, :] = old_embeddings.weight.data[:num_tokens_to_copy, :]

@@ -539,15 +540,6 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
                model_to_load = getattr(model, cls.base_model_prefix)

            load(model_to_load, prefix=start_prefix)
-
-            if model.__class__.__name__ != model_to_load.__class__.__name__:
-                base_model_state_dict = model_to_load.state_dict().keys()
-                head_model_state_dict_without_base_prefix = [
-                    key.split(cls.base_model_prefix + ".")[-1] for key in model.state_dict().keys()
-                ]
-
-                missing_keys.extend(head_model_state_dict_without_base_prefix - base_model_state_dict)
-
            if len(missing_keys) > 0:
                logger.info(
                    "Weights of {} not initialized from pretrained model: {}".format(
@@ -566,7 +558,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
                        model.__class__.__name__, "\n\t".join(error_msgs)
                    )
                )
-        model.tie_weights()  # make sure token embedding weights are still tied if needed
+        model.tie_weights()  # make sure word embedding weights are still tied if needed

        # Set model in evaluation mode to desactivate DropOut modules by default
        model.eval()
@@ -677,7 +669,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):

            tokenizer = AutoTokenizer.from_pretrained('distilgpt2')   # Initialize tokenizer
            model = AutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
-            outputs = model.generate(max_length=40, do_sample=False)  # do greedy decoding
+            outputs = model.generate(max_length=40, bos_token_id=tokenizer.bos_token_id, eos_token_ids=tokenizer.eos_token_id, do_sample=False)  # do greedy decoding
            print('Generated: {}'.format(tokenizer.decode(outputs[0], skip_special_tokens=True)))

            tokenizer = AutoTokenizer.from_pretrained('openai-gpt')   # Initialize tokenizer
@@ -692,7 +684,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
            model = AutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
            input_context = 'The dog'
            input_ids = torch.tensor(tokenizer.encode(input_context)).unsqueeze(0)  # encode input context
-            outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3)  # 3 generate sequences using by sampling
+            outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, bos_token_id=tokenizer.bos_token_id, pad_token_id=tokenizer.pad_token_id, eos_token_ids=tokenizer.eos_token_id, num_return_sequences=3)  # 3 generate sequences using by sampling
            for i in range(3): #  3 output sequences were generated
                print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True)))

@@ -766,7 +758,6 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
        else:
            assert input_ids.dim() == 2, "Input prompt should be of shape (batch_size, sequence length)."

-        # not allow to duplicate outputs when greedy decoding
        if do_sample is False:
            if num_beams == 1:
                # no_beam_search greedy generation conditions
@@ -790,21 +781,15 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
        cur_len = input_ids.shape[1]
        vocab_size = self.config.vocab_size

-        # set effective batch size and effective batch multiplier according to do_sample
-        if do_sample:
+        if num_return_sequences != 1 and do_sample:
+            # Expand input to num return sequences
+            input_ids = input_ids.unsqueeze(1).expand(batch_size, num_return_sequences, cur_len)
+            input_ids = input_ids.contiguous().view(
+                batch_size * num_return_sequences, cur_len
+            )  # shape: (batch_size * num_return_sequences, cur_len)
            effective_batch_size = batch_size * num_return_sequences
-            effective_batch_mult = num_return_sequences
        else:
            effective_batch_size = batch_size
-            effective_batch_mult = 1
-
-        # Expand input ids if num_beams > 1 or num_return_sequences > 1
-        if num_return_sequences > 1 or num_beams > 1:
-            input_ids_len = input_ids.shape[-1]
-            input_ids = input_ids.unsqueeze(1).expand(batch_size, effective_batch_mult * num_beams, input_ids_len)
-            input_ids = input_ids.contiguous().view(
-                effective_batch_size * num_beams, input_ids_len
-            )  # shape: (batch_size * num_return_sequences * num_beams, cur_len)

        if num_beams > 1:
            output = self._generate_beam_search(
@@ -858,7 +843,8 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
        """ Generate sequences for each example without beam search (num_beams == 1).
            All returned sequence are generated independantly.
        """
-        # length of generated sentences / unfinished sentences
+        # current position / max lengths / length of generated sentences / unfinished sentences
+
        unfinished_sents = input_ids.new(batch_size).fill_(1)
        sent_lengths = input_ids.new(batch_size).fill_(max_length)

@@ -907,12 +893,12 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
                    # unfinished_sents is set to zero if eos in sentence
                    unfinished_sents.mul_((~eos_in_sents).long())

+            cur_len = cur_len + 1
+
            # stop when there is a </s> in each sentence, or if we exceed the maximul length
            if unfinished_sents.max() == 0:
                break

-            cur_len = cur_len + 1
-
        # if there are different sentences lengths in the batch, some batches have to be padded
        if sent_lengths.min().item() != sent_lengths.max().item():
            assert pad_token_id is not None, "`Pad_token_id` has to be defined if batches have different lengths"
@@ -947,6 +933,11 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
        """ Generate sequences for each example with beam search.
        """

+        # Expand input to num beams
+        # assert input_ids.shape == (batch_size * num_beams, cur_len)
+        input_ids = input_ids.unsqueeze(1).expand(batch_size, num_beams, cur_len)
+        input_ids = input_ids.contiguous().view(batch_size * num_beams, cur_len)  # (batch_size * num_beams, cur_len)
+
        # generated hypotheses
        generated_hyps = [
            BeamHypotheses(num_beams, max_length, length_penalty, early_stopping=False) for _ in range(batch_size)
@@ -954,7 +945,8 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):

        # scores for each sentence in the beam
        beam_scores = torch.zeros((batch_size, num_beams), dtype=torch.float, device=input_ids.device)
-        # Greedy decoding it is made sure that only tokens of the first beam are considered to avoid sampling the exact same tokens three times
+
+        # Greedy decoding it is made sure that only words of the first beam are considered to avoid sampling the exact same words three times
        if do_sample is False:
            beam_scores[:, 1:] = -1e9
        beam_scores = beam_scores.view(-1)  # shape (batch_size * num_beams,)
@@ -968,7 +960,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
        while cur_len < max_length:
            model_inputs = self.prepare_inputs_for_generation(input_ids, past=past)
            outputs = self(**model_inputs)  # (batch_size * num_beams, cur_len, vocab_size)
-            next_token_logits = outputs[0][:, -1, :]  # (batch_size * num_beams, vocab_size)
+            scores = outputs[0][:, -1, :]  # (batch_size * num_beams, vocab_size)

            # if model has past, then set the past variable to speed up decoding
            if self._do_output_past(outputs):
@@ -976,16 +968,14 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):

            # repetition penalty (from CTRL paper https://arxiv.org/abs/1909.05858)
            if repetition_penalty != 1.0:
-                self.enforce_repetition_penalty_(
-                    next_token_logits, batch_size, num_beams, input_ids, repetition_penalty
-                )
+                self.enforce_repetition_penalty_(scores, batch_size, num_beams, input_ids, repetition_penalty)

            if do_sample:
                # Temperature (higher temperature => more likely to sample low probability tokens)
                if temperature != 1.0:
-                    next_token_logits = next_token_logits / temperature
+                    scores = scores / temperature

-                scores = F.log_softmax(next_token_logits, dim=-1)  # (batch_size * num_beams, vocab_size)
+                scores = F.log_softmax(scores, dim=-1)  # (batch_size * num_beams, vocab_size)
                _scores = scores + beam_scores[:, None].expand_as(scores)  # (batch_size * num_beams, vocab_size)

                # Top-p/top-k filtering
@@ -998,31 +988,25 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
                    batch_size, num_beams * vocab_size
                )  # (batch_size, num_beams * vocab_size)

-                # Sample 2 next tokens for each beam (so we have some spare tokens and match output of greedy beam search)
-                next_tokens = torch.multinomial(
+                # Sample 2 next words for each beam (so we have some spare tokens and match output of greedy beam search)
+                next_words = torch.multinomial(
                    F.softmax(_scores, dim=-1), num_samples=2 * num_beams
                )  # (batch_size, num_beams * 2)

                # Compute next scores
-                next_scores = torch.gather(_scores, -1, next_tokens)  # (batch_size, num_beams * 2)
+                next_scores = torch.gather(_scores, -1, next_words)  # (batch_size, num_beams * 2)

-                # sort the sampled vector to make sure that the first num_beams samples are the best
-                next_scores, next_scores_indices = torch.sort(next_scores, descending=True, dim=1)
-                next_tokens = torch.gather(next_tokens, -1, next_scores_indices)  # (batch_size, num_beams * 2)
            else:
                # do greedy beam search
-                scores = F.log_softmax(next_token_logits, dim=-1)  # (batch_size * num_beams, vocab_size)
+                scores = F.log_softmax(scores, dim=-1)  # (batch_size * num_beams, vocab_size)
                assert scores.size() == (batch_size * num_beams, vocab_size)
                # Add the log prob of the new beams to the log prob of the beginning of the sequence (sum of logs == log of the product)
-                next_scores = scores + beam_scores[:, None].expand_as(scores)  # (batch_size * num_beams, vocab_size)
+                _scores = scores + beam_scores[:, None].expand_as(scores)  # (batch_size * num_beams, vocab_size)
                # re-organize to group the beam together (we are keeping top hypothesis accross beams)
-                next_scores = next_scores.view(
-                    batch_size, num_beams * vocab_size
-                )  # (batch_size, num_beams * vocab_size)
+                _scores = _scores.view(batch_size, num_beams * vocab_size)  # (batch_size, num_beams * vocab_size)
+                next_scores, next_words = torch.topk(_scores, 2 * num_beams, dim=1, largest=True, sorted=True)

-                next_scores, next_tokens = torch.topk(next_scores, 2 * num_beams, dim=1, largest=True, sorted=True)
-
-            assert next_scores.size() == next_tokens.size() == (batch_size, 2 * num_beams)
+            assert next_scores.size() == next_words.size() == (batch_size, 2 * num_beams)

            # next batch beam content
            # list of (batch_size * num_beams) tuple(next hypothesis score, next word, current position in the batch)
@@ -1048,22 +1032,21 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
                # next sentence beam content
                next_sent_beam = []

-                # next tokens for this sentence
-                for idx, score in zip(next_tokens[batch_idx], next_scores[batch_idx]):
+                # next words for this sentence
+                for idx, score in zip(next_words[batch_idx], next_scores[batch_idx]):

                    # get beam and word IDs
                    beam_id = idx // vocab_size
-                    token_id = idx % vocab_size
+                    word_id = idx % vocab_size

-                    effective_beam_id = batch_idx * num_beams + beam_id
-                    # add to generated hypotheses if end of sentence
-                    if eos_token_ids is not None and token_id.item() in eos_token_ids:
+                    # add to generated hypotheses if end of sentence or last iteration
+                    if eos_token_ids is not None and word_id.item() in eos_token_ids:
                        generated_hyps[batch_idx].add(
-                            input_ids[effective_beam_id].clone(), score.item(),
+                            input_ids[batch_idx * num_beams + beam_id, :cur_len].clone(), score.item(),
                        )
                    else:
                        # add next predicted word if it is not eos_token
-                        next_sent_beam.append((score, token_id, effective_beam_id))
+                        next_sent_beam.append((score, word_id, batch_idx * num_beams + beam_id))

                    # the beam for next step is full
                    if len(next_sent_beam) == num_beams:
@@ -1077,45 +1060,35 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin):
            # sanity check / prepare next batch
            assert len(next_batch_beam) == batch_size * num_beams
            beam_scores = beam_scores.new([x[0] for x in next_batch_beam])
-            beam_tokens = input_ids.new([x[1] for x in next_batch_beam])
+            beam_words = input_ids.new([x[1] for x in next_batch_beam])
            beam_idx = input_ids.new([x[2] for x in next_batch_beam])

            # re-order batch
            input_ids = input_ids[beam_idx, :]
-            input_ids = torch.cat([input_ids, beam_tokens.unsqueeze(1)], dim=-1)
+            input_ids = torch.cat([input_ids, beam_words.unsqueeze(1)], dim=-1)

            # re-order internal states
            if past:
                past = self._reorder_cache(past, beam_idx)

+            # update current length
+            cur_len = cur_len + 1
+
            # stop when we are done with each sentence
            if all(done):
                break

-            # update current length
-            cur_len = cur_len + 1
-
-        # finalize all open beam hypotheses and end to generated hypotheses
        for batch_idx in range(batch_size):
-            if done[batch_idx]:
-                continue
+            # Add all open beam hypothesis to generated_hyps
+            if not done[batch_idx]:
+                for idx, score in zip(next_words[batch_idx], next_scores[batch_idx]):

-            # test that beam scores match previously calculated scores if not eos and batch_idx not done
-            if eos_token_ids is not None and all(
-                (token_id % vocab_size).item() not in eos_token_ids for token_id in next_tokens[batch_idx]
-            ):
-                assert torch.all(
-                    next_scores[batch_idx, :num_beams] == beam_scores.view(batch_size, num_beams)[batch_idx]
-                ), "If batch_idx is not done, final next scores: {} have to equal to accumulated beam_scores: {}".format(
-                    next_scores[:, :num_beams][batch_idx], beam_scores.view(batch_size, num_beams)[batch_idx]
-                )
-
-            # need to add best num_beams hypotheses to generated hyps
-            for beam_id in range(num_beams):
-                effective_beam_id = batch_idx * num_beams + beam_id
-                final_score = beam_scores[effective_beam_id].item()
-                final_tokens = input_ids[effective_beam_id]
-                generated_hyps[batch_idx].add(final_tokens, final_score)
+                    # get beam and word IDs
+                    beam_id = idx // vocab_size
+                    word_id = idx % vocab_size
+                    generated_hyps[batch_idx].add(
+                        input_ids[batch_idx * num_beams + beam_id, :cur_len].clone(), score.item()
+                    )

        # depending on whether greedy generation is wanted or not define different output_batch_size and output_num_return_sequences_per_batch
        output_batch_size = batch_size if do_sample else batch_size * num_return_sequences
@@ -1852,8 +1852,8 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer):
                stack = tf.stack(stack, axis=0)
            elif return_tensors == "pt":
                stack = torch.stack(stack, dim=0)
-            # elif not return_tensors and len(stack) == 1:
-            #     stack = stack[0]
+            elif not return_tensors and len(stack) == 1:
+                stack = stack[0]

            sanitized[key] = stack

@@ -1902,10 +1902,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer):

        # Return tensor is None, then we can remove the leading batch axis
        if not return_tensors:
-            return {
-                key: value[0] if len(value) > 0 and isinstance(value[0], list) else value
-                for key, value in batched_output.items()
-            }
+            return {key: value[0] if isinstance(value[0], list) else value for key, value in batched_output.items()}
        else:
            return batched_output

@@ -57,18 +57,8 @@ class ConfigTester(object):

        self.parent.assertEqual(config_second.to_dict(), config_first.to_dict())

-    def create_and_test_config_with_num_labels(self):
-        config = self.config_class(**self.inputs_dict, num_labels=5)
-        self.parent.assertEqual(len(config.id2label), 5)
-        self.parent.assertEqual(len(config.label2id), 5)
-
-        config.num_labels = 3
-        self.parent.assertEqual(len(config.id2label), 3)
-        self.parent.assertEqual(len(config.label2id), 3)
-
    def run_common_tests(self):
        self.create_and_test_config_common_properties()
        self.create_and_test_config_to_json_string()
        self.create_and_test_config_to_json_file()
        self.create_and_test_config_from_and_save_pretrained()
-        self.create_and_test_config_with_num_labels()
@@ -78,7 +78,6 @@ class TestCodeExamples(unittest.TestCase):

        for file in files:
            # Open all files
-            print("Testing", file, end=" ")
            with open(os.path.join(directory, file)) as f:
                # Retrieve examples
                examples = get_examples_from_file(f)
@@ -100,7 +99,7 @@ class TestCodeExamples(unittest.TestCase):
                            joined_examples.append(example)
                            joined_examples_index += 1

-                print(str(len(joined_examples)) + "/" + str(len(joined_examples)))
+                print("Testing", file, str(len(joined_examples)) + "/" + str(len(joined_examples)))

                # Execute sub tests with every example.
                for index, code_example in enumerate(joined_examples):
@@ -115,8 +114,7 @@ class TestCodeExamples(unittest.TestCase):

    def test_main_doc_examples(self):
        doc_directory = "docs/source"
-        ignore_files = ["favicon.ico"]
-        self.analyze_directory(doc_directory, ignore_files=ignore_files)
+        self.analyze_directory(doc_directory)

    def test_modeling_examples(self):
        transformers_directory = "src/transformers"
@@ -127,7 +125,5 @@ class TestCodeExamples(unittest.TestCase):
            "modeling_tf_auto.py",
            "modeling_utils.py",
            "modeling_tf_t5.py",
-            "modeling_bart.py",
-            "modeling_tf_utils.py",
        ]
        self.analyze_directory(transformers_directory, identifier=modeling_files, ignore_files=ignore_files)
@@ -29,7 +29,7 @@ if is_torch_available():
    from transformers import (
        AutoModelForSequenceClassification,
        BartModel,
-        BartForConditionalGeneration,
+        BartForMaskedLM,
        BartForSequenceClassification,
        BartConfig,
    )
@@ -97,9 +97,7 @@ def prepare_bart_inputs_dict(
@require_torch
 class BARTModelTest(ModelTesterMixin, unittest.TestCase):

-    all_model_classes = (
-        (BartModel, BartForConditionalGeneration, BartForSequenceClassification) if is_torch_available() else ()
-    )
+    all_model_classes = (BartModel, BartForMaskedLM, BartForSequenceClassification) if is_torch_available() else ()
    is_encoder_decoder = True
    # TODO(SS): fix the below in a separate PR
    test_pruning = False
@@ -223,8 +221,8 @@ class BartHeadTests(unittest.TestCase):

    def test_lm_forward(self):
        config, input_ids, batch_size = self._get_config_and_data(output_past=False)
-        decoder_lm_labels = ids_tensor([batch_size, input_ids.shape[1]], self.vocab_size).to(torch_device)
-        lm_model = BartForConditionalGeneration(config)
+        decoder_lm_labels = ids_tensor([batch_size, input_ids.shape[1]], self.vocab_size)
+        lm_model = BartForMaskedLM(config)
        lm_model.to(torch_device)
        loss, logits, enc_features = lm_model.forward(
            input_ids=input_ids, lm_labels=decoder_lm_labels, decoder_input_ids=input_ids
@@ -245,15 +243,15 @@ class BartHeadTests(unittest.TestCase):
            decoder_ffn_dim=32,
            max_position_embeddings=48,
        )
-        lm_model = BartForConditionalGeneration(config).to(torch_device)
-        context = torch.Tensor([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]]).long().to(torch_device)
-        summary = torch.Tensor([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]]).long().to(torch_device)
-        loss, logits, enc_features = lm_model.forward(input_ids=context, decoder_input_ids=summary, lm_labels=summary)
+        lm_model = BartForMaskedLM(config)
+        context = torch.Tensor([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]]).long()
+        summary = torch.Tensor([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]]).long()
+        logits, enc_features = lm_model.forward(input_ids=context, decoder_input_ids=summary)
        expected_shape = (*summary.shape, config.vocab_size)
        self.assertEqual(logits.shape, expected_shape)

    def test_generate_beam_search(self):
-        input_ids = torch.Tensor([[71, 82, 2], [68, 34, 2]]).long().to(torch_device)
+        input_ids = torch.Tensor([[71, 82, 2], [68, 34, 2]]).long()
        config = BartConfig(
            vocab_size=self.vocab_size,
            d_model=24,
@@ -266,7 +264,7 @@ class BartHeadTests(unittest.TestCase):
            max_position_embeddings=48,
            output_past=True,
        )
-        lm_model = BartForConditionalGeneration(config).to(torch_device)
+        lm_model = BartForMaskedLM(config)
        lm_model.eval()

        new_input_ids = lm_model.generate(
@@ -296,13 +294,6 @@ class BartHeadTests(unittest.TestCase):
            bart_toks = tokenizer.encode(ex, return_tensors="pt")
            _assert_tensors_equal(desired_result.long(), bart_toks, prefix=ex)

-    @unittest.skipIf(torch_device == "cpu", "Cant do half precision")
-    def test_generate_fp16(self):
-        config, input_ids, batch_size = self._get_config_and_data(output_past=True)
-        attention_mask = input_ids.ne(1)
-        lm_model = BartForConditionalGeneration(config).eval().to(torch_device).half()
-        lm_model.generate(input_ids, attention_mask)
-

 def _assert_tensors_equal(a, b, atol=1e-12, prefix=""):
    """If tensors not close, or a and b arent both tensors, raise a nice Assertion error."""
@@ -378,7 +369,7 @@ class BartModelIntegrationTest(unittest.TestCase):

    @slow
    def test_cnn_summarization_same_as_fairseq(self):
-        hf = BartForConditionalGeneration.from_pretrained("bart-large-cnn", output_past=True,).to(torch_device)
+        hf = BartForMaskedLM.from_pretrained("bart-large-cnn", output_past=True,).to(torch_device)
        tok = BartTokenizer.from_pretrained("bart-large")
        text = " (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian"
        tokens = tok.encode(text, return_tensors="pt").to(torch_device)
@@ -526,21 +526,6 @@ class ModelTesterMixin:
            x = model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(x, torch.nn.Linear))

-    def test_correct_missing_keys(self):
-        config, _ = self.model_tester.prepare_config_and_inputs_for_common()
-
-        for model_class in self.all_model_classes:
-            model = model_class(config)
-            base_model_prefix = model.base_model_prefix
-
-            if hasattr(model, base_model_prefix):
-                with tempfile.TemporaryDirectory() as temp_dir_name:
-                    model.base_model.save_pretrained(temp_dir_name)
-                    model, loading_info = model_class.from_pretrained(temp_dir_name, output_loading_info=True)
-
-                    with self.subTest(msg="Missing keys for {}".format(model.__class__.__name__)):
-                        self.assertGreater(len(loading_info["missing_keys"]), 0)
-
    def test_tie_model_weights(self):
        if not self.test_torchscript:
            return
@@ -339,7 +339,14 @@ class GPT2ModelTest(ModelTesterMixin, unittest.TestCase):
            self.assertIsNotNone(model)


+def prepare_generation_special_tokens():
+    return {"bos_token_id": 50256, "eos_token_id": 50256}
+
+
 class GPT2ModelLanguageGenerationTest(unittest.TestCase):
+
+    special_tokens = prepare_generation_special_tokens()
+
    @slow
    def test_lm_generate_gpt2(self):
        model = GPT2LMHeadModel.from_pretrained("gpt2")
@@ -368,7 +375,11 @@ class GPT2ModelLanguageGenerationTest(unittest.TestCase):
        ]  # The dog is cute too. It likes to rub on me and is good for me (the dog
        torch.manual_seed(0)

-        output_ids = model.generate(input_ids)
+        output_ids = model.generate(
+            input_ids,
+            bos_token_id=self.special_tokens["bos_token_id"],
+            eos_token_ids=self.special_tokens["eos_token_id"],
+        )

        self.assertListEqual(output_ids[0].tolist(), expected_output_ids)

@@ -399,5 +410,11 @@ class GPT2ModelLanguageGenerationTest(unittest.TestCase):
            2635,
        ]  # The president of the United States, and the president of the United Kingdom, have been in the White

-        output_ids = model.generate(input_ids, do_sample=False)
+        output_ids = model.generate(
+            input_ids,
+            do_sample=False,
+            bos_token_id=self.special_tokens["bos_token_id"],
+            eos_token_ids=self.special_tokens["eos_token_id"],
+        )
+
        self.assertListEqual(output_ids[0].tolist(), expected_output_ids)
@@ -381,6 +381,7 @@ class TFModelTesterMixin:
        )  # TODO (PVP): ugly workaround to make code work for t5 for the moment - has to changed when t5 is fixed.

        for model_class in self.all_generative_model_classes:
+            # TODO (PVP): add beam search tests when beam search is implemented
            model = model_class(config)

            if config.bos_token_id is None:
@@ -388,34 +389,15 @@ class TFModelTesterMixin:
                    model.generate(max_length=5)
                # batch_size = 1
                self._check_generated_tokens(model.generate(input_ids))
-                # batch_size = 1, num_beams > 1
-                self._check_generated_tokens(model.generate(input_ids, num_beams=3))
            else:
                # batch_size = 1
                self._check_generated_tokens(model.generate(max_length=5))
                # batch_size = 1, num_beams > 1
-                self._check_generated_tokens(model.generate(max_length=5, num_beams=3))
-
-            with self.assertRaises(AssertionError):
-                # generating multiple sequences when greedy no beam generation
-                # is not allowed as it would always generate the same sequences
-                model.generate(input_ids, do_sample=False, num_return_sequences=2)
-
-            with self.assertRaises(AssertionError):
-                # generating more sequences than having beams leads is not possible
-                model.generate(input_ids, do_sample=False, num_return_sequences=3, num_beams=2)

            # batch_size > 1, sample
            self._check_generated_tokens(model.generate(input_ids, num_return_sequences=3))
            # batch_size > 1, greedy
-            self._check_generated_tokens(model.generate(input_ids, do_sample=False))
-
-            # batch_size > 1, num_beams > 1, sample
-            self._check_generated_tokens(model.generate(input_ids, num_beams=3, num_return_sequences=3,))
-            # batch_size > 1, num_beams > 1, greedy
-            self._check_generated_tokens(
-                model.generate(input_ids, do_sample=False, num_beams=3, num_return_sequences=3)
-            )
+            self._check_generated_tokens(model.generate(input_ids, do_sample=False, num_return_sequences=3))

    def _check_generated_tokens(self, output_ids):
        for token_id in output_ids[0].numpy().tolist():
@@ -30,7 +30,6 @@ if is_tf_available():
        TFGPT2LMHeadModel,
        TFGPT2DoubleHeadsModel,
        TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP,
-        shape_list,
    )


@@ -168,73 +167,6 @@ class TFGPT2ModelTest(TFModelTesterMixin, unittest.TestCase):
                list(result["sequence_output"].shape), [self.batch_size, self.seq_length, self.hidden_size],
            )

-        def create_and_check_gpt2_model_past(self, config, input_ids, input_mask, head_mask, token_type_ids, *args):
-            model = TFGPT2Model(config=config)
-
-            # first forward pass
-            output, past = model(input_ids, token_type_ids=token_type_ids)
-
-            # create hypothetical next token and extent to next_input_ids
-            next_tokens = ids_tensor((self.batch_size, 1), config.vocab_size)
-            next_token_types = ids_tensor([self.batch_size, 1], self.type_vocab_size)
-
-            # append to next input_ids and token_type_ids
-            next_input_ids = tf.concat([input_ids, next_tokens], axis=-1)
-            next_token_type_ids = tf.concat([token_type_ids, next_token_types], axis=-1)
-
-            output_from_no_past, _ = model(next_input_ids, token_type_ids=next_token_type_ids)
-            output_from_past, _ = model(next_tokens, token_type_ids=next_token_types, past=past)
-
-            # select random slice
-            random_slice_idx = int(ids_tensor((1,), shape_list(output_from_past)[-1]))
-            output_from_no_past_slice = output_from_no_past[:, -1, random_slice_idx]
-            output_from_past_slice = output_from_past[:, 0, random_slice_idx]
-
-            # test that outputs are equal for slice
-            tf.debugging.assert_near(output_from_past_slice, output_from_no_past_slice, rtol=1e-12)
-
-        def create_and_check_gpt2_model_attention_mask_past(
-            self, config, input_ids, input_mask, head_mask, token_type_ids, *args
-        ):
-            model = TFGPT2Model(config=config)
-
-            # create attention mask
-            half_seq_length = self.seq_length // 2
-            attn_mask_begin = tf.ones((self.batch_size, half_seq_length), dtype=tf.int32)
-            attn_mask_end = tf.zeros((self.batch_size, self.seq_length - half_seq_length), dtype=tf.int32)
-            attn_mask = tf.concat([attn_mask_begin, attn_mask_end], axis=1)
-
-            # first forward pass
-            output, past = model(input_ids, attention_mask=attn_mask)
-
-            # create hypothetical next token and extent to next_input_ids
-            next_tokens = ids_tensor((self.batch_size, 1), config.vocab_size)
-
-            # change a random masked slice from input_ids
-            random_seq_idx_to_change = ids_tensor((1,), half_seq_length).numpy() + 1
-            random_other_next_tokens = ids_tensor((self.batch_size, self.seq_length), config.vocab_size)
-            vector_condition = tf.range(self.seq_length) == (self.seq_length - random_seq_idx_to_change)
-            condition = tf.transpose(
-                tf.broadcast_to(tf.expand_dims(vector_condition, -1), (self.seq_length, self.batch_size))
-            )
-            input_ids = tf.where(condition, random_other_next_tokens, input_ids)
-
-            # append to next input_ids and attn_mask
-            next_input_ids = tf.concat([input_ids, next_tokens], axis=-1)
-            attn_mask = tf.concat([attn_mask, tf.ones((shape_list(attn_mask)[0], 1), dtype=tf.int32)], axis=1)
-
-            # get two different outputs
-            output_from_no_past, _ = model(next_input_ids, attention_mask=attn_mask)
-            output_from_past, _ = model(next_tokens, past=past, attention_mask=attn_mask)
-
-            # select random slice
-            random_slice_idx = int(ids_tensor((1,), shape_list(output_from_past)[-1]))
-            output_from_no_past_slice = output_from_no_past[:, -1, random_slice_idx]
-            output_from_past_slice = output_from_past[:, 0, random_slice_idx]
-
-            # test that outputs are equal for slice
-            tf.debugging.assert_near(output_from_past_slice, output_from_no_past_slice, rtol=1e-12)
-
        def create_and_check_gpt2_lm_head(self, config, input_ids, input_mask, head_mask, token_type_ids, *args):
            model = TFGPT2LMHeadModel(config=config)
            inputs = {
@@ -305,14 +237,6 @@ class TFGPT2ModelTest(TFModelTesterMixin, unittest.TestCase):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_gpt2_model(*config_and_inputs)

-    def test_gpt2_model_past(self):
-        config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_gpt2_model_past(*config_and_inputs)
-
-    def test_gpt2_model_att_mask_past(self):
-        config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_gpt2_model_attention_mask_past(*config_and_inputs)
-
    def test_gpt2_lm_head(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_gpt2_lm_head(*config_and_inputs)
@@ -214,7 +214,14 @@ class TransfoXLModelTest(ModelTesterMixin, unittest.TestCase):
            self.assertIsNotNone(model)


+def prepare_generation_special_tokens():
+    return {"eos_token_id": 0}
+
+
 class TransfoXLModelLanguageGenerationTest(unittest.TestCase):
+
+    special_tokens = prepare_generation_special_tokens()
+
    @slow
    def test_lm_generate_transfo_xl_wt103(self):
        model = TransfoXLLMHeadModel.from_pretrained("transfo-xl-wt103")
@@ -571,5 +578,6 @@ class TransfoXLModelLanguageGenerationTest(unittest.TestCase):

        torch.manual_seed(0)

-        output_ids = model.generate(input_ids, max_length=200)
+        output_ids = model.generate(input_ids, eos_token_ids=self.special_tokens["eos_token_id"], max_length=200)
+
        self.assertListEqual(output_ids[0].tolist(), expected_output_ids)
@@ -399,7 +399,14 @@ class XLMModelTest(ModelTesterMixin, unittest.TestCase):
            self.assertIsNotNone(model)


+def prepare_generation_special_tokens():
+    return {"bos_token_id": 0, "pad_token_id": 2}
+
+
 class XLMModelLanguageGenerationTest(unittest.TestCase):
+
+    special_tokens = prepare_generation_special_tokens()
+
    @slow
    def test_lm_generate_xlm_mlm_en_2048(self):
        model = XLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048")
@@ -428,6 +435,10 @@ class XLMModelLanguageGenerationTest(unittest.TestCase):
        ]  # The dog is nothing is it!!!!!!!!!!!! TODO (PVP): this sentence (and others I tried) does not make much sense, there seems to be a problem with xlm language generation.
        torch.manual_seed(0)

-        output_ids = model.generate(input_ids)
+        output_ids = model.generate(
+            input_ids,
+            bos_token_id=self.special_tokens["bos_token_id"],
+            pad_token_id=self.special_tokens["pad_token_id"],
+        )

        self.assertListEqual(output_ids[0].tolist(), expected_output_ids)
@@ -513,7 +513,14 @@ class XLNetModelTest(ModelTesterMixin, unittest.TestCase):
            self.assertIsNotNone(model)


+def prepare_generation_special_tokens():
+    return {"bos_token_id": 1, "pad_token_id": 5, "eos_token_id": 2}
+
+
 class XLNetModelLanguageGenerationTest(unittest.TestCase):
+
+    special_tokens = prepare_generation_special_tokens()
+
    @slow
    def test_lm_generate_xlnet_base_cased(self):
        model = XLNetLMHeadModel.from_pretrained("xlnet-base-cased")
@@ -910,6 +917,12 @@ class XLNetModelLanguageGenerationTest(unittest.TestCase):
        #  Since, however, he has had difficulty walking with Maria

        torch.manual_seed(0)
-        output_ids = model.generate(input_ids, max_length=200)
+        output_ids = model.generate(
+            input_ids,
+            bos_token_id=self.special_tokens["bos_token_id"],
+            pad_token_id=self.special_tokens["pad_token_id"],
+            eos_token_ids=self.special_tokens["eos_token_id"],
+            max_length=200,
+        )

        self.assertListEqual(output_ids[0].tolist(), expected_output_ids)
@@ -272,30 +272,6 @@ class FastTokenizerMatchingTest(unittest.TestCase):
            # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
            # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

-    def assert_empty_output_no_special_tokens(self, ru_class, py_class, model):
-        tokenizer_r = ru_class.from_pretrained(model, add_special_tokens=False)
-        tokenizer_p = py_class.from_pretrained(model)
-
-        # add_special_tokens=False makes nothing for now.
-        self.assertEqual(
-            tokenizer_p.tokenize(" ", add_special_tokens=False), tokenizer_r.tokenize(" ", add_special_tokens=False)
-        )
-
-        self.assertEqual(
-            tokenizer_p.encode_plus(" ", add_special_tokens=False),
-            tokenizer_r.encode_plus(" ", add_special_tokens=False),
-        )
-
-        self.assertEqual(
-            tokenizer_p.encode_plus(" ", add_special_tokens=False),
-            tokenizer_r.encode_plus(" ", add_special_tokens=False),
-        )
-
-        self.assertEqual(
-            tokenizer_p.batch_encode_plus([" "], add_special_tokens=False),
-            tokenizer_r.batch_encode_plus([" "], add_special_tokens=False),
-        )
-
    def test_bert(self):
        for tokenizer_name in BertTokenizer.pretrained_vocab_files_map["vocab_file"].keys():
            tokenizer_p = BertTokenizer.from_pretrained(tokenizer_name)
@@ -337,9 +313,6 @@ class FastTokenizerMatchingTest(unittest.TestCase):
            # Check for padding
            self.assert_padding(tokenizer_r, tokenizer_p)

-            # Check for space-only input
-            self.assert_empty_output_no_special_tokens(tokenizer_r.__class__, tokenizer_p.__class__, tokenizer_name)
-
    @require_torch
    def test_transfoxl(self):
        for tokenizer_name in TransfoXLTokenizer.pretrained_vocab_files_map["pretrained_vocab_file"].keys():
@@ -396,9 +369,6 @@ class FastTokenizerMatchingTest(unittest.TestCase):
            # self.assertIsNotNone(tokenizer_p.__class__.from_pretrained('./'))
            self.assertIsNotNone(tokenizer_r.__class__.from_pretrained("./"))

-            # Check for space-only input
-            self.assert_empty_output_no_special_tokens(tokenizer_r.__class__, tokenizer_p.__class__, tokenizer_name)
-
    def test_distilbert(self):
        for tokenizer_name in DistilBertTokenizer.pretrained_vocab_files_map["vocab_file"].keys():
            tokenizer_p = DistilBertTokenizer.from_pretrained(tokenizer_name)
@@ -441,9 +411,6 @@ class FastTokenizerMatchingTest(unittest.TestCase):
            # Check for padding
            self.assert_padding(tokenizer_r, tokenizer_p)

-            # Check for space-only input
-            self.assert_empty_output_no_special_tokens(tokenizer_r.__class__, tokenizer_p.__class__, tokenizer_name)
-
    def test_gpt2(self):
        for tokenizer_name in GPT2Tokenizer.pretrained_vocab_files_map["vocab_file"].keys():
            tokenizer_p = GPT2Tokenizer.from_pretrained(tokenizer_name)
@@ -485,9 +452,6 @@ class FastTokenizerMatchingTest(unittest.TestCase):
            # Check for padding
            self.assertRaises(ValueError, self.assert_padding, tokenizer_r, tokenizer_p)

-            # Check for space-only input
-            self.assert_empty_output_no_special_tokens(tokenizer_r.__class__, tokenizer_p.__class__, tokenizer_name)
-
    def test_roberta(self):
        for tokenizer_name in RobertaTokenizer.pretrained_vocab_files_map["vocab_file"].keys():
            tokenizer_p = RobertaTokenizer.from_pretrained(tokenizer_name)
@@ -530,9 +494,6 @@ class FastTokenizerMatchingTest(unittest.TestCase):
            # TODO: Re-enable this test as soon as Roberta align with the python tokenizer.
            # self.assert_padding(tokenizer_r, tokenizer_p)

-            # Check for space-only input
-            self.assert_empty_output_no_special_tokens(tokenizer_r.__class__, tokenizer_p.__class__, tokenizer_name)
-
    def test_openai(self):
        for tokenizer_name in OpenAIGPTTokenizer.pretrained_vocab_files_map["vocab_file"].keys():
            tokenizer_p = OpenAIGPTTokenizer.from_pretrained(tokenizer_name)
@@ -575,6 +536,3 @@ class FastTokenizerMatchingTest(unittest.TestCase):

            # Check the number of returned files for save_vocabulary
            self.assert_save_pretrained(tokenizer_r, tokenizer_p)
-
-            # Check for space-only input
-            self.assert_empty_output_no_special_tokens(tokenizer_r.__class__, tokenizer_p.__class__, tokenizer_name)
Author	SHA1	Message	Date
sshleifer	4b15df209e	fix ==	2020-03-04 11:38:18 -05:00
sshleifer	0c9cc060ee	circleci	2020-03-04 11:36:03 -05:00
				`@@ -1 +0,0 @@`
				`Slavic BERT from https://github.com/deepmipt/Slavic-BERT-NER http://files.deeppavlov.ai/deeppavlov_data/bg_cs_pl_ru_cased_L-12_H-768_A-12.tar.gz`