CircleCI test config

Co-authored-by: sgugger <sylvain.gugger@gmail.com>

.circleci/config.yml

@@ -19,7 +19,7 @@ references:
             command: |
                 gcloud --quiet auth configure-docker
                 cd docker/transformers-pytorch-tpu
-                if [ -z "$CIRCLE_PR_NUMBER" ]; then docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" . ; else docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" --build-arg "GITHUB_REF=pull/$CIRCLE_PR_NUMBER/head" . ; fi
+                if [ -z "$CIRCLE_PR_NUMBER" ]; then docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1"; else docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" --build-arg "GITHUB_REF=pull/$CIRCLE_PR_NUMBER/head" . ; fi
                 docker push "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID"
     deploy_cluster: &deploy_cluster
         run:
@@ -42,9 +42,10 @@ references:
                 # non-zero status code.
                 while [ $i -lt $max_checks ]; do ((i++)); if kubectl get jobs $job_name -o jsonpath='Failed:{.status.failed}' | grep "Failed:1"; then status_code=1 && break; elif kubectl get jobs $job_name -o jsonpath='Succeeded:{.status.succeeded}' | grep "Succeeded:1" ; then status_code=0 && break; else echo "Job not finished yet"; fi; sleep 30; done && \
                 echo "Done waiting. Job status code: $status_code" && \
-                pod_name=$(kubectl get po -l controller-uid=`kubectl get job $job_name -o "jsonpath={.metadata.labels.controller-uid}"` | awk 'match($0,!/NAME/) {print $1}') && \
-                echo "GKE pod name: $pod_name" && \
-                kubectl logs -f $pod_name --container=train
+                # Allow time for logs to flush.
+                sleep 60 && \
+                echo "JOB_NAME: $job_name" && \
+                gcloud logging read "resource.type=k8s_container resource.labels.project_id=$GOOGLE_PROJECT_ID resource.labels.location=$GOOGLE_COMPUTE_ZONE resource.labels.cluster_name=$GKE_CLUSTER resource.labels.namespace_name=default resource.labels.pod_name:$job_name" --limit 10000000 --order asc --format 'value(textPayload)' --project=$GOOGLE_PROJECT_ID && \
                 echo "Done with log retrieval attempt." && \
                 gcloud container images delete "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" --force-delete-tags && \
                 exit $status_code
@@ -244,7 +245,6 @@ jobs:
             - run: black --check --line-length 119 --target-version py35 examples templates tests src utils
             - run: isort --check-only --recursive examples templates tests src utils
             - run: flake8 examples templates tests src utils
-            - run: python utils/check_repo.py
     check_repository_consistency:
         working_directory: ~/transformers
         docker:
@@ -255,6 +255,20 @@ jobs:
             - checkout
             - run: pip install requests
             - run: python ./utils/link_tester.py
+    deploy_notebooks:
+        working_directory: ~/transformers
+        docker:
+            - image: circleci/python:3.6
+        steps:
+            - add_ssh_keys:
+                  fingerprints:
+                      - "5b:7a:95:18:07:8c:aa:76:4c:60:35:88:ad:60:56:71"
+            - checkout
+            - run: pip install --upgrade pip
+            - run: pip install nbformat
+            - run: python .circleci/generate_notebooks.py
+            - run: scp -r -oStrictHostKeyChecking=no ./doc_notebooks/* $doc:$dir/doc_notebooks
+
 
 # TPU JOBS
     run_examples_tpu:
@@ -293,24 +307,35 @@ workflows:
     version: 2
     build_and_test:
         jobs:
-            - check_code_quality
-            - check_repository_consistency
-            - run_examples_torch
-            - run_tests_custom_tokenizers
-            - run_tests_torch_and_tf
-            - run_tests_torch
-            - run_tests_tf
-            - build_doc
-            - deploy_doc: *workflow_filters
-    tpu_testing_jobs:
+#            - check_code_quality
+#            - check_repository_consistency
+#            - run_examples_torch
+#            - run_tests_custom_tokenizers
+#            - run_tests_torch_and_tf
+#            - run_tests_torch
+#            - run_tests_tf
+#            - build_doc
+#            - deploy_doc: *workflow_filters
+            - deploy_notebooks
+    notebooks:
         triggers:
             - schedule:
-                # Set to run at the first minute of every hour.
-                cron: "0 8 * * *"
-                filters:
-                    branches:
-                        only:
-                            - master
+                  cron: "0 8 * * *"
+                  filters:
+                      branches:
+                          only:
+                              - master
         jobs:
-            - cleanup-gke-jobs
-            - run_examples_tpu
+            - deploy_notebooks
+#    tpu_testing_jobs:
+#        triggers:
+#            - schedule:
+#                # Set to run at the first minute of every hour.
+#                cron: "0 8 * * *"
+#                filters:
+#                    branches:
+#                        only:
+#                            - master
+#        jobs:
+#            - cleanup-gke-jobs
+#            - run_examples_tpu

.circleci/generate_notebooks.py

@@ -0,0 +1,476 @@
+import nbformat
+import os
+import re
+import shutil
+
+# Launch from root of repo
+PATH_TO_DOCS = 'docs/source'
+PATH_TO_DEST = 'doc_notebooks'
+DOC_BASE_URL = "https://huggingface.co/transformers/"
+
+# These are the doc files converted, add any new tutorial to this list if you want it handled by the conversion
+# script.
+TUTORIAL_FILES = [
+    "benchmarks.rst",
+    "multilingual.rst",
+    "perplexity.rst",
+    "preprocessing.rst",
+    "quicktour.rst",
+    "task_summary.rst",
+    "tokenizer_summary.rst",
+    "training.rst"
+]
+
+###################################
+# Parsing the rst file            #
+###################################
+
+# Re pattern that catches markdown titles.
+_re_title = re.compile(r"^#+\s+(\S+)")
+# Re pattern that catches rst blocks of the form `.. block_name::`.
+_re_block = re.compile(r"^\.\.\s+(\S+)::")
+# Re pattern that catches what's after the :: in rst blocks of the form `.. block_name:: something`.
+_re_block_lang = re.compile(r"^\.\.\s+\S+::\s*(\S+)(\s+|$)")
+# Re pattern that catchers section names like `.. _name:`.
+_re_anchor_section = re.compile(r"^\.\.\s+_(\S+):")
+# Re pattern that catches indentation at the start of a line.
+_re_indent = re.compile(r"^(\s*)\S")
+
+
+def split_blocks(lines):
+    """ Read the lines of a doc file and group them by blocks."""
+    blocks = []
+    block_type = None
+    current_block = []
+    i = 0
+
+    def _move_to_next_non_empty_line(i):
+        while i < len(lines) and len(lines[i]) == 0:
+            i += 1
+        return i
+
+    def _build_block(blocks, current_block, block_type):
+        if len(current_block) > 0:
+            while len(current_block[-1]) == 0:
+                current_block = current_block[:-1]
+            blocks.append(('\n'.join(current_block), block_type))
+        return blocks, []
+
+    while i < len(lines):
+        line = lines[i]
+        if _re_title.search(line) is not None:
+            blocks, current_block = _build_block(blocks, current_block, "prose")
+            blocks.append((line, "title"))
+            i += 1
+            i = _move_to_next_non_empty_line(i)
+        elif _re_block.search(line) is not None:
+            blocks, current_block = _build_block(blocks, current_block, "prose")
+            block_type = _re_block.search(line).groups()[0]
+            if _re_block_lang.search(line):
+                block_type += " " + _re_block_lang.search(line).groups()[0]
+            i += 1
+            i = _move_to_next_non_empty_line(i)
+            indent = _re_indent.search(lines[i]).groups()[0]
+            if len(indent) > 0:
+                while i < len(lines) and (lines[i].startswith(indent) or len(lines[i]) == 0):
+                    current_block.append(lines[i])
+                    i += 1
+            blocks, current_block = _build_block(blocks, current_block, block_type)
+        elif _re_anchor_section.search(line):
+            blocks, current_block = _build_block(blocks, current_block, "prose")
+            blocks.append((line, "anchor"))
+            i += 1
+            i = _move_to_next_non_empty_line(i)
+        else:
+            current_block.append(line)
+            i += 1
+    blocks, current_block = _build_block(blocks, current_block, "prose")
+    return blocks
+
+
+###################################
+# Text formatting and cleaning    #
+###################################
+
+def process_titles(lines):
+    """ Converts rst titles to markdown titles."""
+    title_chars = """= - ` : ' " ~ ^ _ * + # < >""".split(" ")
+    title_levels = {}
+    new_lines = []
+    for line in lines:
+        if len(new_lines) > 0 and len(line) >= len(new_lines[-1]) and len(set(line)) == 1 and line[
+            0] in title_chars and line != "::":
+            char = line[0]
+            level = title_levels.get(char, len(title_levels) + 1)
+            if level not in title_levels:
+                title_levels[char] = level
+            new_lines[-1] = f"{'#' * level} {new_lines[-1]}"
+        else:
+            new_lines.append(line)
+    return new_lines
+
+
+# Re pattern to catch things inside ` ` in :obj:`thing`.
+_re_obj = re.compile(r":obj:`([^`]+)`")
+# Re pattern to catch things inside ` ` in :math:`thing`.
+_re_math = re.compile(r":math:`([^`]+)`")
+# Re pattern to catch things between single backquotes.
+_re_single_backquotes = re.compile(r"(^|[^`])`([^`]+)`([^`]|$)")
+# Re pattern to catch things between stars.
+_re_stars = re.compile(r"\*([^\*]+)\*")
+# Re pattern to catch things between double backquotes.
+_re_double_backquotes = re.compile(r"``([^`]+)``")
+# Re pattern to catch things inside ` ` in :func/class/meth:`thing`.
+_re_func_class = re.compile(r":(?:func|class|meth):`([^`]+)`")
+
+
+def convert_rst_formatting(text):
+    """ Convert rst syntax for formatting to markdown in text."""
+
+    # Remove :class:, :func: and :meth: markers. Simplify what's inside and put double backquotes
+    # (to not be caught by the italic conversion).
+    def _rep_func_class(match):
+        name = match.groups()[0]
+        splits = name.split('.')
+        i = 0
+        while i < len(splits) - 1 and not splits[i][0].isupper():
+            i += 1
+        return f"``{'.'.join(splits[i:])}``"
+
+    text = _re_func_class.sub(_rep_func_class, text)
+    # Remove :obj: markers. What's after is in a single backquotes so we put in double backquotes
+    # (to not be caught by the italic conversion).
+    text = _re_obj.sub(r"``\1``", text)
+    # Remove :math: markers.
+    text = _re_math.sub(r"$\1$", text)
+    # Convert content in stars to bold
+    text = _re_stars.sub(r'**\1**', text)
+    # Convert content in single backquotes to italic.
+    text = _re_single_backquotes.sub(r'\1*\2*\3', text)
+    # Convert content in double backquotes to single backquotes.
+    text = _re_double_backquotes.sub(r'`\1`', text)
+    # Remove remaining ::
+    text = re.sub(r"::\n", "", text)
+    return text
+
+
+# Re pattern to catch description and url in links of the form `description <url>`_.
+_re_links = re.compile(r"`([^`]+\S)\s+</*([^/][^>`]*)>`_+")
+# Re pattern to catch reference in links of the form :doc:`reference`.
+_re_simple_doc = re.compile(r":doc:`([^`<]*)`")
+# Re pattern to catch description and reference in links of the form :doc:`description <reference>`.
+_re_doc_with_description = re.compile(r":doc:`([^`<]+\S)\s+</*([^/][^>`]*)>`")
+# Re pattern to catch reference in links of the form :ref:`reference`.
+_re_simple_ref = re.compile(r":ref:`([^`<]*)`")
+# Re pattern to catch description and reference in links of the form :ref:`description <reference>`.
+_re_ref_with_description = re.compile(r":ref:`([^`<]+\S)\s+<([^>]*)>`")
+
+
+def convert_rst_links(text):
+    """ Convert the rst links in text to markdown."""
+    # Links of the form :doc:`page`
+    text = _re_simple_doc.sub(r'[\1](' + DOC_BASE_URL + r'\1.html)', text)
+    # Links of the form :doc:`text <page>`
+    text = _re_doc_with_description.sub(r'[\1](' + DOC_BASE_URL + r'\2.html)', text)
+    # Refs of the form :ref:`page`
+    text = _re_simple_ref.sub(r'[\1](#\1)', text)
+    # Refs of the form :ref:`text <page>`
+    text = _re_ref_with_description.sub(r'[\1](#\2)', text)
+
+    # Other links
+    def _rep_links(match):
+        text, url = match.groups()
+        if not url.startswith('http'):
+            url = DOC_BASE_URL + url
+        return f"[{text}]({url})"
+
+    text = _re_links.sub(_rep_links, text)
+    return text
+
+
+###################################
+# Notes, math and reference       #
+###################################
+
+def remove_indentation(text):
+    """ Remove the indendation found in the first line in text."""
+    lines = text.split("\n")
+    indent = _re_indent.search(lines[0]).groups()[0]
+    new_lines = [line[len(indent):] for line in lines]
+    return "\n".join(new_lines)
+
+
+# For now we just do **NOTE_TYPE:** text, maybe there is some clever html solution to have something nicer.
+def convert_to_note(text, note_type):
+    """ Convert text to a note of note_type."""
+    text = remove_indentation(text)
+    lines = text.split("\n")
+    new_lines = [f"> **{note_type.upper()}:** {lines[0]}"]
+    new_lines += [f"> {line}" for line in lines[1:]]
+    return "\n".join(new_lines)
+
+
+def convert_math(text):
+    """ Convert text to disaply mode LaTeX."""
+    text = remove_indentation(text)
+    return f"$${text}$$"
+
+
+def convert_anchor(text):
+    """ Convert text to an anchor that can be used in the notebook."""
+    anchor_name = _re_anchor_section.search(text).groups()[0]
+    return f"<a id='{anchor_name}'></a>"
+
+
+###################################
+# Images                          #
+###################################
+
+_re_attr_rst = re.compile(r"^\s*:(\S+):\s*(\S.*)$")
+
+
+def convert_image(image_name, text, pref=None, origin_folder=None, dest_folder=None):
+    """ Convert text to proper html code for image_name.
+    Optionally copy image from origin_folder to dest_folder."""
+    # Copy the image if necessary
+    if origin_folder is not None and dest_folder is not None:
+        origin_file = os.path.join(origin_folder, image_name)
+        dest_file = os.path.join(dest_folder, image_name)
+        if not os.path.isfile(dest_file):
+            os.makedirs(os.path.dirname(dest_file), exist_ok=True)
+            shutil.copy(origin_file, dest_file)
+    attrs = {'src': image_name if pref is None else os.path.join(pref, image_name)}
+    for line in text.split("\n"):
+        if _re_attr_rst.search(line) is not None:
+            key, attr = _re_attr_rst.search(line).groups()
+            attrs[key] = attr
+    html = " ".join([f'{key}="{value}"' for key, value in attrs.items()])
+    return f"<img {html}/>"
+
+
+###################################
+# Tables                          #
+###################################
+
+# Matches lines with a pattern of a table new line in rst.
+_re_ignore_line_table = re.compile("^(\+[\-\s]+)+\+\s*$")
+# Matches lines with a pattern of a table new line in rst, with a first column empty.
+_re_ignore_line_table1 = re.compile("^\|\s+(\+[\-\s]+)+\+\s*$")
+# Matches lines with a pattern of a first table line in rst.
+_re_sep_line_table = re.compile("^(\+[=\s]+)+\+\s*$")
+
+
+def convert_table(text):
+    """ Convert a table in text from rst to markdown."""
+    lines = text.split("\n")
+    new_lines = []
+    for line in lines:
+        if _re_ignore_line_table.search(line) is not None:
+            continue
+        if _re_ignore_line_table1.search(line) is not None:
+            continue
+        if _re_sep_line_table.search(line) is not None:
+            line = line.replace('=', '-').replace('+', '|')
+        new_lines.append(line)
+    return "\n".join(new_lines)
+
+
+###################################
+# Code cleaning                   #
+###################################
+
+# Matches the pytorch code tag.
+_re_pytorch = re.compile(r"## PYTORCH CODE")
+# Matches the tensorflow code tag.
+_re_tensorflow = re.compile(r"## TENSORFLOW CODE")
+
+
+def split_frameworks(code):
+    """ Split code between the two frameworks (if it has two versions) with PyTorch first."""
+    if _re_pytorch.search(code) is None or _re_tensorflow.search(code) is None:
+        return (code,)
+    lines = code.split("\n")
+    is_pytorch_first = _re_pytorch.search(lines[0]) is not None
+    re_split = _re_tensorflow if is_pytorch_first else _re_pytorch
+    i = 1
+    while re_split.search(lines[i]) is None:
+        i += 1
+    j = i - 1
+    while len(lines[j]) == 0:
+        j -= 1
+    return ("\n".join(lines[:j + 1]), "\n".join(lines[i:])) if is_pytorch_first else (
+    "\n".join(lines[i:]), "\n".join(lines[:j + 1]))
+
+
+# Matches any doctest pattern.
+_re_doctest = re.compile(r"^(>>>|\.\.\.)")
+
+
+def parse_code_and_output(code):
+    """ Parse code to remove indentation, doctest prompts and split between source and theoretical output."""
+    lines = code.split("\n")
+    indent = _re_indent.search(lines[0]).groups()[0]
+    has_doctest = False
+    input_lines = []
+    output_lines = []
+    for line in lines:
+        if len(line) > 0:
+            line = line[len(indent):]
+        if _re_doctest.search(line):
+            has_doctest = True
+            line = line[4:]
+            input_lines.append(line)
+        elif has_doctest:
+            if len(line) > 0:
+                output_lines.append(line)
+        else:
+            input_lines.append(line)
+    return "\n".join(input_lines), "\n".join(output_lines)
+
+
+###################################
+# All together!                   #
+###################################
+
+def markdown_cell(md):
+    """ Create a markdown cell with md inside."""
+    return nbformat.notebooknode.NotebookNode({'cell_type': 'markdown', 'source': md, 'metadata': {}})
+
+
+def code_cell(code, output=None):
+    """ Create a code cell with `code` and optionally, `output`."""
+    if output is None or len(output) == 0:
+        outputs = []
+    else:
+        outputs = [nbformat.notebooknode.NotebookNode({
+            'data': {'text/plain': output},
+            'execution_count': None,
+            'metadata': {},
+            'output_type': 'execute_result'
+        })]
+    return nbformat.notebooknode.NotebookNode(
+        {'cell_type': 'code',
+         'execution_count': None,
+         'source': code,
+         'metadata': {},
+         'outputs': outputs})
+
+
+def create_notebook(cells):
+    """ Create a notebook with `cells`."""
+    return nbformat.notebooknode.NotebookNode(
+        {'cells': cells,
+         'metadata': {},
+         'nbformat': 4,
+         'nbformat_minor': 4,
+         })
+
+
+def rm_first_line(text):
+    """ Remove the first line in `text`."""
+    return '\n'.join(text.split('\n')[1:])
+
+
+# For the first cell of the notebook
+INSTALL_CODE = """# Transformers installation
+! pip install transformers
+# To install from source instead of the last release, comment the command above and uncomment the following one.
+# ! pip install git+https://github.com/huggingface/transformers.git
+"""
+
+
+def convert_rst_file_to_notebook(
+        rst_file,
+        notebook_fname,
+        framework=None,
+        img_prefix=None,
+        origin_folder=None,
+        dest_folder=None
+):
+    r"""
+    Convert rst_file to a notebook named notebook_fname.
+
+    Args:
+        - rst_file (:obj:`str`):
+            The doc file to convert (in rst format).
+        - notebook_fname (:obj:`str`):
+            The output notebook file name (will be replaced if it exists).
+        - framework (:obj:`str`, `optional`):
+            If provided, must be :obj:`"pt"` or :obj:`"tf"`. In this case, only the PyTorch (resp. TensorFlow) version
+            of the code is kept.
+        - img_prefix (:obj:`str`, `optional`):
+            If provided, will be inserted at the beginning of each image filename (in the `pytorch` or `tensorflow`
+            folder, we need to add ../ to each image file to find them).
+        - origin_folder (:obj:`str`, `optional`):
+            If provided in conjunction with :obj:`dest_folder`, images encountered will be copied from this folder to
+            :obj:`dest_folder`.
+        - dest_folder (:obj:`str`, `optional`):
+            If provided in conjunction with :obj:`origin_folder`, images encountered will be copied from
+            :obj:`origin_folder` to this folder.
+    """
+    with open(rst_file, 'r') as f:
+        content = f.read()
+    lines = content.split("\n")
+    lines = process_titles(lines)
+    blocks = split_blocks(lines)
+    cells = [code_cell(INSTALL_CODE)]
+    for block, block_type in blocks:
+        if block_type == 'title' or block_type == 'prose':
+            block = convert_table(convert_rst_formatting(convert_rst_links(block)))
+            cells.append(markdown_cell(block))
+        elif block_type == 'anchor':
+            block = convert_anchor(block)
+            cells.append(markdown_cell(block))
+        elif block_type.startswith('code-block'):
+            codes = split_frameworks(block)
+            if framework == 'pt' and len(codes) > 1:
+                codes = (rm_first_line(codes[0]),)
+            elif framework == 'tf' and len(codes) > 1:
+                codes = (rm_first_line(codes[1]),)
+            for code in codes:
+                source, output = parse_code_and_output(code)
+                if block_type.endswith('bash'):
+                    lines = source.split("\n")
+                    new_lines = [line if line.startswith("#") else f"! {line}" for line in lines]
+                    source = "\n".join(new_lines)
+                cells.append(code_cell(source, output=output))
+        elif block_type.startswith("image"):
+            image_name = block_type[len("image "):]
+            block = convert_image(
+                image_name,
+                block,
+                pref=img_prefix,
+                origin_folder=origin_folder,
+                dest_folder=dest_folder
+            )
+            cells.append(markdown_cell(block))
+        elif block_type == "math":
+            block = convert_math(block)
+            cells.append(markdown_cell(block))
+        else:
+            block = convert_rst_formatting(convert_rst_links(block))
+            block = convert_to_note(block, block_type)
+            cells.append(markdown_cell(block))
+
+    notebook = create_notebook(cells)
+    nbformat.write(notebook, notebook_fname, version=4)
+
+
+def convert_all_tutorials(path_to_docs=None, path_to_dest=None):
+    """ Convert all tutorials into notebooks."""
+    path_to_docs = PATH_TO_DOCS if path_to_docs is None else path_to_docs
+    path_to_dest = PATH_TO_DEST if path_to_dest is None else path_to_dest
+    for folder in ["pytorch", "tensorflow"]:
+        os.makedirs(os.path.join(path_to_dest, folder), exist_ok=True)
+    for file in TUTORIAL_FILES:
+        notebook_name = os.path.splitext(file)[0] + ".ipynb"
+        doc_file = os.path.join(path_to_docs, file)
+        notebook_file = os.path.join(path_to_dest, notebook_name)
+        convert_rst_file_to_notebook(doc_file, notebook_file, origin_folder=path_to_docs, dest_folder=path_to_dest)
+        for folder, framework in zip(["pytorch", "tensorflow"], ["pt", "tf"]):
+            notebook_file = os.path.join(os.path.join(path_to_dest, folder), notebook_name)
+            convert_rst_file_to_notebook(doc_file, notebook_file, framework=framework, img_prefix="..")
+
+
+if __name__ == "__main__":
+    convert_all_tutorials()

.github/workflows/github-push.yml

@@ -0,0 +1,19 @@
+name: GitHub-hosted runner
+
+on: push
+
+jobs:
+  check_code_quality:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    # - name: Install dependencies
+    #   run: |
+    #     pip install .[tf,torch,quality]
+
+
+

.github/workflows/github-torch-hub.yml

@@ -18,14 +18,6 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: 3.7
-
-    - name: Loading cache
-      uses: actions/cache@v2
-      id: cache
-      with:
-        path: ~/.cache/pip
-        key: v0-torch_hub-${{ hashFiles('setup.py') }}
-
     - name: Install dependencies
       run: |
         pip install --upgrade pip

.github/workflows/self-push.yml

@@ -25,14 +25,6 @@ jobs:
     - name: Current dir
       run: pwd
     - run: nvidia-smi
-
-    - name: Loading cache.
-      uses: actions/cache@v2
-      id: cache
-      with:
-        path: .env
-        key: v0-tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
-
     - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
       run: |
         python -m venv .env

.github/workflows/self-scheduled.yml

@@ -13,14 +13,6 @@ jobs:
     runs-on: self-hosted
     steps:
     - uses: actions/checkout@v2
-
-    - name: Loading cache.
-      uses: actions/cache@v2
-      id: cache
-      with:
-        path: .env
-        key: v0-slow_tests_tf_torch_gpu-${{ hashFiles('setup.py') }}
-
     - name: Python version
       run: |
         which python
@@ -30,7 +22,6 @@ jobs:
       run: pwd
     - run: nvidia-smi
     - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
-      if: steps.cache.outputs.cache-hit != 'true'
       run: |
         python -m venv .env
         source .env/bin/activate
@@ -41,7 +32,7 @@ jobs:
       run: |
         source .env/bin/activate
         pip install --upgrade pip
-        pip install torch!=1.6.0
+        pip install torch!=1.6.0 --no-cache-dir
         pip install .[sklearn,testing]
 
     - name: Are GPUs recognized by our DL frameworks

docs/source/_static/css/huggingface.css

@@ -1,36 +1,5 @@
 /* Our DOM objects */
 
-/* Colab dropdown */
-
-.colab-dropdown {
-    position: relative;
-    display: inline-block;
-}
-  
-.colab-dropdown-content {
-    display: none;
-    position: absolute;
-    background-color: #f9f9f9;
-    min-width: 117px;
-    box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
-    z-index: 1;
-}
-  
-.colab-dropdown-content button {
-    color: #6670FF;
-    background-color: #f9f9f9;
-    font-size: 12px;
-    border: none;
-    min-width: 117px;
-    padding: 5px 5px;
-    text-decoration: none;
-    display: block;
-}
-  
-.colab-dropdown-content button:hover {background-color: #eee;}
-  
-.colab-dropdown:hover .colab-dropdown-content {display: block;}
-
 /* Version control */
 
 .version-button {

docs/source/_static/js/custom.js

@@ -21,17 +21,6 @@ const versionMapping = {
     "v1.1.0": "v1.1.0",
     "v1.0.0": "v1.0.0"
 }
-// The page that have a notebook and therefore should have the open in colab badge.
-const hasNotebook = [
-    "benchmarks",
-    "multilingual",
-    "perplexity",
-    "preprocessing",
-    "quicktour",
-    "task_summary",
-    "tokenizer_summary",
-    "training"
-];
 
 function addIcon() {
     const huggingFaceLogo = "https://huggingface.co/landing/assets/transformers-docs/huggingface_logo.svg";
@@ -93,26 +82,6 @@ function addGithubButton() {
     document.querySelector(".wy-side-nav-search .icon-home").insertAdjacentHTML('afterend', div);
 }
 
-function addColabLink() {
-    const parts = location.toString().split('/');
-    const pageName = parts[parts.length - 1].split(".")[0];
-
-    if (hasNotebook.includes(pageName)) {
-        const baseURL = "https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/"
-        const linksColab = `
-        <div class="colab-dropdown">
-            <img alt="Open In Colab" src="https://colab.research.google.com/assets/colab-badge.svg">
-            <div class="colab-dropdown-content">
-                <button onclick=" window.open('${baseURL}${pageName}.ipynb')">Mixed</button>
-                <button onclick=" window.open('${baseURL}pytorch/${pageName}.ipynb')">PyTorch</button>
-                <button onclick=" window.open('${baseURL}tensorflow/${pageName}.ipynb')">TensorFlow</button>
-            </div>
-        </div>`
-        const leftMenu = document.querySelector(".wy-breadcrumbs-aside")
-        leftMenu.innerHTML = linksColab + '\n' + leftMenu.innerHTML
-    }
-}
-
 function addVersionControl() {
     // To grab the version currently in view, we parse the url
     const parts = location.toString().split('/');
@@ -286,7 +255,6 @@ function onLoad() {
     addGithubButton();
     parseGithubButtons();
     addHfMenu();
-    addColabLink();
     platformToggle();
 }
 

docs/source/index.rst

@@ -124,11 +124,7 @@ conversion utilities for the following models:
 22. `DPR <https://github.com/facebookresearch/DPR>`_ (from Facebook) released with the paper `Dense Passage Retrieval
     for Open-Domain Question Answering <https://arxiv.org/abs/2004.04906>`_ by Vladimir Karpukhin, Barlas Oğuz, Sewon
     Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
-23. `Pegasus <https://github.com/google-research/pegasus>`_ (from Google) released with the paper `PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization
-    <https://arxiv.org/abs/1912.08777>`_ by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
-24. `MBart <https://github.com/pytorch/fairseq/tree/master/examples/mbart>`_ (from Facebook) released with the paper  `Multilingual Denoising Pre-training for Neural Machine Translation <https://arxiv.org/abs/2001.08210>`_ by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov
-    Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.  
-25. `Other community models <https://huggingface.co/models>`_, contributed by the `community
+23. `Other community models <https://huggingface.co/models>`_, contributed by the `community
     <https://huggingface.co/users>`_.
 
 .. toctree::
@@ -209,8 +205,6 @@ conversion utilities for the following models:
     model_doc/retribert
     model_doc/mobilebert
     model_doc/dpr
-    model_doc/pegasus
-    model_doc/mbart
     internal/modeling_utils
     internal/tokenization_utils
-    internal/pipelines_utils
+    internal/pipelines_utils

docs/source/model_doc/bart.rst

@@ -17,23 +17,13 @@ According to the abstract,
 The Authors' code can be found `here <https://github.com/pytorch/fairseq/tree/master/examples/bart>`_
 
 
-Implementation Notes
-~~~~~~~~~~~~~~~~~~~~
+Implementation Notes:
 
 - Bart doesn't use :obj:`token_type_ids` for sequence classification. Use BartTokenizer.encode to get the proper splitting.
 - The forward pass of ``BartModel`` will create decoder inputs (using the helper function ``transformers.modeling_bart._prepare_bart_decoder_inputs``)  if they are not passed. This is different than some other modeling APIs.
 - Model predictions are intended to be identical to the original implementation. This only works, however, if the string you pass to ``fairseq.encode`` starts with a space.
 - ``BartForConditionalGeneration.generate`` should be used for conditional generation tasks like summarization, see the example in that docstrings
 - Models that load the ``"facebook/bart-large-cnn"`` weights will not have a ``mask_token_id``, or be able to perform mask filling tasks.
-- for training/forward passes that don't involve beam search, pass ``use_cache=False``
-
-
-BartForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.BartForConditionalGeneration
-    :members: generate, forward
-
 
 BartConfig
 ~~~~~~~~~~~~~~~~~~~~~
@@ -49,6 +39,17 @@ BartTokenizer
     :members:
 
 
+MBartTokenizer
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.MBartTokenizer
+    :members: build_inputs_with_special_tokens, prepare_translation_batch
+
+BartForConditionalGeneration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: transformers.BartForConditionalGeneration
+    :members: generate, forward
 
 BartModel
 ~~~~~~~~~~~~~

docs/source/model_doc/longformer.rst

@@ -102,25 +102,3 @@ LongformerForQuestionAnswering
 
 .. autoclass:: transformers.LongformerForQuestionAnswering
     :members:
-
-
-TFLongformerModel
-~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFLongformerModel
-    :members:
-
-
-TFLongformerForMaskedLM
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFLongformerForMaskedLM
-    :members:
-
-
-TFLongformerForQuestionAnswering
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.TFLongformerForQuestionAnswering
-    :members:
-

docs/source/model_doc/marian.rst

@@ -48,7 +48,7 @@ Example of translating english to many romance languages, using language codes:
     tokenizer = MarianTokenizer.from_pretrained(model_name)
     print(tokenizer.supported_language_codes)
     model = MarianMTModel.from_pretrained(model_name)
-    translated = model.generate(**tokenizer.prepare_seq2seq_batch(src_text))
+    translated = model.generate(**tokenizer.prepare_translation_batch(src_text))
     tgt_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
     # ["c'est une phrase en anglais que nous voulons traduire en français",
     # 'Isto deve ir para o português.',
@@ -86,14 +86,6 @@ Code to see available pretrained models:
     suffix = [x.split('/')[1] for x in model_ids]
     multi_models = [f'{org}/{s}' for s in suffix if s != s.lower()]
 
-MarianMTModel
-~~~~~~~~~~~~~
-
-Pytorch version of marian-nmt's transformer.h (c++). Designed for the OPUS-NMT translation checkpoints.
-Model API is identical to BartForConditionalGeneration.
-Available models are listed at `Model List <https://huggingface.co/models?search=Helsinki-NLP>`__
-This class inherits nearly all functionality from ``BartForConditionalGeneration``, see that page for method signatures.
-
 MarianConfig
 ~~~~~~~~~~~~~~~~~~~
 .. autoclass:: transformers.MarianConfig
@@ -104,8 +96,16 @@ MarianTokenizer
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: transformers.MarianTokenizer
-    :members: prepare_seq2seq_batch
+    :members: prepare_translation_batch
 
 
+MarianMTModel
+~~~~~~~~~~~~~
 
+Pytorch version of marian-nmt's transformer.h (c++). Designed for the OPUS-NMT translation checkpoints.
+Model API is identical to BartForConditionalGeneration.
+Available models are listed at `Model List <https://huggingface.co/models?search=Helsinki-NLP>`__
+This class inherits all functionality from ``BartForConditionalGeneration``, see that page for method signatures.
 
+.. autoclass:: transformers.MarianMTModel
+    :members:

docs/source/model_doc/mbart.rst

@@ -1,37 +0,0 @@
-MBart
-----------------------------------------------------
-**DISCLAIMER:** If you see something strange,
-file a `Github Issue <https://github.com/huggingface/transformers/issues/new?assignees=&labels=&template=bug-report.md&title>`__ and assign
-@sshleifer
-
-Overview
-~~~~~~~~~~~~~~~~~~~~~
-The MBart model was presented in `Multilingual Denoising Pre-training for Neural Machine Translation <https://arxiv.org/abs/2001.08210>`_ by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov
-Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer. According to the abstract,
-
-MBART is a sequence-to-sequence denoising auto-encoder pre-trained on large-scale monolingual corpora in many languages using the BART objective. mBART is one of the first methods for pre-training a complete sequence-to-sequence model by denoising full texts in multiple languages, while previous approaches have focused only on the encoder, decoder, or reconstructing parts of the text.
-
-The Authors' code can be found `here <https://github.com/pytorch/fairseq/tree/master/examples/mbart>`__
-
-
-MBartConfig
-~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MBartConfig
-    :members:
-
-
-MBartTokenizer
-~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MBartTokenizer
-    :members: build_inputs_with_special_tokens, prepare_seq2seq_batch
-
-
-MBartForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.MBartForConditionalGeneration
-    :members: generate, forward
-
-

docs/source/model_doc/pegasus.rst

@@ -1,115 +0,0 @@
-Pegasus
-----------------------------------------------------
-**DISCLAIMER:** If you see something strange,
-file a `Github Issue <https://github.com/huggingface/transformers/issues/new?assignees=sshleifer&labels=&template=bug-report.md&title>`__ and assign
-@sshleifer.
-
-
-Overview
-~~~~~~~~~~~~~~~~~~~~~
-
-The Pegasus model was `proposed <https://arxiv.org/abs/1910.13461>`_ by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu on Dec 18, 2019.
-According to the abstract,
-
-- Pegasus' pretraining task is intentionally similar to summarization: important sentences are removed/masked from an input document and are generated together as one output sequence from the remaining sentences, similar to an extractive summary.
-- Pegasus achieves SOTA summarization performance on all 12 downstream tasks, as measured by ROUGE and human eval.
-
-The Authors' code can be found `here <https://github.com/google-research/pegasus>`_
-
-
-Checkpoints
-~~~~~~~~~~~
-The `checkpoints <https://huggingface.co/models?search=pegasus>`_ all checkpoints are finetuned for summarization, besides ``pegasus-large``, whence the other checkpoints are finetuned.
-- Each checkpoint is 2.2 GB on disk and 568M parameters.
-- FP16 is not supported (help/ideas on this appreciated!).
-- Summarizing xsum in fp32 takes about 400ms/sample, with default parameters on a v100 GPU.
-- For XSUM, The paper reports rouge1,rouge2, rougeL of paper: 47.21/24.56/39.25. As of Aug 9, this port scores 46.91/24.34/39.1.
-The gap is likely because of different alpha/length_penalty implementations in beam search.
-
-
-Implementation Notes
-~~~~~~~~~~~~~~~~~~~~
-
-- All models are transformer encoder-decoders with 16 layers in each component.
-- The implementation is completely inherited from ``BartForConditionalGeneration``
-- Some key configuration differences:
-    - static, sinusoidal position embeddings
-    - no ``layernorm_embedding`` (``PegasusConfig.normalize_embedding=False``)
-    - the model starts generating with pad_token_id (which has 0 token_embedding) as the prefix.
-    - ``num_beams=8``
-- All pretrained pegasus checkpoints are the same besides three attributes: ``tokenizer.model_max_length`` (max input size),  ``max_length`` (max num tokens to generate) and ``length_penalty``
-- Code to convert checkpoints trained in the author's `repo <https://github.com/google-research/pegasus>`_ can be found in ``convert_pegasus_tf_to_pytorch.py``
-
-
-Usage Example
-~~~~~~~~~~~~~~~~~~~~
-
-.. code-block:: python
-
-    from transformers import PegasusForConditionalGeneration, PegasusTokenizer
-    src_text = [
-        """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."""
-    ]
-
-    model_name = 'google/pegasus-xsum'
-    torch_device = 'cuda' if torch.cuda.is_available() else 'cpu'
-    tokenizer = PegasusTokenizer.from_pretrained(model_name)
-    model = PegasusForConditionalGeneration.from_pretrained(model_name).to(torch_device)
-    batch = tokenizer.prepare_seq2seq_batch(src_text, truncation=True, padding='longest').to(torch_device)
-    translated = model.generate(**batch)
-    tgt_text = tokenizer.batch_decode(translated, skip_special_tokens=True)
-    assert tgt_text[0] == "California's largest electricity provider has turned off power to tens of thousands of customers."
-
-PegasusForConditionalGeneration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This class inherits all functionality from ``BartForConditionalGeneration``, see that page for method signatures.
-Available models are listed at `Model List <https://huggingface.co/models?search=pegasus>`__
-
-.. autoclass:: transformers.PegasusForConditionalGeneration
-    :members:
-
-
-PegasusConfig
-~~~~~~~~~~~~~~~~~~~
-This config fully inherits from ``BartConfig``, but pegasus uses different default values:
-Up to date parameter values can be seen in `S3 <https://s3.amazonaws.com/models.huggingface.co/bert/google/pegasus-xsum/config.json>`_.
-As of Aug 10, 2020, they are:
-
-.. code-block:: python
-
-    dict(
-    vocab_size=96103,
-    max_position_embeddings=512,
-    d_model=1024,
-    encoder_ffn_dim=4096,
-    decoder_ffn_dim=4096,
-    encoder_attention_heads=16,
-    decoder_attention_heads=16,
-    encoder_layers=16,
-    decoder_layers=16,
-    dropout=0.1,
-    attention_dropout=0.1,
-    activation_dropout=0.1,
-    pad_token_id=0,
-    eos_token_id=1,
-    is_encoder_decoder=True,
-    normalize_before=True,
-    scale_embedding=True,
-    normalize_embedding=False,
-    add_final_layer_norm=True,
-    static_position_embeddings=True,
-    num_beams=8,
-    activation_function="relu",
-    )
-
-
-PegasusTokenizer
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-warning: ``add_tokens`` does not work at the moment.
-
-.. autoclass:: transformers.PegasusTokenizer
-    :members: __call__, prepare_seq2seq_batch
-
-
-

docs/source/model_doc/roberta.rst

@@ -63,13 +63,6 @@ RobertaModel
     :members:
 
 
-RobertaForCausalLM
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autoclass:: transformers.RobertaForCausalLM
-    :members:
-
-
 RobertaForMaskedLM
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/pretrained_models.rst

@@ -353,8 +353,6 @@ For a list that includes community-uploaded models, refer to `https://huggingfac
 | MarianMT          | ``Helsinki-NLP/opus-mt-{src}-{tgt}``                       | | 12-layer, 512-hidden, 8-heads, ~74M parameter Machine translation models. Parameter counts vary depending on vocab size.            |
 |                   |                                                            | | (see `model list <https://huggingface.co/Helsinki-NLP>`_)                                                                           |
 +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
-| Pegasus           | ``google/pegasus-{dataset}``                               | | 16-layer, 1024-hidden, 16-heads, ~568M parameter, 2.2 GB for summary. `model list <https://huggingface.co/models?search=pegasus>`__ |
-+-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
 | Longformer        | ``allenai/longformer-base-4096``                           | | 12-layer, 768-hidden, 12-heads, ~149M parameters                                                                                    |
 |                   |                                                            | | Starting from RoBERTa-base checkpoint, trained on documents of max length 4,096                                                     |
 |                   +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+

examples/bert-loses-patience/test_run_glue_with_pabee.py

@@ -1,6 +1,5 @@
 import argparse
 import logging
-import shutil
 import sys
 import unittest
 from unittest.mock import patch
@@ -20,10 +19,6 @@ def get_setup_file():
     return args.f
 
 
-def clean_test_dir(path):
-    shutil.rmtree(path, ignore_errors=True)
-
-
 class PabeeTests(unittest.TestCase):
     def test_run_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
@@ -37,6 +32,7 @@ class PabeeTests(unittest.TestCase):
             --task_name mrpc
             --do_train
             --do_eval
+            --output_dir ./tests/fixtures/tests_samples/temp_dir
             --per_gpu_train_batch_size=2
             --per_gpu_eval_batch_size=1
             --learning_rate=2e-5
@@ -45,13 +41,8 @@ class PabeeTests(unittest.TestCase):
             --overwrite_output_dir
             --seed=42
             --max_seq_length=128
-            """
-        output_dir = "./tests/fixtures/tests_samples/temp_dir_{}".format(hash(testargs))
-        testargs += "--output_dir " + output_dir
-        testargs = testargs.split()
+            """.split()
         with patch.object(sys, "argv", testargs):
             result = run_glue_with_pabee.main()
             for value in result.values():
                 self.assertGreaterEqual(value, 0.75)
-
-        clean_test_dir(output_dir)

examples/lightning_base.py

@@ -25,7 +25,6 @@ from transformers.optimization import (
     get_cosine_schedule_with_warmup,
     get_cosine_with_hard_restarts_schedule_with_warmup,
     get_linear_schedule_with_warmup,
-    get_polynomial_decay_schedule_with_warmup,
 )
 
 
@@ -49,7 +48,7 @@ arg_to_scheduler = {
     "linear": get_linear_schedule_with_warmup,
     "cosine": get_cosine_schedule_with_warmup,
     "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
-    "polynomial": get_polynomial_decay_schedule_with_warmup,
+    # polynomial': '',                       # TODO
     # '': get_constant_schedule,             # not supported for now
     # '': get_constant_schedule_with_warmup, # not supported for now
 }
@@ -246,8 +245,7 @@ class BaseTransformer(pl.LightningModule):
 
 class LoggingCallback(pl.Callback):
     def on_batch_end(self, trainer, pl_module):
-        lr_scheduler = trainer.lr_schedulers[0]["scheduler"]
-        lrs = {f"lr_group_{i}": lr for i, lr in enumerate(lr_scheduler.get_lr())}
+        lrs = {f"lr_group_{i}": param["lr"] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups)}
         pl_module.logger.log_metrics(lrs)
 
     def on_validation_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule):

examples/requirements.txt

@@ -14,5 +14,3 @@ elasticsearch
 pandas
 nlp
 fire
-pytest
-conllu

examples/seq2seq/README.md

@@ -63,7 +63,7 @@ Summarization Tips:
 (It rarely makes sense to start from `bart-large` unless you are a researching finetuning methods).
 
 **Update 2018-07-18**
-Datasets: `Seq2SeqDataset` should be used for all tokenizers without a `prepare_seq2seq_batch` method. For those who do (like Marian, MBart), `TranslationDataset` should be used.**
+Datasets: `Seq2SeqDataset` should be used for all tokenizers without a `prepare_translation_batch` method. For those who do (like Marian, MBart), `TranslationDataset` should be used.**
 A new dataset is needed to support multilingual tasks.
 
 

examples/seq2seq/distillation.py

@@ -10,40 +10,20 @@ from torch import nn
 from torch.nn import functional as F
 
 from lightning_base import generic_train
-from transformers import (
-    AdamW,
-    BartConfig,
-    BartForConditionalGeneration,
-    MBartTokenizer,
-    T5Config,
-    T5ForConditionalGeneration,
-)
+from transformers import AdamW, BartConfig, BartForConditionalGeneration, T5Config, T5ForConditionalGeneration
 
 
 try:
-    from .finetune import SummarizationModule, TranslationModule
-    from .initialization_utils import init_student, copy_layers
-    from .utils import (
-        use_task_specific_params,
-        pickle_load,
-        freeze_params,
-        assert_all_frozen,
-        any_requires_grad,
-        calculate_bleu_score,
-    )
+    from .finetune import SummarizationModule
     from .finetune import main as ft_main
+    from .initialization_utils import init_student, copy_layers
+    from .utils import use_task_specific_params, pickle_load, freeze_params, assert_all_frozen, any_requires_grad
+
 except ImportError:
-    from finetune import SummarizationModule, TranslationModule
+    from finetune import SummarizationModule
     from finetune import main as ft_main
     from initialization_utils import init_student, copy_layers
-    from utils import (
-        use_task_specific_params,
-        pickle_load,
-        freeze_params,
-        assert_all_frozen,
-        any_requires_grad,
-        calculate_bleu_score,
-    )
+    from utils import use_task_specific_params, pickle_load, freeze_params, assert_all_frozen, any_requires_grad
 
 
 class BartSummarizationDistiller(SummarizationModule):
@@ -179,7 +159,17 @@ class BartSummarizationDistiller(SummarizationModule):
     @staticmethod
     def add_model_specific_args(parser, root_dir):
         SummarizationModule.add_model_specific_args(parser, root_dir)
-        add_distill_args(parser)
+        parser.add_argument("--teacher", default="facebook/bart-large-cnn", type=str)
+        parser.add_argument("--alpha_ce", default=0.8, type=float)
+        parser.add_argument("--alpha_mlm", default=0.2, type=float)
+        # parser.add_argument("--alpha_cos", default=0.0, type=float)
+        parser.add_argument("--alpha_encoder_loss", default=0.0, type=float)
+        parser.add_argument("--alpha_hid", default=0.0, type=float, required=False)
+        parser.add_argument("--student_decoder_layers", default=12, type=int, required=False)
+        parser.add_argument("--student_encoder_layers", default=12, type=int, required=False)
+        parser.add_argument("--no_teacher", action="store_true", default=False)
+        parser.add_argument("--length_penalty", type=float, default=-1)
+
         return parser
 
     def _step(self, batch):
@@ -257,44 +247,6 @@ class BartSummarizationDistiller(SummarizationModule):
         return sum(hidden_losses)
 
 
-def add_distill_args(parser):
-    parser.add_argument("--teacher", default="facebook/bart-large-cnn", type=str)
-    parser.add_argument("--alpha_ce", default=0.8, type=float)
-    parser.add_argument("--alpha_mlm", default=0.2, type=float)
-    parser.add_argument("--alpha_encoder_loss", default=0.0, type=float)
-    parser.add_argument("--alpha_hid", default=0.0, type=float, required=False)
-    parser.add_argument("--student_decoder_layers", default=12, type=int, required=False)
-    parser.add_argument("--student_encoder_layers", default=12, type=int, required=False)
-    parser.add_argument("--no_teacher", action="store_true", default=False)
-    parser.add_argument("--length_penalty", type=float, default=-1)
-
-
-class BartTranslationDistiller(BartSummarizationDistiller):
-    mode = "translation"
-    loss_names = ["loss"]
-    metric_names = ["bleu"]
-    val_metric = "bleu"
-
-    def __init__(self, hparams, **kwargs):
-        super().__init__(hparams, **kwargs)
-        assert isinstance(self.tokenizer, MBartTokenizer)
-        assert hparams.src_lang is not None
-        assert hparams.tgt_lang is not None
-        self.dataset_kwargs["src_lang"] = hparams.src_lang
-        self.dataset_kwargs["tgt_lang"] = hparams.tgt_lang
-        if self.model.config.decoder_start_token_id is None and isinstance(self.tokenizer, MBartTokenizer):
-            self.decoder_start_token_id = self.tokenizer.lang_code_to_id[hparams.tgt_lang]
-
-    def calc_generative_metrics(self, preds, target) -> dict:
-        return calculate_bleu_score(preds, target)
-
-    @staticmethod
-    def add_model_specific_args(parser, root_dir):
-        TranslationModule.add_model_specific_args(parser, root_dir)
-        add_distill_args(parser)
-        return parser
-
-
 class T5SummarizationDistiller(BartSummarizationDistiller):
     def pre_init(self, hparams):
         raise NotImplementedError("T5 Distillation does not work yet")
@@ -412,14 +364,15 @@ class T5SummarizationDistiller(BartSummarizationDistiller):
 def create_module(args):
     t5 = "t5" in args.model_name_or_path
     if args.no_teacher:
-        module_cls = TranslationModule if "translation" in args.task else SummarizationModule
-    elif t5:  # DISTILL T5 WITH TEACHER FOR SUMMARIZATION
-        assert "translation" not in args.task, "t5 translation distillation not supported"
+        assert not args.enc_only
+        module_cls = SummarizationModule
+    elif t5:
         module_cls = T5SummarizationDistiller
-    else:  # DISTILL WITH TEACHER
-        module_cls = BartTranslationDistiller if "translation" in args.task else BartSummarizationDistiller
+    elif args.enc_only:
+        raise ValueError("Deleted that")
+    else:
+        module_cls = BartSummarizationDistiller
     args.setup_cls: str = module_cls.__name__
-    print(f"using module {args.setup_cls}")
     model = module_cls(args)
     return model
 
@@ -460,18 +413,6 @@ def get_layers_to_copy(n_to_get, tot):
             12: all_layers,
         }
         return layers_to_copy[n_to_get]
-    elif tot == 16:
-        layers_to_copy = {  # maps  num layers in student -> which teacher layers to copy
-            1: [0],
-            2: [0, 8],
-            3: [0, 8, 15],
-            4: [0, 5, 10, 15],
-            6: [0, 3, 6, 9, 12, 15],
-            8: [0, 2, 4, 6, 8, 10, 12, 15],
-            9: [0, 1, 3, 5, 7, 9, 11, 13, 15],
-            16: all_layers,
-        }
-        return layers_to_copy[n_to_get]
     else:
         return all_layers[:n_to_get]  # TODO: better version on theseus-bart branch
 

examples/seq2seq/download_wmt.py

@@ -1,52 +0,0 @@
-from pathlib import Path
-
-import fire
-from tqdm import tqdm
-
-
-def download_wmt_dataset(src_lang="ro", tgt_lang="en", dataset="wmt16", save_dir=None) -> None:
-    """Download a dataset using the nlp package and save it to the format expected by finetune.py
-    Format of save_dir: train.source, train.target, val.source, val.target, test.source, test.target.
-
-    Args:
-        src_lang: <str> source language
-        tgt_lang: <str> target language
-        dataset: <str> wmt16, wmt17, etc. wmt16 is a good start as it's small. To get the full list run `import nlp; print([d.id for d in nlp.list_datasets() if "wmt" in d.id])`
-        save_dir: <str>, where to save the datasets, defaults to f'{dataset}-{src_lang}-{tgt_lang}'
-
-    Usage:
-        >>> download_wmt_dataset('ro', 'en', dataset='wmt16') # saves to wmt16-ro-en
-    """
-    try:
-        import nlp
-    except (ModuleNotFoundError, ImportError):
-        raise ImportError("run pip install nlp")
-    pair = f"{src_lang}-{tgt_lang}"
-    print(f"Converting {dataset}-{pair}")
-    ds = nlp.load_dataset(dataset, pair)
-    if save_dir is None:
-        save_dir = f"{dataset}-{pair}"
-    save_dir = Path(save_dir)
-    save_dir.mkdir(exist_ok=True)
-
-    for split in ds.keys():
-        print(f"Splitting {split} with {ds[split].num_rows} records")
-
-        # to save to val.source, val.target like summary datasets
-        fn = "val" if split == "validation" else split
-        src_path = save_dir.joinpath(f"{fn}.source")
-        tgt_path = save_dir.joinpath(f"{fn}.target")
-        src_fp = src_path.open("w+")
-        tgt_fp = tgt_path.open("w+")
-
-        # reader is the bottleneck so writing one record at a time doesn't slow things down
-        for x in tqdm(ds[split]):
-            ex = x["translation"]
-            src_fp.write(ex[src_lang] + "\n")
-            tgt_fp.write(ex[tgt_lang] + "\n")
-
-    print(f"Saved {dataset} dataset to {save_dir}")
-
-
-if __name__ == "__main__":
-    fire.Fire(download_wmt_dataset)

examples/seq2seq/finetune_pegasus_xsum.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-export PYTHONPATH="../":"${PYTHONPATH}"
-
-# From appendix C of paper https://arxiv.org/abs/1912.08777
-# Set --gradient_accumulation_steps  so that effective batch size is 256 (2*128, 4*64, 8*32, 16*16)
-python finetune.py \
-    --learning_rate=1e-4 \
-    --do_train \
-    --do_predict \
-    --n_val 1000 \
-    --val_check_interval 0.25 \
-    --max_source_length 512 --max_target_length 56 \
-    --freeze_embeds --max_target_length 56 --label_smoothing 0.1 \
-    $@

examples/seq2seq/test_seq2seq_examples.py

@@ -15,7 +15,7 @@ from torch.utils.data import DataLoader
 
 import lightning_base
 from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
-from transformers.testing_utils import CaptureStderr, CaptureStdout, require_multigpu
+from transformers.testing_utils import require_multigpu
 
 from .distillation import distill_main, evaluate_checkpoint
 from .finetune import SummarizationModule, main
@@ -166,31 +166,6 @@ class TestSummarizationDistiller(unittest.TestCase):
             # TODO: understand why this breaks
             self.assertEqual(nll_loss, model_computed_loss)
 
-    def test_distill_mbart(self):
-        updates = dict(
-            student_encoder_layers=2,
-            student_decoder_layers=1,
-            num_train_epochs=4,
-            val_check_interval=0.25,
-            alpha_hid=2.0,
-            task="translation",
-            model_name_or_path="IGNORE_THIS_IT_DOESNT_GET_USED",
-            tokenizer_name=MBART_TINY,
-            teacher=MBART_TINY,
-            src_lang="en_XX",
-            tgt_lang="ro_RO",
-        )
-        model = self._test_distiller_cli(updates, check_contents=False)
-
-        ckpts = list(Path(model.output_dir).glob("*.ckpt"))
-        self.assertEqual(1, len(ckpts))
-        transformer_ckpts = list(Path(model.output_dir).glob("**/*.bin"))
-        all_files = list(Path(model.output_dir).glob("best_tfmr/*"))
-        assert len(all_files) > 2
-        self.assertEqual(len(transformer_ckpts), 2)
-
-        evaluate_checkpoint(ckpts[0], dest_dir=Path(tempfile.mkdtemp()))
-
     @unittest.skip("T5 distillation is broken at the moment")
     def test_distill_t5(self):
         updates = dict(
@@ -205,7 +180,7 @@ class TestSummarizationDistiller(unittest.TestCase):
 
     def _test_distiller_cli(self, updates, check_contents=True):
         default_updates = dict(
-            label_smoothing=0.0,
+            label_smoothing_eps=0.0,
             early_stopping_patience=-1,
             train_batch_size=1,
             eval_batch_size=2,
@@ -354,7 +329,7 @@ def test_finetune_extra_model_args():
     assert str(excinfo.value) == f"model config doesn't have a `{unsupported_param}` attribute"
 
 
-def test_finetune_lr_schedulers():
+def test_finetune_lr_shedulers(capsys):
     args_d: dict = CHEAP_ARGS.copy()
 
     task = "summarization"
@@ -386,23 +361,23 @@ def test_finetune_lr_schedulers():
 
     # --help test
     with pytest.raises(SystemExit) as excinfo:
-        with CaptureStdout() as cs:
-            args = parser.parse_args(args)
+        args = parser.parse_args(args)
         assert False, "--help is expected to sys.exit"
     assert excinfo.type == SystemExit
+    captured = capsys.readouterr()
     expected = lightning_base.arg_to_scheduler_metavar
-    assert expected in cs.out, "--help is expected to list the supported schedulers"
+    assert expected in captured.out, "--help is expected to list the supported schedulers"
 
     # --lr_scheduler=non_existing_scheduler test
     unsupported_param = "non_existing_scheduler"
     args = {f"--lr_scheduler={unsupported_param}"}
     with pytest.raises(SystemExit) as excinfo:
-        with CaptureStderr() as cs:
-            args = parser.parse_args(args)
+        args = parser.parse_args(args)
         assert False, "invalid argument is expected to sys.exit"
     assert excinfo.type == SystemExit
+    captured = capsys.readouterr()
     expected = f"invalid choice: '{unsupported_param}'"
-    assert expected in cs.err, f"should have bailed on invalid choice of scheduler {unsupported_param}"
+    assert expected in captured.err, f"should have bailed on invalid choice of scheduler {unsupported_param}"
 
     # --lr_scheduler=existing_scheduler test
     supported_param = "cosine"

examples/seq2seq/utils.py

@@ -145,7 +145,7 @@ class Seq2SeqDataset(Dataset):
 
 
 class TranslationDataset(Seq2SeqDataset):
-    """A dataset that calls prepare_seq2seq_batch."""
+    """A dataset that calls prepare_translation_batch."""
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -167,7 +167,7 @@ class TranslationDataset(Seq2SeqDataset):
         }
 
     def collate_fn(self, batch) -> Dict[str, torch.Tensor]:
-        batch_encoding = self.tokenizer.prepare_seq2seq_batch(
+        batch_encoding = self.tokenizer.prepare_translation_batch(
             [x["src_texts"] for x in batch],
             src_lang=self.src_lang,
             tgt_texts=[x["tgt_texts"] for x in batch],
@@ -271,7 +271,7 @@ def calculate_rouge(output_lns: List[str], reference_lns: List[str], use_stemmer
         aggregator.add_scores(scores)
 
     result = aggregator.aggregate()
-    return {k: v.mid.fmeasure * 100 for k, v in result.items()}
+    return {k: v.mid.fmeasure for k, v in result.items()}
 
 
 def freeze_params(model: nn.Module):

examples/test_examples.py

@@ -17,13 +17,10 @@
 import argparse
 import logging
 import os
-import shutil
 import sys
 import unittest
 from unittest.mock import patch
 
-import torch
-
 
 SRC_DIRS = [
     os.path.join(os.path.dirname(__file__), dirname)
@@ -35,7 +32,6 @@ sys.path.extend(SRC_DIRS)
 if SRC_DIRS is not None:
     import run_generation
     import run_glue
-    import run_pl_glue
     import run_language_modeling
     import run_squad
 
@@ -52,10 +48,6 @@ def get_setup_file():
     return args.f
 
 
-def clean_test_dir(path):
-    shutil.rmtree(path, ignore_errors=True)
-
-
 class ExamplesTests(unittest.TestCase):
     def test_run_glue(self):
         stream_handler = logging.StreamHandler(sys.stdout)
@@ -68,6 +60,7 @@ class ExamplesTests(unittest.TestCase):
             --task_name mrpc
             --do_train
             --do_eval
+            --output_dir ./tests/fixtures/tests_samples/temp_dir
             --per_device_train_batch_size=2
             --per_device_eval_batch_size=1
             --learning_rate=1e-4
@@ -76,54 +69,12 @@ class ExamplesTests(unittest.TestCase):
             --overwrite_output_dir
             --seed=42
             --max_seq_length=128
-            """
-        output_dir = "./tests/fixtures/tests_samples/temp_dir_{}".format(hash(testargs))
-        testargs += "--output_dir " + output_dir
-        testargs = testargs.split()
+            """.split()
         with patch.object(sys, "argv", testargs):
             result = run_glue.main()
             del result["eval_loss"]
             for value in result.values():
                 self.assertGreaterEqual(value, 0.75)
-        clean_test_dir(output_dir)
-
-    def test_run_pl_glue(self):
-        stream_handler = logging.StreamHandler(sys.stdout)
-        logger.addHandler(stream_handler)
-
-        testargs = """
-            run_pl_glue.py
-            --model_name_or_path bert-base-cased
-            --data_dir ./tests/fixtures/tests_samples/MRPC/
-            --task mrpc
-            --do_train
-            --do_predict
-            --train_batch_size=32
-            --learning_rate=1e-4
-            --num_train_epochs=1
-            --seed=42
-            --max_seq_length=128
-            """
-        output_dir = "./tests/fixtures/tests_samples/temp_dir_{}".format(hash(testargs))
-        testargs += "--output_dir " + output_dir
-        testargs = testargs.split()
-
-        if torch.cuda.is_available():
-            testargs += ["--fp16", "--gpus=1"]
-
-        with patch.object(sys, "argv", testargs):
-            result = run_pl_glue.main()
-            # for now just testing that the script can run to a completion
-            self.assertGreater(result["acc"], 0.25)
-            #
-            # TODO: this fails on CI - doesn't get acc/f1>=0.75:
-            #
-            #     # remove all the various *loss* attributes
-            #     result = {k: v for k, v in result.items() if "loss" not in k}
-            #     for k, v in result.items():
-            #         self.assertGreaterEqual(v, 0.75, f"({k})")
-            #
-        clean_test_dir(output_dir)
 
     def test_run_language_modeling(self):
         stream_handler = logging.StreamHandler(sys.stdout)
@@ -137,19 +88,16 @@ class ExamplesTests(unittest.TestCase):
             --line_by_line
             --train_data_file ./tests/fixtures/sample_text.txt
             --eval_data_file ./tests/fixtures/sample_text.txt
+            --output_dir ./tests/fixtures/tests_samples/temp_dir
             --overwrite_output_dir
             --do_train
             --do_eval
             --num_train_epochs=1
             --no_cuda
-            """
-        output_dir = "./tests/fixtures/tests_samples/temp_dir_{}".format(hash(testargs))
-        testargs += "--output_dir " + output_dir
-        testargs = testargs.split()
+            """.split()
         with patch.object(sys, "argv", testargs):
             result = run_language_modeling.main()
             self.assertLess(result["perplexity"], 35)
-        clean_test_dir(output_dir)
 
     def test_run_squad(self):
         stream_handler = logging.StreamHandler(sys.stdout)
@@ -160,6 +108,7 @@ class ExamplesTests(unittest.TestCase):
             --model_type=distilbert
             --model_name_or_path=sshleifer/tiny-distilbert-base-cased-distilled-squad
             --data_dir=./tests/fixtures/tests_samples/SQUAD
+            --output_dir=./tests/fixtures/tests_samples/temp_dir
             --max_steps=10
             --warmup_steps=2
             --do_train
@@ -170,15 +119,11 @@ class ExamplesTests(unittest.TestCase):
             --per_gpu_eval_batch_size=1
             --overwrite_output_dir
             --seed=42
-        """
-        output_dir = "./tests/fixtures/tests_samples/temp_dir_{}".format(hash(testargs))
-        testargs += "--output_dir " + output_dir
-        testargs = testargs.split()
+        """.split()
         with patch.object(sys, "argv", testargs):
             result = run_squad.main()
             self.assertGreaterEqual(result["f1"], 25)
             self.assertGreaterEqual(result["exact"], 21)
-        clean_test_dir(output_dir)
 
     def test_generation(self):
         stream_handler = logging.StreamHandler(sys.stdout)

examples/text-classification/run_pl.sh

@@ -1,3 +1,5 @@
+# Install newest ptl.
+pip install -U git+http://github.com/PyTorchLightning/pytorch-lightning/
 # Install example requirements
 pip install -r ../requirements.txt
 

examples/text-classification/run_pl_glue.py

@@ -44,8 +44,8 @@ class GLUETransformer(BaseTransformer):
         outputs = self(**inputs)
         loss = outputs[0]
 
-        lr_scheduler = self.trainer.lr_schedulers[0]["scheduler"]
-        tensorboard_logs = {"loss": loss, "rate": lr_scheduler.get_last_lr()[-1]}
+        # tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
+        tensorboard_logs = {"loss": loss}
         return {"loss": loss, "log": tensorboard_logs}
 
     def prepare_data(self):
@@ -176,7 +176,7 @@ class GLUETransformer(BaseTransformer):
         return parser
 
 
-def main():
+if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     add_generic_args(parser, os.getcwd())
     parser = GLUETransformer.add_model_specific_args(parser, os.getcwd())
@@ -194,8 +194,4 @@ def main():
     if args.do_predict:
         checkpoints = list(sorted(glob.glob(os.path.join(args.output_dir, "checkpointepoch=*.ckpt"), recursive=True)))
         model = model.load_from_checkpoint(checkpoints[-1])
-        return trainer.test(model)
-
-
-if __name__ == "__main__":
-    main()
+        trainer.test(model)

examples/token-classification/run.sh

@@ -18,7 +18,6 @@ export SAVE_STEPS=750
 export SEED=1
 
 python3 run_ner.py \
---task_type NER \
 --data_dir . \
 --labels ./labels.txt \
 --model_name_or_path $BERT_MODEL \

examples/token-classification/run_chunk.sh

@@ -1,37 +0,0 @@
-if ! [ -f ./dev.txt ]; then
-  echo "Downloading CONLL2003 dev dataset...."
-  curl -L -o ./dev.txt 'https://github.com/davidsbatista/NER-datasets/raw/master/CONLL2003/valid.txt'
-fi
-
-if ! [ -f ./test.txt ]; then
-  echo "Downloading CONLL2003 test dataset...."
-  curl -L -o ./test.txt 'https://github.com/davidsbatista/NER-datasets/raw/master/CONLL2003/test.txt'
-fi
-
-if ! [ -f ./train.txt ]; then
-  echo "Downloading CONLL2003 train dataset...."
-  curl -L -o ./train.txt 'https://github.com/davidsbatista/NER-datasets/raw/master/CONLL2003/train.txt'
-fi
-
-export MAX_LENGTH=200
-export BERT_MODEL=bert-base-uncased
-export OUTPUT_DIR=chunker-model
-export BATCH_SIZE=32
-export NUM_EPOCHS=3
-export SAVE_STEPS=750
-export SEED=1
-
-python3 run_ner.py \
---task_type Chunk \
---data_dir . \
---model_name_or_path $BERT_MODEL \
---output_dir $OUTPUT_DIR \
---max_seq_length  $MAX_LENGTH \
---num_train_epochs $NUM_EPOCHS \
---per_gpu_train_batch_size $BATCH_SIZE \
---save_steps $SAVE_STEPS \
---seed $SEED \
---do_train \
---do_eval \
---do_predict
-

examples/token-classification/run_ner.py

@@ -14,15 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Fine-tuning the library models for named entity recognition on CoNLL-2003. """
+
+
 import logging
 import os
 import sys
 from dataclasses import dataclass, field
-from importlib import import_module
 from typing import Dict, List, Optional, Tuple
 
 import numpy as np
-from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
+from seqeval.metrics import f1_score, precision_score, recall_score
 from torch import nn
 
 from transformers import (
@@ -35,7 +36,7 @@ from transformers import (
     TrainingArguments,
     set_seed,
 )
-from utils_ner import Split, TokenClassificationDataset, TokenClassificationTask
+from utils_ner import NerDataset, Split, get_labels
 
 
 logger = logging.getLogger(__name__)
@@ -53,9 +54,6 @@ class ModelArguments:
     config_name: Optional[str] = field(
         default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
     )
-    task_type: Optional[str] = field(
-        default="NER", metadata={"help": "Task type to fine tune in training (e.g. NER, POS, etc)"}
-    )
     tokenizer_name: Optional[str] = field(
         default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
     )
@@ -115,16 +113,6 @@ def main():
             f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
         )
 
-    module = import_module("tasks")
-    try:
-        token_classification_task_clazz = getattr(module, model_args.task_type)
-        token_classification_task: TokenClassificationTask = token_classification_task_clazz()
-    except AttributeError:
-        raise ValueError(
-            f"Task {model_args.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. "
-            f"Available tasks classes are: {TokenClassificationTask.__subclasses__()}"
-        )
-
     # Setup logging
     logging.basicConfig(
         format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
@@ -145,7 +133,7 @@ def main():
     set_seed(training_args.seed)
 
     # Prepare CONLL-2003 task
-    labels = token_classification_task.get_labels(data_args.labels)
+    labels = get_labels(data_args.labels)
     label_map: Dict[int, str] = {i: label for i, label in enumerate(labels)}
     num_labels = len(labels)
 
@@ -176,8 +164,7 @@ def main():
 
     # Get datasets
     train_dataset = (
-        TokenClassificationDataset(
-            token_classification_task=token_classification_task,
+        NerDataset(
             data_dir=data_args.data_dir,
             tokenizer=tokenizer,
             labels=labels,
@@ -190,8 +177,7 @@ def main():
         else None
     )
     eval_dataset = (
-        TokenClassificationDataset(
-            token_classification_task=token_classification_task,
+        NerDataset(
             data_dir=data_args.data_dir,
             tokenizer=tokenizer,
             labels=labels,
@@ -223,7 +209,6 @@ def main():
     def compute_metrics(p: EvalPrediction) -> Dict:
         preds_list, out_label_list = align_predictions(p.predictions, p.label_ids)
         return {
-            "accuracy_score": accuracy_score(out_label_list, preds_list),
             "precision": precision_score(out_label_list, preds_list),
             "recall": recall_score(out_label_list, preds_list),
             "f1": f1_score(out_label_list, preds_list),
@@ -268,8 +253,7 @@ def main():
 
     # Predict
     if training_args.do_predict:
-        test_dataset = TokenClassificationDataset(
-            token_classification_task=token_classification_task,
+        test_dataset = NerDataset(
             data_dir=data_args.data_dir,
             tokenizer=tokenizer,
             labels=labels,
@@ -294,7 +278,19 @@ def main():
         if trainer.is_world_master():
             with open(output_test_predictions_file, "w") as writer:
                 with open(os.path.join(data_args.data_dir, "test.txt"), "r") as f:
-                    token_classification_task.write_predictions_to_file(writer, f, preds_list)
+                    example_id = 0
+                    for line in f:
+                        if line.startswith("-DOCSTART-") or line == "" or line == "\n":
+                            writer.write(line)
+                            if not preds_list[example_id]:
+                                example_id += 1
+                        elif preds_list[example_id]:
+                            output_line = line.split()[0] + " " + preds_list[example_id].pop(0) + "\n"
+                            writer.write(output_line)
+                        else:
+                            logger.warning(
+                                "Maximum sequence length exceeded: No prediction for '%s'.", line.split()[0]
+                            )
 
     return results
 

examples/token-classification/run_pl.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# Install newest ptl.
+pip install -U git+http://github.com/PyTorchLightning/pytorch-lightning/
 # for seqeval metrics import
 pip install -r ../requirements.txt
 
@@ -38,4 +40,4 @@ python3 run_pl_ner.py --data_dir ./ \
 --train_batch_size $BATCH_SIZE \
 --seed $SEED \
 --do_train \
---do_predict
+--do_predict

examples/token-classification/run_pl_ner.py

@@ -2,17 +2,15 @@ import argparse
 import glob
 import logging
 import os
-from argparse import Namespace
-from importlib import import_module
 
 import numpy as np
 import torch
-from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
+from seqeval.metrics import f1_score, precision_score, recall_score
 from torch.nn import CrossEntropyLoss
 from torch.utils.data import DataLoader, TensorDataset
 
 from lightning_base import BaseTransformer, add_generic_args, generic_train
-from utils_ner import TokenClassificationTask
+from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file
 
 
 logger = logging.getLogger(__name__)
@@ -26,20 +24,10 @@ class NERTransformer(BaseTransformer):
     mode = "token-classification"
 
     def __init__(self, hparams):
-        if type(hparams) == dict:
-            hparams = Namespace(**hparams)
-        module = import_module("tasks")
-        try:
-            token_classification_task_clazz = getattr(module, hparams.task_type)
-            self.token_classification_task: TokenClassificationTask = token_classification_task_clazz()
-        except AttributeError:
-            raise ValueError(
-                f"Task {hparams.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. "
-                f"Available tasks classes are: {TokenClassificationTask.__subclasses__()}"
-            )
-        self.labels = self.token_classification_task.get_labels(hparams.labels)
+        self.labels = get_labels(hparams.labels)
+        num_labels = len(self.labels)
         self.pad_token_label_id = CrossEntropyLoss().ignore_index
-        super().__init__(hparams, len(self.labels), self.mode)
+        super().__init__(hparams, num_labels, self.mode)
 
     def forward(self, **inputs):
         return self.model(**inputs)
@@ -54,8 +42,8 @@ class NERTransformer(BaseTransformer):
 
         outputs = self(**inputs)
         loss = outputs[0]
-        # tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
-        return {"loss": loss}
+        tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
+        return {"loss": loss, "log": tensorboard_logs}
 
     def prepare_data(self):
         "Called to initialize data. Use the call to construct features"
@@ -67,8 +55,8 @@ class NERTransformer(BaseTransformer):
                 features = torch.load(cached_features_file)
             else:
                 logger.info("Creating features from dataset file at %s", args.data_dir)
-                examples = self.token_classification_task.read_examples_from_file(args.data_dir, mode)
-                features = self.token_classification_task.convert_examples_to_features(
+                examples = read_examples_from_file(args.data_dir, mode)
+                features = convert_examples_to_features(
                     examples,
                     self.labels,
                     args.max_seq_length,
@@ -86,7 +74,7 @@ class NERTransformer(BaseTransformer):
                 logger.info("Saving features into cached file %s", cached_features_file)
                 torch.save(features, cached_features_file)
 
-    def get_dataloader(self, mode: int, batch_size: int) -> DataLoader:
+    def load_dataset(self, mode, batch_size):
         "Load datasets. Called after prepare data."
         cached_features_file = self._feature_file(mode)
         logger.info("Loading features from cached file %s", cached_features_file)
@@ -136,7 +124,6 @@ class NERTransformer(BaseTransformer):
 
         results = {
             "val_loss": val_loss_mean,
-            "accuracy_score": accuracy_score(out_label_list, preds_list),
             "precision": precision_score(out_label_list, preds_list),
             "recall": recall_score(out_label_list, preds_list),
             "f1": f1_score(out_label_list, preds_list),
@@ -167,9 +154,6 @@ class NERTransformer(BaseTransformer):
     def add_model_specific_args(parser, root_dir):
         # Add NER specific options
         BaseTransformer.add_model_specific_args(parser, root_dir)
-        parser.add_argument(
-            "--task_type", default="NER", type=str, help="Task type to fine tune in training (e.g. NER, POS, etc)"
-        )
         parser.add_argument(
             "--max_seq_length",
             default=128,

examples/token-classification/run_pos.sh

@@ -1,37 +0,0 @@
-if ! [ -f ./dev.txt ]; then
-  echo "Download dev dataset...."
-  curl -L -o ./dev.txt 'https://github.com/UniversalDependencies/UD_English-EWT/raw/master/en_ewt-ud-dev.conllu'
-fi
-
-if ! [ -f ./test.txt ]; then
-  echo "Download test dataset...."
-  curl -L -o ./test.txt 'https://github.com/UniversalDependencies/UD_English-EWT/raw/master/en_ewt-ud-test.conllu'
-fi
-
-if ! [ -f ./train.txt ]; then
-  echo "Download train dataset...."
-  curl -L -o ./train.txt 'https://github.com/UniversalDependencies/UD_English-EWT/raw/master/en_ewt-ud-train.conllu'
-fi
-
-export MAX_LENGTH=200
-export BERT_MODEL=bert-base-uncased
-export OUTPUT_DIR=postagger-model
-export BATCH_SIZE=32
-export NUM_EPOCHS=3
-export SAVE_STEPS=750
-export SEED=1
-
-python3 run_ner.py \
---task_type POS \
---data_dir . \
---model_name_or_path $BERT_MODEL \
---output_dir $OUTPUT_DIR \
---max_seq_length  $MAX_LENGTH \
---num_train_epochs $NUM_EPOCHS \
---per_gpu_train_batch_size $BATCH_SIZE \
---save_steps $SAVE_STEPS \
---seed $SEED \
---do_train \
---do_eval \
---do_predict
-

examples/token-classification/run_pos_pl.sh

@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-if ! [ -f ./dev.txt ]; then
-  echo "Download dev dataset...."
-  curl -L -o ./dev.txt 'https://github.com/UniversalDependencies/UD_English-EWT/raw/master/en_ewt-ud-dev.conllu'
-fi
-
-if ! [ -f ./test.txt ]; then
-  echo "Download test dataset...."
-  curl -L -o ./test.txt 'https://github.com/UniversalDependencies/UD_English-EWT/raw/master/en_ewt-ud-test.conllu'
-fi
-
-if ! [ -f ./train.txt ]; then
-  echo "Download train dataset...."
-  curl -L -o ./train.txt 'https://github.com/UniversalDependencies/UD_English-EWT/raw/master/en_ewt-ud-train.conllu'
-fi
-
-export MAX_LENGTH=200
-export BERT_MODEL=bert-base-uncased
-export OUTPUT_DIR=postagger-model
-export BATCH_SIZE=32
-export NUM_EPOCHS=3
-export SAVE_STEPS=750
-export SEED=1
-
-
-# Add parent directory to python path to access lightning_base.py
-export PYTHONPATH="../":"${PYTHONPATH}"
-
-python3 run_pl_ner.py --data_dir ./ \
---task_type POS \
---model_name_or_path $BERT_MODEL \
---output_dir $OUTPUT_DIR \
---max_seq_length  $MAX_LENGTH \
---num_train_epochs $NUM_EPOCHS \
---train_batch_size $BATCH_SIZE \
---seed $SEED \
---gpus 1 \
---do_train \
---do_predict

examples/token-classification/tasks.py

@@ -1,163 +0,0 @@
-import logging
-import os
-from typing import List, TextIO, Union
-
-from conllu import parse_incr
-
-from utils_ner import InputExample, Split, TokenClassificationTask
-
-
-logger = logging.getLogger(__name__)
-
-
-class NER(TokenClassificationTask):
-    def __init__(self, label_idx=-1):
-        # in NER datasets, the last column is usually reserved for NER label
-        self.label_idx = label_idx
-
-    def read_examples_from_file(self, data_dir, mode: Union[Split, str]) -> List[InputExample]:
-        if isinstance(mode, Split):
-            mode = mode.value
-        file_path = os.path.join(data_dir, f"{mode}.txt")
-        guid_index = 1
-        examples = []
-        with open(file_path, encoding="utf-8") as f:
-            words = []
-            labels = []
-            for line in f:
-                if line.startswith("-DOCSTART-") or line == "" or line == "\n":
-                    if words:
-                        examples.append(InputExample(guid=f"{mode}-{guid_index}", words=words, labels=labels))
-                        guid_index += 1
-                        words = []
-                        labels = []
-                else:
-                    splits = line.split(" ")
-                    words.append(splits[0])
-                    if len(splits) > 1:
-                        labels.append(splits[self.label_idx].replace("\n", ""))
-                    else:
-                        # Examples could have no label for mode = "test"
-                        labels.append("O")
-            if words:
-                examples.append(InputExample(guid=f"{mode}-{guid_index}", words=words, labels=labels))
-        return examples
-
-    def write_predictions_to_file(self, writer: TextIO, test_input_reader: TextIO, preds_list: List):
-        example_id = 0
-        for line in test_input_reader:
-            if line.startswith("-DOCSTART-") or line == "" or line == "\n":
-                writer.write(line)
-                if not preds_list[example_id]:
-                    example_id += 1
-            elif preds_list[example_id]:
-                output_line = line.split()[0] + " " + preds_list[example_id].pop(0) + "\n"
-                writer.write(output_line)
-            else:
-                logger.warning("Maximum sequence length exceeded: No prediction for '%s'.", line.split()[0])
-
-    def get_labels(self, path: str) -> List[str]:
-        if path:
-            with open(path, "r") as f:
-                labels = f.read().splitlines()
-            if "O" not in labels:
-                labels = ["O"] + labels
-            return labels
-        else:
-            return ["O", "B-MISC", "I-MISC", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC"]
-
-
-class Chunk(NER):
-    def __init__(self):
-        # in CONLL2003 dataset chunk column is second-to-last
-        super().__init__(label_idx=-2)
-
-    def get_labels(self, path: str) -> List[str]:
-        if path:
-            with open(path, "r") as f:
-                labels = f.read().splitlines()
-            if "O" not in labels:
-                labels = ["O"] + labels
-            return labels
-        else:
-            return [
-                "O",
-                "B-ADVP",
-                "B-INTJ",
-                "B-LST",
-                "B-PRT",
-                "B-NP",
-                "B-SBAR",
-                "B-VP",
-                "B-ADJP",
-                "B-CONJP",
-                "B-PP",
-                "I-ADVP",
-                "I-INTJ",
-                "I-LST",
-                "I-PRT",
-                "I-NP",
-                "I-SBAR",
-                "I-VP",
-                "I-ADJP",
-                "I-CONJP",
-                "I-PP",
-            ]
-
-
-class POS(TokenClassificationTask):
-    def read_examples_from_file(self, data_dir, mode: Union[Split, str]) -> List[InputExample]:
-        if isinstance(mode, Split):
-            mode = mode.value
-        file_path = os.path.join(data_dir, f"{mode}.txt")
-        guid_index = 1
-        examples = []
-
-        with open(file_path, encoding="utf-8") as f:
-            for sentence in parse_incr(f):
-                words = []
-                labels = []
-                for token in sentence:
-                    words.append(token["form"])
-                    labels.append(token["upos"])
-                assert len(words) == len(labels)
-                if words:
-                    examples.append(InputExample(guid=f"{mode}-{guid_index}", words=words, labels=labels))
-                    guid_index += 1
-        return examples
-
-    def write_predictions_to_file(self, writer: TextIO, test_input_reader: TextIO, preds_list: List):
-        example_id = 0
-        for sentence in parse_incr(test_input_reader):
-            s_p = preds_list[example_id]
-            out = ""
-            for token in sentence:
-                out += f'{token["form"]} ({token["upos"]}|{s_p.pop(0)}) '
-            out += "\n"
-            writer.write(out)
-            example_id += 1
-
-    def get_labels(self, path: str) -> List[str]:
-        if path:
-            with open(path, "r") as f:
-                return f.read().splitlines()
-        else:
-            return [
-                "ADJ",
-                "ADP",
-                "ADV",
-                "AUX",
-                "CCONJ",
-                "DET",
-                "INTJ",
-                "NOUN",
-                "NUM",
-                "PART",
-                "PRON",
-                "PROPN",
-                "PUNCT",
-                "SCONJ",
-                "SYM",
-                "VERB",
-                "X",
-            ]

examples/token-classification/utils_ner.py

@@ -66,148 +66,12 @@ class Split(Enum):
     test = "test"
 
 
-class TokenClassificationTask:
-    def read_examples_from_file(self, data_dir, mode: Union[Split, str]) -> List[InputExample]:
-        raise NotImplementedError
-
-    def get_labels(self, path: str) -> List[str]:
-        raise NotImplementedError
-
-    def convert_examples_to_features(
-        self,
-        examples: List[InputExample],
-        label_list: List[str],
-        max_seq_length: int,
-        tokenizer: PreTrainedTokenizer,
-        cls_token_at_end=False,
-        cls_token="[CLS]",
-        cls_token_segment_id=1,
-        sep_token="[SEP]",
-        sep_token_extra=False,
-        pad_on_left=False,
-        pad_token=0,
-        pad_token_segment_id=0,
-        pad_token_label_id=-100,
-        sequence_a_segment_id=0,
-        mask_padding_with_zero=True,
-    ) -> List[InputFeatures]:
-        """ Loads a data file into a list of `InputFeatures`
-            `cls_token_at_end` define the location of the CLS token:
-                - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
-                - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
-            `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)
-        """
-        # TODO clean up all this to leverage built-in features of tokenizers
-
-        label_map = {label: i for i, label in enumerate(label_list)}
-
-        features = []
-        for (ex_index, example) in enumerate(examples):
-            if ex_index % 10_000 == 0:
-                logger.info("Writing example %d of %d", ex_index, len(examples))
-
-            tokens = []
-            label_ids = []
-            for word, label in zip(example.words, example.labels):
-                word_tokens = tokenizer.tokenize(word)
-
-                # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
-                if len(word_tokens) > 0:
-                    tokens.extend(word_tokens)
-                    # Use the real label id for the first token of the word, and padding ids for the remaining tokens
-                    label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(word_tokens) - 1))
-
-            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
-            special_tokens_count = tokenizer.num_special_tokens_to_add()
-            if len(tokens) > max_seq_length - special_tokens_count:
-                tokens = tokens[: (max_seq_length - special_tokens_count)]
-                label_ids = label_ids[: (max_seq_length - special_tokens_count)]
-
-            # The convention in BERT is:
-            # (a) For sequence pairs:
-            #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
-            #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
-            # (b) For single sequences:
-            #  tokens:   [CLS] the dog is hairy . [SEP]
-            #  type_ids:   0   0   0   0  0     0   0
-            #
-            # Where "type_ids" are used to indicate whether this is the first
-            # sequence or the second sequence. The embedding vectors for `type=0` and
-            # `type=1` were learned during pre-training and are added to the wordpiece
-            # embedding vector (and position vector). This is not *strictly* necessary
-            # since the [SEP] token unambiguously separates the sequences, but it makes
-            # it easier for the model to learn the concept of sequences.
-            #
-            # For classification tasks, the first vector (corresponding to [CLS]) is
-            # used as as the "sentence vector". Note that this only makes sense because
-            # the entire model is fine-tuned.
-            tokens += [sep_token]
-            label_ids += [pad_token_label_id]
-            if sep_token_extra:
-                # roberta uses an extra separator b/w pairs of sentences
-                tokens += [sep_token]
-                label_ids += [pad_token_label_id]
-            segment_ids = [sequence_a_segment_id] * len(tokens)
-
-            if cls_token_at_end:
-                tokens += [cls_token]
-                label_ids += [pad_token_label_id]
-                segment_ids += [cls_token_segment_id]
-            else:
-                tokens = [cls_token] + tokens
-                label_ids = [pad_token_label_id] + label_ids
-                segment_ids = [cls_token_segment_id] + segment_ids
-
-            input_ids = tokenizer.convert_tokens_to_ids(tokens)
-
-            # The mask has 1 for real tokens and 0 for padding tokens. Only real
-            # tokens are attended to.
-            input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
-
-            # Zero-pad up to the sequence length.
-            padding_length = max_seq_length - len(input_ids)
-            if pad_on_left:
-                input_ids = ([pad_token] * padding_length) + input_ids
-                input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
-                segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids
-                label_ids = ([pad_token_label_id] * padding_length) + label_ids
-            else:
-                input_ids += [pad_token] * padding_length
-                input_mask += [0 if mask_padding_with_zero else 1] * padding_length
-                segment_ids += [pad_token_segment_id] * padding_length
-                label_ids += [pad_token_label_id] * padding_length
-
-            assert len(input_ids) == max_seq_length
-            assert len(input_mask) == max_seq_length
-            assert len(segment_ids) == max_seq_length
-            assert len(label_ids) == max_seq_length
-
-            if ex_index < 5:
-                logger.info("*** Example ***")
-                logger.info("guid: %s", example.guid)
-                logger.info("tokens: %s", " ".join([str(x) for x in tokens]))
-                logger.info("input_ids: %s", " ".join([str(x) for x in input_ids]))
-                logger.info("input_mask: %s", " ".join([str(x) for x in input_mask]))
-                logger.info("segment_ids: %s", " ".join([str(x) for x in segment_ids]))
-                logger.info("label_ids: %s", " ".join([str(x) for x in label_ids]))
-
-            if "token_type_ids" not in tokenizer.model_input_names:
-                segment_ids = None
-
-            features.append(
-                InputFeatures(
-                    input_ids=input_ids, attention_mask=input_mask, token_type_ids=segment_ids, label_ids=label_ids
-                )
-            )
-        return features
-
-
 if is_torch_available():
     import torch
     from torch import nn
     from torch.utils.data.dataset import Dataset
 
-    class TokenClassificationDataset(Dataset):
+    class NerDataset(Dataset):
         """
         This will be superseded by a framework-agnostic approach
         soon.
@@ -220,7 +84,6 @@ if is_torch_available():
 
         def __init__(
             self,
-            token_classification_task: TokenClassificationTask,
             data_dir: str,
             tokenizer: PreTrainedTokenizer,
             labels: List[str],
@@ -244,9 +107,9 @@ if is_torch_available():
                     self.features = torch.load(cached_features_file)
                 else:
                     logger.info(f"Creating features from dataset file at {data_dir}")
-                    examples = token_classification_task.read_examples_from_file(data_dir, mode)
+                    examples = read_examples_from_file(data_dir, mode)
                     # TODO clean up all this to leverage built-in features of tokenizers
-                    self.features = token_classification_task.convert_examples_to_features(
+                    self.features = convert_examples_to_features(
                         examples,
                         labels,
                         max_seq_length,
@@ -289,7 +152,6 @@ if is_tf_available():
 
         def __init__(
             self,
-            token_classification_task: TokenClassificationTask,
             data_dir: str,
             tokenizer: PreTrainedTokenizer,
             labels: List[str],
@@ -298,9 +160,9 @@ if is_tf_available():
             overwrite_cache=False,
             mode: Split = Split.train,
         ):
-            examples = token_classification_task.read_examples_from_file(data_dir, mode)
+            examples = read_examples_from_file(data_dir, mode)
             # TODO clean up all this to leverage built-in features of tokenizers
-            self.features = token_classification_task.convert_examples_to_features(
+            self.features = convert_examples_to_features(
                 examples,
                 labels,
                 max_seq_length,
@@ -368,3 +230,171 @@ if is_tf_available():
 
         def __getitem__(self, i) -> InputFeatures:
             return self.features[i]
+
+
+def read_examples_from_file(data_dir, mode: Union[Split, str]) -> List[InputExample]:
+    if isinstance(mode, Split):
+        mode = mode.value
+    file_path = os.path.join(data_dir, f"{mode}.txt")
+    guid_index = 1
+    examples = []
+    with open(file_path, encoding="utf-8") as f:
+        words = []
+        labels = []
+        for line in f:
+            if line.startswith("-DOCSTART-") or line == "" or line == "\n":
+                if words:
+                    examples.append(InputExample(guid=f"{mode}-{guid_index}", words=words, labels=labels))
+                    guid_index += 1
+                    words = []
+                    labels = []
+            else:
+                splits = line.split(" ")
+                words.append(splits[0])
+                if len(splits) > 1:
+                    labels.append(splits[-1].replace("\n", ""))
+                else:
+                    # Examples could have no label for mode = "test"
+                    labels.append("O")
+        if words:
+            examples.append(InputExample(guid=f"{mode}-{guid_index}", words=words, labels=labels))
+    return examples
+
+
+def convert_examples_to_features(
+    examples: List[InputExample],
+    label_list: List[str],
+    max_seq_length: int,
+    tokenizer: PreTrainedTokenizer,
+    cls_token_at_end=False,
+    cls_token="[CLS]",
+    cls_token_segment_id=1,
+    sep_token="[SEP]",
+    sep_token_extra=False,
+    pad_on_left=False,
+    pad_token=0,
+    pad_token_segment_id=0,
+    pad_token_label_id=-100,
+    sequence_a_segment_id=0,
+    mask_padding_with_zero=True,
+) -> List[InputFeatures]:
+    """ Loads a data file into a list of `InputFeatures`
+        `cls_token_at_end` define the location of the CLS token:
+            - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
+            - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
+        `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)
+    """
+    # TODO clean up all this to leverage built-in features of tokenizers
+
+    label_map = {label: i for i, label in enumerate(label_list)}
+
+    features = []
+    for (ex_index, example) in enumerate(examples):
+        if ex_index % 10_000 == 0:
+            logger.info("Writing example %d of %d", ex_index, len(examples))
+
+        tokens = []
+        label_ids = []
+        for word, label in zip(example.words, example.labels):
+            word_tokens = tokenizer.tokenize(word)
+
+            # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
+            if len(word_tokens) > 0:
+                tokens.extend(word_tokens)
+                # Use the real label id for the first token of the word, and padding ids for the remaining tokens
+                label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(word_tokens) - 1))
+
+        # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
+        special_tokens_count = tokenizer.num_special_tokens_to_add()
+        if len(tokens) > max_seq_length - special_tokens_count:
+            tokens = tokens[: (max_seq_length - special_tokens_count)]
+            label_ids = label_ids[: (max_seq_length - special_tokens_count)]
+
+        # The convention in BERT is:
+        # (a) For sequence pairs:
+        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
+        #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
+        # (b) For single sequences:
+        #  tokens:   [CLS] the dog is hairy . [SEP]
+        #  type_ids:   0   0   0   0  0     0   0
+        #
+        # Where "type_ids" are used to indicate whether this is the first
+        # sequence or the second sequence. The embedding vectors for `type=0` and
+        # `type=1` were learned during pre-training and are added to the wordpiece
+        # embedding vector (and position vector). This is not *strictly* necessary
+        # since the [SEP] token unambiguously separates the sequences, but it makes
+        # it easier for the model to learn the concept of sequences.
+        #
+        # For classification tasks, the first vector (corresponding to [CLS]) is
+        # used as as the "sentence vector". Note that this only makes sense because
+        # the entire model is fine-tuned.
+        tokens += [sep_token]
+        label_ids += [pad_token_label_id]
+        if sep_token_extra:
+            # roberta uses an extra separator b/w pairs of sentences
+            tokens += [sep_token]
+            label_ids += [pad_token_label_id]
+        segment_ids = [sequence_a_segment_id] * len(tokens)
+
+        if cls_token_at_end:
+            tokens += [cls_token]
+            label_ids += [pad_token_label_id]
+            segment_ids += [cls_token_segment_id]
+        else:
+            tokens = [cls_token] + tokens
+            label_ids = [pad_token_label_id] + label_ids
+            segment_ids = [cls_token_segment_id] + segment_ids
+
+        input_ids = tokenizer.convert_tokens_to_ids(tokens)
+
+        # The mask has 1 for real tokens and 0 for padding tokens. Only real
+        # tokens are attended to.
+        input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)
+
+        # Zero-pad up to the sequence length.
+        padding_length = max_seq_length - len(input_ids)
+        if pad_on_left:
+            input_ids = ([pad_token] * padding_length) + input_ids
+            input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
+            segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids
+            label_ids = ([pad_token_label_id] * padding_length) + label_ids
+        else:
+            input_ids += [pad_token] * padding_length
+            input_mask += [0 if mask_padding_with_zero else 1] * padding_length
+            segment_ids += [pad_token_segment_id] * padding_length
+            label_ids += [pad_token_label_id] * padding_length
+
+        assert len(input_ids) == max_seq_length
+        assert len(input_mask) == max_seq_length
+        assert len(segment_ids) == max_seq_length
+        assert len(label_ids) == max_seq_length
+
+        if ex_index < 5:
+            logger.info("*** Example ***")
+            logger.info("guid: %s", example.guid)
+            logger.info("tokens: %s", " ".join([str(x) for x in tokens]))
+            logger.info("input_ids: %s", " ".join([str(x) for x in input_ids]))
+            logger.info("input_mask: %s", " ".join([str(x) for x in input_mask]))
+            logger.info("segment_ids: %s", " ".join([str(x) for x in segment_ids]))
+            logger.info("label_ids: %s", " ".join([str(x) for x in label_ids]))
+
+        if "token_type_ids" not in tokenizer.model_input_names:
+            segment_ids = None
+
+        features.append(
+            InputFeatures(
+                input_ids=input_ids, attention_mask=input_mask, token_type_ids=segment_ids, label_ids=label_ids
+            )
+        )
+    return features
+
+
+def get_labels(path: str) -> List[str]:
+    if path:
+        with open(path, "r") as f:
+            labels = f.read().splitlines()
+        if "O" not in labels:
+            labels = ["O"] + labels
+        return labels
+    else:
+        return ["O", "B-MISC", "I-MISC", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC"]

model_cards/akhooli/gpt2-small-arabic-poetry/README.md

@@ -1,45 +0,0 @@
----
-language: "ar"
-tags:
-- text-generation
-license: ""
-datasets:
-- Arabic poetry from several eras
----
-
-# GPT2-Small-Arabic-Poetry
-
-## Model description
-
-Fine-tuned model of Arabic poetry dataset based on gpt2-small-arabic.
-
-## Intended uses & limitations
-
-#### How to use
-
-An example is provided in this [colab notebook](https://colab.research.google.com/drive/1mRl7c-5v-Klx27EEAEOAbrfkustL4g7a?usp=sharing).
-
-#### Limitations and bias
-
-Both the GPT2-small-arabic (trained on Arabic Wikipedia) and this model have several limitations in terms of coverage and training performance. 
-Use them as demonstrations or proof of concepts but not as production code.
-
-## Training data
-
-This pretrained model used the [Arabic Poetry dataset](https://www.kaggle.com/ahmedabelal/arabic-poetry) from 9 different eras with a total of around 40k poems. 
-The dataset was trained (fine-tuned) based on the [gpt2-small-arabic](https://huggingface.co/akhooli/gpt2-small-arabic) transformer model.
-
-## Training procedure
-
-Training was done using [Simple Transformers](https://github.com/ThilinaRajapakse/simpletransformers) library on Kaggle, using free GPU.
-
-## Eval results 
-Final perplexity reached ws 76.3, loss: 4.33
-
-### BibTeX entry and citation info
-
-```bibtex
-@inproceedings{Abed Khooli,
-  year={2020}
-}
-```

model_cards/akhooli/gpt2-small-arabic/README.md

@@ -1,44 +0,0 @@
----
-language: "ar"
-datasets:
-- Arabic Wikipedia
-metrics:
-- none
----
-
-# GPT2-Small-Arabic
-
-## Model description
-
-GPT2 model from Arabic Wikipedia dataset based on gpt2-small (using Fastai2).
-
-## Intended uses & limitations
-
-#### How to use
-
-An example is provided in this [colab notebook](https://colab.research.google.com/drive/1mRl7c-5v-Klx27EEAEOAbrfkustL4g7a?usp=sharing). 
-Both text and poetry (fine-tuned model) generation are included.
-
-#### Limitations and bias
-
-GPT2-small-arabic (trained on Arabic Wikipedia) has several limitations in terms of coverage (Arabic Wikipeedia quality, no diacritics) and training performance. 
-Use as demonstration or proof of concepts but not as production code.
-
-## Training data
-
-This pretrained model used the Arabic Wikipedia dump (around 900 MB). 
-
-## Training procedure
-
-Training was done using [Fastai2](https://github.com/fastai/fastai2/) library on Kaggle, using free GPU.
-
-## Eval results 
-Final perplexity reached was 72.19,  loss: 4.28, accuracy: 0.307
-
-### BibTeX entry and citation info
-
-```bibtex
-@inproceedings{Abed Khooli,
-  year={2020}
-}
-```

model_cards/cedpsam/chatbot_fr/README.md

@@ -1,46 +0,0 @@
----
-language: fr
-tags:
-- conversational
-widget:
-- text: "bonjour."
-widget:
-- text: "mais encore"
-widget:
-- text: "est ce que l'argent achete le bonheur?"
----
-
-## a dialoggpt model trained on french opensubtitles with custom tokenizer
-trained with this notebook
-https://colab.research.google.com/drive/1pfCV3bngAmISNZVfDvBMyEhQKuYw37Rl#scrollTo=AyImj9qZYLRi&uniqifier=3
-
-config from microsoft/DialoGPT-medium
-### How to use
-
-Now we are ready to try out how the model works as a chatting partner!
-
-```python
-import torch
-from transformers import AutoTokenizer, AutoModelWithLMHead
-
-tokenizer = AutoTokenizer.from_pretrained("cedpsam/chatbot_fr")
-
-model = AutoModelWithLMHead.from_pretrained("cedpsam/chatbot_fr")
-
-for step in range(6):
-    # encode the new user input, add the eos_token and return a tensor in Pytorch
-    new_user_input_ids = tokenizer.encode(input(">> User:") + tokenizer.eos_token, return_tensors='pt')
-    # print(new_user_input_ids)
-
-    # append the new user input tokens to the chat history
-    bot_input_ids = torch.cat([chat_history_ids, new_user_input_ids], dim=-1) if step > 0 else new_user_input_ids
-
-    # generated a response while limiting the total chat history to 1000 tokens, 
-    chat_history_ids = model.generate(
-        bot_input_ids, max_length=1000,
-        pad_token_id=tokenizer.eos_token_id,
-        top_p=0.92, top_k = 50
-    )
-    
-    # pretty print last ouput tokens from bot
-    print("DialoGPT: {}".format(tokenizer.decode(chat_history_ids[:, bot_input_ids.shape[-1]:][0], skip_special_tokens=True)))

model_cards/monsoon-nlp/hindi-bert/README.md

@@ -0,0 +1,68 @@
+---
+language: hi
+---
+
+# Hindi-BERT (Discriminator)
+
+This is a first run of a Hindi language model trained with Google Research's [ELECTRA](https://github.com/google-research/electra).  **I don't modify ELECTRA until we get into finetuning**
+
+Tokenization and training CoLab: https://colab.research.google.com/drive/1R8TciRSM7BONJRBc9CBZbzOmz39FTLl_
+
+Blog post: https://medium.com/@mapmeld/teaching-hindi-to-electra-b11084baab81
+
+Greatly influenced by: https://huggingface.co/blog/how-to-train
+
+## Corpus
+
+Download: https://drive.google.com/drive/u/1/folders/1WikYHHMI72hjZoCQkLPr45LDV8zm9P7p
+
+The corpus is two files:
+- Hindi CommonCrawl deduped by OSCAR https://traces1.inria.fr/oscar/
+- latest Hindi Wikipedia ( https://dumps.wikimedia.org/hiwiki/20200420/ ) + WikiExtractor to txt 
+
+Bonus notes:
+- Adding English wiki text or parallel corpus could help with cross-lingual tasks and training
+
+## Vocabulary
+
+https://drive.google.com/file/d/1-02Um-8ogD4vjn4t-wD2EwCE-GtBjnzh/view?usp=sharing
+
+Bonus notes:
+- Created with HuggingFace Tokenizers; could be longer or shorter, review ELECTRA vocab_size param
+
+## Pretrain TF Records
+
+[build_pretraining_dataset.py](https://github.com/google-research/electra/blob/master/build_pretraining_dataset.py) splits the corpus into training documents
+
+Set the ELECTRA model size and whether to split the corpus by newlines.  This process can take hours on its own.
+
+https://drive.google.com/drive/u/1/folders/1--wBjSH59HSFOVkYi4X-z5bigLnD32R5
+
+Bonus notes:
+- I am not sure of the meaning of the corpus newline split (what is the alternative?) and given this corpus, which creates the better training docs
+
+## Training
+
+Structure your files, with data-dir named "trainer" here
+
+```
+trainer
+- vocab.txt
+- pretrain_tfrecords
+-- (all .tfrecord... files)
+- models
+-- modelname
+--- checkpoint
+--- graph.pbtxt
+--- model.*
+```
+
+CoLab notebook gives examples of GPU vs. TPU setup
+
+[configure_pretraining.py](https://github.com/google-research/electra/blob/master/configure_pretraining.py)
+
+Model https://drive.google.com/drive/folders/1cwQlWryLE4nlke4OixXA7NK8hzlmUR0c?usp=sharing
+
+## Using this model with Transformers
+
+Sample movie reviews classifier: https://colab.research.google.com/drive/1mSeeSfVSOT7e-dVhPlmSsQRvpn6xC05w

model_cards/mrm8488/RuPERTa-base-finetuned-squadv1/README.md

@@ -1,5 +0,0 @@
----
-language: es
-datasets:
-- squad
----

model_cards/mrm8488/RuPERTa-base-finetuned-squadv2/README.md

@@ -1,5 +0,0 @@
----
-language: es
-datasets:
-- squad_v2
----

model_cards/mrm8488/t5-base-finetuned-e2m-intent/README.md

@@ -1,64 +0,0 @@
----
-language: en
-datasets:
-- event2Mind
----
-
-# T5-base fine-tuned on event2Mind for **Intent Prediction** 🤔
-
-[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [event2Mind](https://huggingface.co/nlp/viewer/?dataset=event2Mind) dataset for **Intent Prediction**.
-
-## Details of T5 📜 ➡️ 📜
-
-The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract:
-
-Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
-
-![model image](https://i.imgur.com/jVFMMWR.png)
-
-
-## Details of the downstream task (Intent Prediction) - Dataset 📚 
-
-Dataset ID: ```event2Mind``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
-
-| Dataset  | Split | # samples |
-| -------- | ----- | --------- |
-| event2Mind | train | 46472    |
-| event2Mind | valid  | 1960    |
-
-Events without **intent** were not used!
-
-Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/)
-
-
-## Model fine-tuning 🏋️‍
-The training script is a slightly modified version of [this  awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28).
-
-
-## Model in Action 🚀
-
-```python
-# Tip: By now, install transformers from source
-
-from transformers import AutoModelWithLMHead, AutoTokenizer
-
-tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-e2m-intent")
-model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-e2m-intent")
-
-def get_intent(event, max_length=16):
-  input_text = "%s </s>" % event
-  features = tokenizer([input_text], return_tensors='pt')
-
-  output = model.generate(input_ids=features['input_ids'], 
-               attention_mask=features['attention_mask'],
-               max_length=max_length)
-
-  return tokenizer.decode(output[0])
-
-event = "PersonX takes PersonY home"
-get_intent(event)
-
-# output: 'to be helpful'
-```
-> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
-> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/t5-base-finetuned-question-generation-ap/README.md

@@ -1,75 +0,0 @@
----
-language: en
-datasets:
-- squad
----
-
-# T5-base fine-tuned on SQuAD for **Question Generation**
-
-[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [SQuAD v1.1](https://rajpurkar.github.io/SQuAD-explorer/) for **Question Generation** by just prepending the *answer* to the *context*.
-
-## Details of T5
-
-The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract:
-
-Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
-
-![model image](https://i.imgur.com/jVFMMWR.png)
-
-
-## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓
-
-Dataset ID: ```squad``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
-
-| Dataset  | Split | # samples |
-| -------- | ----- | --------- |
-| squad | train | 87599    |
-| squad | valid  | 10570    |
-
-How to load it from [nlp](https://github.com/huggingface/nlp)
-
-```python
-train_dataset  = nlp.load_dataset('squad, split=nlp.Split.TRAIN)
-valid_dataset = nlp.load_dataset('squad', split=nlp.Split.VALIDATION)
-```
-Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/)
-
-
-## Model fine-tuning 🏋️‍
-
-The training script is a slightly modified version of [this  awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28)
-
-He also made a great research on [**Question Generation**](https://github.com/patil-suraj/question_generation)
-
-
-## Model in Action 🚀
-
-```python
-# Tip: By now, install transformers from source
-
-from transformers import AutoModelWithLMHead, AutoTokenizer
-
-tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-question-generation-ap")
-model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-question-generation-ap")
-
-def get_question(answer, context, max_length=64):
-  input_text = "answer: %s  context: %s </s>" % (answer, context)
-  features = tokenizer([input_text], return_tensors='pt')
-
-  output = model.generate(input_ids=features['input_ids'], 
-               attention_mask=features['attention_mask'],
-               max_length=max_length)
-
-  return tokenizer.decode(output[0])
-
-context = "Manuel have created RuPERTa-base with the support of HF-Transformers and Google"
-answer = "Manuel"
-
-get_question(answer, context)
-
-# output: question: Who created the RuPERTa-base?
-```
-
-> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
-
-> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/t5-small-finetuned-quora-for-paraphrasing/README.md

@@ -1,73 +0,0 @@
----
-language: en
-datasets:
-- quora
----
-
-# T5-base fine-tuned on Quora question pair dataset for Question Paraphrasing ❓↔️❓
-
-[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [Quodra question pair](https://huggingface.co/nlp/viewer/?dataset=quora) dataset for **Question Paraphrasing** task.
-
-## Details of T5
-
-The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract:
-
-Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code.
-
-![model image](https://i.imgur.com/jVFMMWR.png)
-
-
-## Details of the downstream task (Question Paraphrasing) - Dataset 📚❓↔️❓
-
-Dataset ID: ```quora``` from  [HugginFace/NLP](https://github.com/huggingface/nlp)
-
-| Dataset  | Split | # samples |
-| -------- | ----- | --------- |
-| quora | train | 404290    |
-| quora after filter repeated questions | train  | 149263    |
-
-Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/)
-
-
-## Model fine-tuning 🏋️‍
-
-The training script is a slightly modified version of [this one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb)
-
-
-
-## Model in Action 🚀
-
-```python
-from transformers import AutoModelWithLMHead, AutoTokenizer
-
-tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-small-finetuned-quora-for-paraphrasing")
-model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-small-finetuned-quora-for-paraphrasing")
-
-def paraphrase(text, max_length=128):
-
-  input_ids = tokenizer.encode(text, return_tensors="pt", add_special_tokens=True)
-
-  generated_ids = model.generate(input_ids=input_ids, num_return_sequences=5, num_beams=5, max_length=max_length, no_repeat_ngram_size=2, repetition_penalty=3.5, length_penalty=1.0, early_stopping=True)
-
-  preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]
-
-  return preds
-  
-preds = paraphrase("paraphrase: What is the best framework for dealing with a huge text dataset?")
-
-for pred in preds:
-  print(pred)
-
-# Output:
-'''
-What is the best framework for dealing with a huge text dataset?
-What is the best framework for dealing with a large text dataset?
-What is the best framework to deal with a huge text dataset?
-What are the best frameworks for dealing with a huge text dataset?
-What is the best framework for dealing with huge text datasets?
-'''
-```
-
-> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/)
-
-> Made with <span style="color: #e25555;">&hearts;</span> in Spain

model_cards/mrm8488/t5-small-finetuned-squadv2/README.md

@@ -65,7 +65,7 @@ def get_answer(question, context):
 
   return tokenizer.decode(output[0])
 
-context = "Manuel has created RuPERTa-base (a Spanish RoBERTa) with the support of HF-Transformers and Google"
+context = "Manuel have created RuPERTa-base (a Spanish RoBERTa) with the support of HF-Transformers and Google"
 question = "Who has supported Manuel?"
 
 get_answer(question, context)

model_cards/pranavpsv/gpt2-genre-story-generator/README.md

@@ -1,32 +0,0 @@
-
-# GPT2 Genre Based Story Generator
-
-## Model description
-
-GPT2 fine-tuned on genre-based story generation.
-
-## Intended uses
-
-Used to generate stories based on user inputted genre and starting prompts.
-
-## How to use
-
-#### Supported Genres
-superhero, action, drama, horror, thriller, sci_fi
-#### Input text format
-\<BOS> \<genre> Some optional text...
-
-**Example**: \<BOS> \<sci_fi> After discovering time travel,
-
-```python
-# Example of usage
-from transformers import pipeline
-
-story_gen = pipeline("text-generation", "pranavpsv/gpt2-genre-story-generator")
-print(story_gen("<BOS> <superhero> Batman"))
-
-```
-
-## Training data
-
-Initialized with pre-trained weights of "gpt2" checkpoint. Fine-tuned the model on stories of various genres.

model_cards/rohanrajpal/bert-base-codemixed-uncased-sentiment/README.md

@@ -1,61 +0,0 @@
----
-language:
-- hi
-- en
-tags:
-- hi
-- en
-- codemix
-datasets:
-- SAIL 2017
----
-
-# Model name
-
-## Model description
-
-I took a bert-base-multilingual-cased model from huggingface and finetuned it on SAIL 2017 dataset.  
-
-## Intended uses & limitations
-
-#### How to use
-
-```python
-# You can include sample code which will be formatted
-#Coming soon!
-```
-
-#### Limitations and bias
-
-Provide examples of latent issues and potential remediations.
-
-## Training data
-
-I trained on the SAIL 2017 dataset [link](http://amitavadas.com/SAIL/Data/SAIL_2017.zip) on this [pretrained model](https://huggingface.co/bert-base-multilingual-cased).
-
-
-## Training procedure
-
-No preprocessing.
-
-## Eval results
-
-### BibTeX entry and citation info
-
-```bibtex
-@inproceedings{khanuja-etal-2020-gluecos,
-    title = "{GLUEC}o{S}: An Evaluation Benchmark for Code-Switched {NLP}",
-    author = "Khanuja, Simran  and
-      Dandapat, Sandipan  and
-      Srinivasan, Anirudh  and
-      Sitaram, Sunayana  and
-      Choudhury, Monojit",
-    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
-    month = jul,
-    year = "2020",
-    address = "Online",
-    publisher = "Association for Computational Linguistics",
-    url = "https://www.aclweb.org/anthology/2020.acl-main.329",
-    pages = "3575--3585"
-}
-```

model_cards/rohanrajpal/bert-base-multilingual-codemixed-cased-sentiment/README.md

@@ -1,95 +0,0 @@
----
-language:
-- hi
-- en
-tags:
-- hi
-- en
-- codemix
-license: "apache-2.0"
-datasets:
-- SAIL 2017
-metrics:
-- fscore
-- accuracy
----
-
-# BERT codemixed base model for hinglish (cased)
-
-## Model description
-
-Input for the model: Any codemixed hinglish text
-Output for the model: Sentiment. (0 - Negative, 1 - Neutral, 2 - Positive)
-
-I took a bert-base-multilingual-cased model from Huggingface and finetuned it on [SAIL 2017](http://www.dasdipankar.com/SAILCodeMixed.html) dataset.  
-
-Performance of this model on the SAIL 2017 dataset
-
-| metric     |    score |
-|------------|----------|
-| acc        | 0.588889 |
-| f1         | 0.582678 |
-| acc_and_f1 | 0.585783 |
-| precision  | 0.586516 |
-| recall     | 0.588889 |
-
-## Intended uses & limitations
-
-#### How to use
-
-Here is how to use this model to get the features of a given text in *PyTorch*:
-
-```python
-# You can include sample code which will be formatted
-from transformers import BertTokenizer, BertModelForSequenceClassification
-tokenizer = AutoTokenizer.from_pretrained("rohanrajpal/bert-base-codemixed-uncased-sentiment")
-model = AutoModelForSequenceClassification.from_pretrained("rohanrajpal/bert-base-codemixed-uncased-sentiment")
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='pt')
-output = model(**encoded_input)
-```
-
-and in *TensorFlow*:
-
-```python
-from transformers import BertTokenizer, TFBertModel
-tokenizer = BertTokenizer.from_pretrained('rohanrajpal/bert-base-codemixed-uncased-sentiment')
-model = TFBertModel.from_pretrained("rohanrajpal/bert-base-codemixed-uncased-sentiment")
-text = "Replace me by any text you'd like."
-encoded_input = tokenizer(text, return_tensors='tf')
-output = model(encoded_input)
-```
-
-#### Limitations and bias
-
-Coming soon!
-
-## Training data
-
-I trained on the SAIL 2017 dataset [link](http://amitavadas.com/SAIL/Data/SAIL_2017.zip) on this [pretrained model](https://huggingface.co/bert-base-multilingual-cased).
-
-## Training procedure
-
-No preprocessing.
-
-## Eval results
-
-### BibTeX entry and citation info
-
-```bibtex
-@inproceedings{khanuja-etal-2020-gluecos,
-    title = "{GLUEC}o{S}: An Evaluation Benchmark for Code-Switched {NLP}",
-    author = "Khanuja, Simran  and
-      Dandapat, Sandipan  and
-      Srinivasan, Anirudh  and
-      Sitaram, Sunayana  and
-      Choudhury, Monojit",
-    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
-    month = jul,
-    year = "2020",
-    address = "Online",
-    publisher = "Association for Computational Linguistics",
-    url = "https://www.aclweb.org/anthology/2020.acl-main.329",
-    pages = "3575--3585"
-}
-```

setup.cfg

@@ -5,7 +5,6 @@ include_trailing_comma = True
 known_first_party = transformers
 known_third_party =
     absl
-    conllu
     elasticsearch
     fairseq
     faiss

src/transformers/__init__.py

@@ -22,7 +22,7 @@ import logging
 # Configurations
 from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
 from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, AutoConfig
-from .configuration_bart import BartConfig
+from .configuration_bart import BartConfig, MBartConfig
 from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig
 from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
 from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
@@ -34,11 +34,9 @@ from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, Flau
 from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config
 from .configuration_longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig
 from .configuration_marian import MarianConfig
-from .configuration_mbart import MBartConfig
 from .configuration_mmbt import MMBTConfig
 from .configuration_mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig
 from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
-from .configuration_pegasus import PegasusConfig
 from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig
 from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
 from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig
@@ -132,7 +130,7 @@ from .pipelines import (
 # Tokenizers
 from .tokenization_albert import AlbertTokenizer
 from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
-from .tokenization_bart import BartTokenizer, BartTokenizerFast
+from .tokenization_bart import BartTokenizer, BartTokenizerFast, MBartTokenizer
 from .tokenization_bert import BasicTokenizer, BertTokenizer, BertTokenizerFast, WordpieceTokenizer
 from .tokenization_bert_japanese import BertJapaneseTokenizer, CharacterTokenizer, MecabTokenizer
 from .tokenization_camembert import CamembertTokenizer
@@ -150,10 +148,8 @@ from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast
 from .tokenization_flaubert import FlaubertTokenizer
 from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
 from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast
-from .tokenization_mbart import MBartTokenizer
 from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFast
 from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
-from .tokenization_pegasus import PegasusTokenizer
 from .tokenization_reformer import ReformerTokenizer
 from .tokenization_retribert import RetriBertTokenizer, RetriBertTokenizerFast
 from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
@@ -291,7 +287,6 @@ if is_torch_available():
         XLMForMultipleChoice,
         XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
     )
-    from .modeling_pegasus import PegasusForConditionalGeneration
     from .modeling_bart import (
         PretrainedBartModel,
         BartForSequenceClassification,
@@ -300,12 +295,10 @@ if is_torch_available():
         BartForQuestionAnswering,
         BART_PRETRAINED_MODEL_ARCHIVE_LIST,
     )
-    from .modeling_mbart import MBartForConditionalGeneration
     from .modeling_marian import MarianMTModel
     from .tokenization_marian import MarianTokenizer
     from .modeling_roberta import (
         RobertaForMaskedLM,
-        RobertaForCausalLM,
         RobertaModel,
         RobertaForSequenceClassification,
         RobertaForMultipleChoice,
@@ -406,7 +399,6 @@ if is_torch_available():
         LongformerForMultipleChoice,
         LongformerForTokenClassification,
         LongformerForQuestionAnswering,
-        LongformerSelfAttention,
         LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
     )
 
@@ -432,7 +424,6 @@ if is_torch_available():
         get_cosine_schedule_with_warmup,
         get_cosine_with_hard_restarts_schedule_with_warmup,
         get_linear_schedule_with_warmup,
-        get_polynomial_decay_schedule_with_warmup,
     )
 
     # Trainer
@@ -442,7 +433,6 @@ if is_torch_available():
         DataCollator,
         DataCollatorForLanguageModeling,
         DataCollatorForPermutationLanguageModeling,
-        DataCollatorWithPadding,
     )
     from .data.datasets import (
         GlueDataset,
@@ -578,14 +568,6 @@ if is_tf_available():
         TFGPT2PreTrainedModel,
     )
 
-    from .modeling_tf_longformer import (
-        TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
-        TFLongformerModel,
-        TFLongformerForMaskedLM,
-        TFLongformerForQuestionAnswering,
-        TFLongformerSelfAttention,
-    )
-
     from .modeling_tf_mobilebert import (
         TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFMobileBertModel,

src/transformers/configuration_auto.py

@@ -19,7 +19,7 @@ import logging
 from collections import OrderedDict
 
 from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig
-from .configuration_bart import BART_PRETRAINED_CONFIG_ARCHIVE_MAP, BartConfig
+from .configuration_bart import BART_PRETRAINED_CONFIG_ARCHIVE_MAP, BartConfig, MBartConfig
 from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig
 from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig
 from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
@@ -30,10 +30,8 @@ from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, Flau
 from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config
 from .configuration_longformer import LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, LongformerConfig
 from .configuration_marian import MarianConfig
-from .configuration_mbart import MBART_PRETRAINED_CONFIG_ARCHIVE_MAP, MBartConfig
 from .configuration_mobilebert import MobileBertConfig
 from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
-from .configuration_pegasus import PegasusConfig
 from .configuration_reformer import ReformerConfig
 from .configuration_retribert import RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RetriBertConfig
 from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig
@@ -53,7 +51,6 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
     for pretrained_map in [
         BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         BART_PRETRAINED_CONFIG_ARCHIVE_MAP,
-        MBART_PRETRAINED_CONFIG_ARCHIVE_MAP,
         OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,
         GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
@@ -84,7 +81,6 @@ CONFIG_MAPPING = OrderedDict(
         ("albert", AlbertConfig,),
         ("camembert", CamembertConfig,),
         ("xlm-roberta", XLMRobertaConfig,),
-        ("pegasus", PegasusConfig),
         ("marian", MarianConfig,),
         ("mbart", MBartConfig,),
         ("bart", BartConfig,),

src/transformers/configuration_bart.py

@@ -18,7 +18,6 @@
 import logging
 
 from .configuration_utils import PretrainedConfig
-from .file_utils import add_start_docstrings_to_callable
 
 
 logger = logging.getLogger(__name__)
@@ -33,73 +32,7 @@ BART_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "yjernite/bart_eli5": "https://s3.amazonaws.com/models.huggingface.co/bert/yjernite/bart_eli5/config.json",
 }
 
-BART_CONFIG_ARGS_DOC = r"""
-    Args:
-        vocab_size (:obj:`int`, optional, defaults to 50265):
-            defines the different tokens that can be represented by `inputs_ids` passed to the forward method.
-        d_model (:obj:`int`, optional, defaults to 1024):
-            Dimensionality of the layers and the pooler layer.
-        encoder_layers (:obj:`int`, optional, defaults to 12):
-            Number of encoder layers, 16 for pegasus, 6 for bart-base and marian
-        decoder_layers (:obj:`int`, optional, defaults to 12):
-            Number of decoder layers, 16 for pegasus, 6 for bart-base and marian
-        encoder_attention_heads (:obj:`int`, optional, defaults to 16):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        decoder_attention_heads (:obj:`int`, optional, defaults to 16):
-            Number of attention heads for each attention layer in the Transformer decoder.
-        decoder_ffn_dim (:obj:`int`, optional, defaults to 4096):
-            Dimensionality of the "intermediate" (i.e., feed-forward) layer in decoder.
-        encoder_ffn_dim (:obj:`int`, optional, defaults to 4096):
-            Dimensionality of the "intermediate" (i.e., feed-forward) layer in decoder.
-        activation_function (:obj:`str` or :obj:`function`, optional, defaults to "gelu"):
-            The non-linear activation function (function or string) in the encoder and pooler.
-            If string, "gelu", "relu", "swish" and "gelu_new" are supported.
-        dropout (:obj:`float`, optional, defaults to 0.1):
-            The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
-        attention_dropout (:obj:`float`, optional, defaults to 0.0):
-            The dropout ratio for the attention probabilities.
-        activation_dropout (:obj:`float`, optional, defaults to 0.0):
-            The dropout ratio for activations inside the fully connected layer.
-        classifier_dropout (:obj:`float`, optional, defaults to 0.0):
-            The dropout ratio for classifier.
-        max_position_embeddings (:obj:`int`, optional, defaults to 1024):
-            The maximum sequence length that this model might ever be used with.
-            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
-        init_std (:obj:`float`, optional, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        add_bias_logits (:obj:`int`, optional, defaults to False):
-            True for marian only.
-        normalize_before (:obj:`bool`, optional, defaults to False):
-            Call layernorm before attention ops. True for pegasus, mbart. False for bart. FIXME: marian?
-        normalize_embedding (:obj:`bool`, optional, defaults to True):
-            Call layernorm after embeddings. Only True for Bart.
-        static_position_embeddings (:obj:`bool`, optional, defaults to False):
-            Don't learn positional embeddings, use sinusoidal. True for marian, pegasus.
-        add_final_layer_norm (:obj:`bool`, optional, defaults to False):
-            Why not add another layernorm?
-        scale_embedding (:obj:`bool`, optional, defaults to False):
-            Scale embeddings by diving by sqrt(d_model).
-        eos_token_id (:obj:`int`, optional, defaults to 2)
-            End of stream token id.
-        pad_token_id (:obj:`int`, optional, defaults to 1)
-            Padding token id.
-        bos_token_id (:obj:`int`, optional, defaults to 0)
-            Beginning of stream token id.
-        encoder_layerdrop: (:obj:`float`, optional, defaults to 0.0):
-            Google "layerdrop arxiv", as its not explainable in one line.
-        decoder_layerdrop: (:obj:`float`, optional, defaults to 0.0):
-            Google "layerdrop arxiv", as its not explainable in one line.
-        extra_pos_embeddings: (:obj:`int`, optional, defaults to 2):
-            How many extra learned positional embeddings to use. Should be pad_token_id+1 for bart.
-        num_labels: (:obj:`int`, optional, defaults to 2):
-            for SequenceClassification
-        is_encoder_decoder (:obj:`int`, optional, defaults to True):
-            True
 
-"""
-
-
-@add_start_docstrings_to_callable(BART_CONFIG_ARGS_DOC)
 class BartConfig(PretrainedConfig):
     r"""
         Configuration class for Bart. Parameters are renamed from the fairseq implementation
@@ -109,7 +42,7 @@ class BartConfig(PretrainedConfig):
     def __init__(
         self,
         activation_dropout=0.0,
-        extra_pos_embeddings=2,  # FIXME(@sshleifer): delete?
+        extra_pos_embeddings=2,
         activation_function="gelu",
         vocab_size=50265,
         d_model=1024,
@@ -148,7 +81,6 @@ class BartConfig(PretrainedConfig):
 
                 >>> config = BartConfig.from_pretrained('facebook/bart-large')
                 >>> model = BartModel(config)
-
         """
         if "hidden_size" in common_kwargs:
             raise ValueError("hidden size is called d_model")
@@ -210,3 +142,7 @@ class BartConfig(PretrainedConfig):
         if self.normalize_before or self.add_final_layer_norm or self.scale_embedding:
             logger.info("This configuration is a mixture of MBART and BART settings")
         return False
+
+
+class MBartConfig(BartConfig):
+    model_type = "mbart"

src/transformers/configuration_encoder_decoder.py

@@ -58,7 +58,6 @@ class EncoderDecoderConfig(PretrainedConfig):
             >>> config_decoder  = model.config.decoder
             >>> # set decoder config to causal lm
             >>> config_decoder.is_decoder = True
-            >>> config_decoder.add_cross_attention = True
 
             >>> # Saving the model, including its configuration
             >>> model.save_pretrained('my-model')
@@ -95,9 +94,8 @@ class EncoderDecoderConfig(PretrainedConfig):
         Returns:
             :class:`EncoderDecoderConfig`: An instance of a configuration object
         """
-        logger.info("Set `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config")
+        logger.info("Set `config.is_decoder=True` for decoder_config")
         decoder_config.is_decoder = True
-        decoder_config.add_cross_attention = True
 
         return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict())
 

src/transformers/configuration_mbart.py

@@ -1,32 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Fairseq Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" MBART configuration """
-
-import logging
-
-from .configuration_bart import BartConfig
-
-
-logger = logging.getLogger(__name__)
-
-MBART_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "facebook/mbart-large-en-ro": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/mbart-large-en-ro/config.json",
-    "facebook/mbart-large-cc25": "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/mbart-large-cc25/config.json",
-}
-
-
-class MBartConfig(BartConfig):
-    model_type = "mbart"
-    """See real config values at https://s3.amazonaws.com/models.huggingface.co/bert/facebook/mbart-large-en-ro/config.json."""

src/transformers/configuration_pegasus.py

@@ -1,62 +0,0 @@
-# coding=utf-8
-# Copyright 2020 Google and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" PEGASUS model configuration """
-
-import logging
-
-from .configuration_bart import BART_CONFIG_ARGS_DOC, BartConfig
-from .file_utils import add_start_docstrings_to_callable
-
-
-logger = logging.getLogger(__name__)
-
-DEFAULTS = dict(
-    vocab_size=96103,
-    max_position_embeddings=512,
-    d_model=1024,
-    encoder_ffn_dim=4096,
-    decoder_ffn_dim=4096,
-    encoder_attention_heads=16,
-    decoder_attention_heads=16,
-    encoder_layers=16,
-    decoder_layers=16,
-    dropout=0.1,
-    attention_dropout=0.1,
-    activation_dropout=0.1,
-    pad_token_id=0,
-    eos_token_id=1,
-    is_encoder_decoder=True,
-    normalize_before=True,
-    scale_embedding=True,
-    normalize_embedding=False,
-    add_final_layer_norm=True,
-    static_position_embeddings=True,
-    num_beams=8,
-    activation_function="relu",
-)
-
-
-@add_start_docstrings_to_callable(BART_CONFIG_ARGS_DOC)
-class PegasusConfig(BartConfig):
-    r"""
-        :class:`~transformers.PegasusConfig` is the configuration class to store the configuration of a
-        `PegasusModel`.
-    """
-    model_type = "pegasus"
-    # The implementation of the config object is in BartConfig
-
-    @property
-    def default_config_parameters(self):
-        return DEFAULTS

src/transformers/configuration_reformer.py

@@ -64,6 +64,11 @@ class ReformerConfig(PretrainedConfig):
                 A chunk size of 0 means that the feed forward layer is not chunked.
                 A chunk size of n means that the feed forward layer processes n < sequence_length embeddings at a time.
                 For more information on feed forward chunking, see `How does Feed Forward Chunking work? <../glossary.html#feed-forward-chunking>`__ .
+            chunk_size_feed_forward (:obj:`int`, optional, defaults to 0):
+                The chunk size of all feed forward layers in the residual attention blocks.
+                A chunk size of 0 means that the feed forward layer is not chunked.
+                A chunk size of n means that the feed forward layer processes n < sequence_length embeddings at a time.
+                For more information on feed forward chunking, see `How does Feed Forward Chunking work? <../glossary.html#feed-forward-chunking>`__ .
             eos_token_id (:obj:`int`, optional, defaults to 2):
                 The token id for the <EOS> token.
             feed_forward_size (:obj:`int`, optional, defaults to 512):
@@ -142,6 +147,7 @@ class ReformerConfig(PretrainedConfig):
         axial_pos_shape=[64, 64],
         axial_pos_embds_dim=[64, 192],
         chunk_size_lm_head=0,
+        chunk_size_feed_forward=0,
         eos_token_id=2,
         feed_forward_size=512,
         hash_seed=None,
@@ -196,4 +202,5 @@ class ReformerConfig(PretrainedConfig):
         self.axial_pos_embds_dim = tuple(axial_pos_embds_dim)
         self.axial_norm_std = axial_norm_std
         self.chunk_size_lm_head = chunk_size_lm_head
+        self.chunk_size_feed_forward = chunk_size_feed_forward
         self.attn_layers = attn_layers

src/transformers/configuration_utils.py

@@ -56,8 +56,6 @@ class PretrainedConfig(object):
                 Whether the model is used as an encoder/decoder or not.
             is_decoder (:obj:`bool`, `optional`, defaults to :obj:`False`):
                 Whether the model is used as decoder or not (in which case it's used as an encoder).
-            add_cross_attention (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Whether cross-attention layers should be added to the model. Note, this option is only relevant for models that can be used as decoder models within the `:class:~transformers.EncoderDecoderModel` class, which consists of all models in ``AUTO_MODELS_FOR_CAUSAL_LM``.
             prune_heads (:obj:`Dict[int, List[int]]`, `optional`, defaults to :obj:`{}`):
                 Pruned heads of the model. The keys are the selected layer indices and the associated values, the list
                 of heads to prune in said layer.
@@ -66,11 +64,6 @@ class PretrainedConfig(object):
                 2.
             xla_device (:obj:`bool`, `optional`):
                 A flag to indicate if TPU are available or not.
-            chunk_size_feed_forward (:obj:`int`, `optional`, defaults to :obj:`0`):
-                The chunk size of all feed forward layers in the residual attention blocks.
-                A chunk size of :obj:`0` means that the feed forward layer is not chunked.
-                A chunk size of n means that the feed forward layer processes :obj:`n` < sequence_length embeddings at a time.
-                For more information on feed forward chunking, see `How does Feed Forward Chunking work? <../glossary.html#feed-forward-chunking>`__ .
 
         Parameters for sequence generation
             - **max_length** (:obj:`int`, `optional`, defaults to 20) -- Maximum length that will be used by
@@ -152,7 +145,6 @@ class PretrainedConfig(object):
         # Is decoder is used in encoder-decoder models to differentiate encoder from decoder
         self.is_encoder_decoder = kwargs.pop("is_encoder_decoder", False)
         self.is_decoder = kwargs.pop("is_decoder", False)
-        self.add_cross_attention = kwargs.pop("add_cross_attention", False)
 
         # Parameters for sequence generation
         self.max_length = kwargs.pop("max_length", 20)
@@ -168,7 +160,6 @@ class PretrainedConfig(object):
         self.no_repeat_ngram_size = kwargs.pop("no_repeat_ngram_size", 0)
         self.bad_words_ids = kwargs.pop("bad_words_ids", None)
         self.num_return_sequences = kwargs.pop("num_return_sequences", 1)
-        self.chunk_size_feed_forward = kwargs.pop("chunk_size_feed_forward", 0)
 
         # Fine-tuning task arguments
         self.architectures = kwargs.pop("architectures", None)

src/transformers/convert_pegasus_tf_to_pytorch.py

@@ -1,167 +0,0 @@
-# coding=utf-8
-# Copyright 2020 Google and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-from pathlib import Path
-from typing import Dict
-
-import tensorflow as tf
-import torch
-from tqdm import tqdm
-
-from transformers import PegasusConfig, PegasusForConditionalGeneration, PegasusTokenizer
-from transformers.configuration_pegasus import DEFAULTS
-
-
-PATTERNS = [
-    # replace left string with right string to get the relevant state_dict key (identical state dict to bart)
-    ["memory_attention", "encoder_attn"],
-    ["attention", "attn"],
-    ["/", "."],
-    [".LayerNorm.gamma", "_layer_norm.weight"],
-    [".LayerNorm.beta", "_layer_norm.bias"],
-    ["r.layer_", "r.layers."],
-    ["output_proj", "out_proj"],
-    ["ffn.dense_1.", "fc2."],
-    ["ffn.dense.", "fc1."],
-    ["ffn_layer_norm", "final_layer_norm"],
-    ["kernel", "weight"],
-    ["encoder_layer_norm.", "encoder.layer_norm."],
-    ["decoder_layer_norm.", "decoder.layer_norm."],
-    ["embeddings.weights", "shared.weight"],
-]
-
-
-def rename_state_dict_key(k):
-
-    for pegasus_name, bart_name in PATTERNS:
-        k = k.replace(pegasus_name, bart_name)
-    return k
-
-
-# See appendix C of paper for all hyperparams
-max_gen_length = {
-    # See appendix C of paper
-    "xsum": 64,
-    "cnn_dailymail": 128,
-    "newsroom": 128,
-    "wikihow": 256,
-    "multi_news": 256,
-    "reddit_tifu": 128,
-    "big_patent": 256,
-    "arxiv": 256,
-    "pubmed": 256,
-    "gigaword": 32,
-    "aeslc": 32,
-    "billsum": 256,
-    "large": 256,  # @sshleifer chose arbitrarily
-}
-max_model_length = {
-    "xsum": 512,
-    "cnn_dailymail": 1024,
-    "newsroom": 512,
-    "wikihow": 512,
-    "multi_news": 1024,
-    "reddit_tifu": 512,
-    "big_patent": 1024,
-    "arxiv": 1024,
-    "pubmed": 1024,
-    "gigaword": 128,
-    "aeslc": 512,
-    "billsum": 1024,
-    "large": 1024,
-}
-
-expected_alpha = {
-    "multinews": 0.9,
-    "wikihow": 0.6,
-    "reddit_tifu": 0.6,
-    "big_patent": 0.7,
-    "gigaword": 0.6,
-    "aeslc": 0.6,
-    "billsum": 0.6,
-}  # otherwise 0.8
-# TODO(SS): one constant
-
-
-def convert_pegasus_to_bart(tf_weights: dict, cfg_updates: dict) -> PegasusForConditionalGeneration:
-    cfg_kwargs = DEFAULTS.copy()
-    cfg_kwargs.update(cfg_updates)
-
-    cfg = PegasusConfig(**cfg_updates)
-    bart = PegasusForConditionalGeneration(cfg)
-    sd = bart.model.state_dict()
-    mapping = {}
-    for k, v in tf_weights.items():
-        new_k = rename_state_dict_key(k)
-        if new_k not in sd:
-            raise ValueError(f"could not find new key {new_k} in state dict. (converted from {k})")
-
-        if "dense" in k or "proj" in new_k:
-            v = v.T
-        mapping[new_k] = torch.tensor(v, dtype=sd[new_k].dtype)
-        assert v.shape == sd[new_k].shape, f"{new_k}, {k}, {v.shape}, {sd[new_k].shape}"
-    # make sure embedding.padding_idx is respected
-    mapping["shared.weight"][cfg.pad_token_id] = torch.zeros_like(mapping["shared.weight"][cfg.pad_token_id + 1])
-    mapping["encoder.embed_tokens.weight"] = mapping["shared.weight"]
-    mapping["decoder.embed_tokens.weight"] = mapping["shared.weight"]
-    empty_biases = {k: torch.zeros_like(v) for k, v in sd.items() if k.endswith("bias") and k not in mapping}
-    mapping.update(**empty_biases)
-    missing, extra = bart.model.load_state_dict(mapping, strict=False)
-    unexpected_missing = [
-        k for k in missing if k not in ["encoder.embed_positions.weight", "decoder.embed_positions.weight"]
-    ]
-    assert unexpected_missing == [], f"no matches found for the following torch keys {unexpected_missing}"
-    assert extra == [], f"no matches found for the following tf keys {extra}"
-    return bart
-
-
-def get_tf_weights_as_numpy(path="./ckpt/aeslc/model.ckpt-32000") -> Dict:
-    init_vars = tf.train.list_variables(path)
-    tf_weights = {}
-    ignore_name = ["Adafactor", "global_step"]
-    for name, shape in tqdm(init_vars, desc="converting tf checkpoint to dict"):
-        skip_key = any([pat in name for pat in ignore_name])
-        if skip_key:
-            continue
-        array = tf.train.load_variable(path, name)
-        tf_weights[name] = array
-    return tf_weights
-
-
-def convert_pegasus_ckpt_to_pytorch(ckpt_path, save_dir):
-    # save tokenizer first
-    dataset = Path(ckpt_path).parent.name
-    desired_max_model_length = max_model_length[dataset]
-    tok = PegasusTokenizer.from_pretrained("sshleifer/pegasus", model_max_length=desired_max_model_length)
-    assert tok.model_max_length == desired_max_model_length
-    tok.save_pretrained(save_dir)
-
-    # convert model
-    tf_weights = get_tf_weights_as_numpy(ckpt_path)
-    cfg_updates = dict(max_length=max_gen_length[dataset], length_penalty=expected_alpha.get(dataset, 0.8))
-    torch_model = convert_pegasus_to_bart(tf_weights, cfg_updates)
-    torch_model.save_pretrained(save_dir)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    # Required parameters
-    parser.add_argument("tf_ckpt_path", type=str, help="passed to tf.train.list_variables")
-    parser.add_argument("save_dir", default=None, type=str, help="Path to the output PyTorch model.")
-    args = parser.parse_args()
-    if args.save_dir is None:
-        args.save_dir = f"pegasus/{Path(args.tf_ckpt_path).parent.name}"
-    convert_pegasus_ckpt_to_pytorch(args.tf_ckpt_path, args.save_dir)

src/transformers/data/data_collator.py

@@ -1,12 +1,11 @@
 from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, NewType, Tuple, Union
 
 import torch
 from torch.nn.utils.rnn import pad_sequence
 
 from ..tokenization_utils import PreTrainedTokenizer
-from ..tokenization_utils_base import BatchEncoding, PaddingStrategy
-from ..tokenization_utils_fast import PreTrainedTokenizerFast
+from ..tokenization_utils_base import BatchEncoding
 
 
 InputDataClass = NewType("InputDataClass", Any)
@@ -67,55 +66,6 @@ def default_data_collator(features: List[InputDataClass]) -> Dict[str, torch.Ten
     return batch
 
 
-@dataclass
-class DataCollatorWithPadding:
-    """
-    Data collator that will dynamically pad the inputs received.
-
-    Args:
-        tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
-            The tokenizer used for encoding the data.
-        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
-            Select a strategy to pad the returned sequences (according to the model's padding side and padding
-            index) among:
-
-            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a
-              single sequence if provided).
-            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
-              maximum acceptable input length for the model if that argument is not provided.
-            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
-              different lengths).
-        max_length (:obj:`int`, `optional`):
-            Maximum length of the returned list and optionally padding length (see above).
-        pad_to_multiple_of (:obj:`int`, `optional`):
-            If set will pad the sequence to a multiple of the provided value.
-
-            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
-            >= 7.5 (Volta).
-    """
-
-    tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
-    padding: Union[bool, str, PaddingStrategy] = True
-    max_length: Optional[int] = None
-    pad_to_multiple_of: Optional[int] = None
-
-    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:
-        batch = self.tokenizer.pad(
-            features,
-            padding=self.padding,
-            max_length=self.max_length,
-            pad_to_multiple_of=self.pad_to_multiple_of,
-            return_tensors="pt",
-        )
-        if "label" in batch:
-            batch["labels"] = batch["label"]
-            del batch["label"]
-        if "label_ids" in batch:
-            batch["labels"] = batch["label_ids"]
-            del batch["label_ids"]
-        return batch
-
-
 @dataclass
 class DataCollatorForLanguageModeling:
     """

src/transformers/data/test_generation_utils.py

@@ -1,90 +0,0 @@
-import random
-import unittest
-
-import timeout_decorator
-
-from transformers import is_torch_available
-from transformers.file_utils import cached_property
-from transformers.testing_utils import require_torch
-
-
-if is_torch_available():
-    import torch
-
-    from transformers import (
-        MarianConfig,
-        MarianMTModel,
-    )
-
-
-@require_torch
-class GenerationUtilsTest(unittest.TestCase):
-    @cached_property
-    def config(self):
-        config = MarianConfig.from_pretrained("sshleifer/tiny-marian-en-de")
-        return config
-
-    @cached_property
-    def model(self):
-        return MarianMTModel(self.config)
-
-    def test_postprocess_next_token_scores(self):
-        config = self.config
-        model = self.model
-        # Initialize an input id tensor with batch size 8 and sequence length 12
-        input_ids = torch.arange(0, 96, 1).view((8, 12))
-        eos = config.eos_token_id
-        bad_words_ids_test_cases = [[[299]], [[23, 24], [54]], [[config.eos_token_id]], []]
-        masked_scores = [
-            [(0, 299), (1, 299), (2, 299), (3, 299), (4, 299), (5, 299), (6, 299), (7, 299)],
-            [(1, 24), (0, 54), (1, 54), (2, 54), (3, 54), (4, 54), (5, 54), (6, 54), (7, 54)],
-            [(0, eos), (1, eos), (2, eos), (3, eos), (4, eos), (5, eos), (6, eos), (7, eos)],
-            [],
-        ]
-
-        for test_case_index, bad_words_ids in enumerate(bad_words_ids_test_cases):
-            # Initialize a scores tensor with batch size 8 and vocabulary size 300
-            scores = torch.rand((8, 300))
-            output = model.postprocess_next_token_scores(
-                scores,
-                input_ids,
-                0,
-                bad_words_ids,
-                13,
-                15,
-                config.max_length,
-                config.eos_token_id,
-                config.repetition_penalty,
-                32,
-                5,
-            )
-            for masked_score in masked_scores[test_case_index]:
-                self.assertTrue(output[masked_score[0], masked_score[1]] == -float("inf"))
-
-    @timeout_decorator.timeout(10)
-    def test_postprocess_next_token_scores_large_bad_words_list(self):
-
-        config = self.config
-        model = self.model
-        # Initialize an input id tensor with batch size 8 and sequence length 12
-        input_ids = torch.arange(0, 96, 1).view((8, 12))
-
-        bad_words_ids = []
-        for _ in range(100):
-            length_bad_word = random.randint(1, 4)
-            bad_words_ids.append(random.sample(range(1, 300), length_bad_word))
-
-        scores = torch.rand((8, 300))
-        _ = model.postprocess_next_token_scores(
-            scores,
-            input_ids,
-            0,
-            bad_words_ids,
-            13,
-            15,
-            config.max_length,
-            config.eos_token_id,
-            config.repetition_penalty,
-            32,
-            5,
-        )

src/transformers/generation_tf_utils.py

@@ -163,7 +163,7 @@ class TFGenerationMixin:
             model = TFAutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
             input_context = 'The dog'
             input_ids = tokenizer.encode(input_context, return_tensors='tf')  # encode input context
-            outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3, do_sample=True)  # generate 3 candidates using sampling
+            outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3)  # 3 generate sequences using by sampling
             for i in range(3): #  3 output sequences were generated
                 print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True)))
 
@@ -936,8 +936,8 @@ def calc_banned_bad_words_ids(prev_input_ids, bad_words_ids):
         if len(tokens) == 0:
             # if bad word tokens is just one token always ban it
             return True
-        if len(tokens) > len(prev_tokens):
-            # if bad word tokens are longer than prev tokens they can't be equal
+        if len(tokens) > len(prev_input_ids):
+            # if bad word tokens are longer then prev input_ids they can't be equal
             return False
 
         if prev_tokens[-len(tokens) :] == tokens:

src/transformers/generation_utils.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import logging
-from typing import Iterable, List, Optional, Tuple
+from typing import Iterable, Optional, Tuple
 
 import torch
 from torch import Tensor
@@ -89,12 +89,11 @@ class GenerationMixin:
                 scores[i, banned_tokens] = -float("inf")
 
         if bad_words_ids is not None:
-            # Exclude EOS token (already processed)
-            bad_words_ids = list(filter(lambda bad_token_seq: bad_token_seq != [eos_token_id], bad_words_ids))
             # calculate a list of banned tokens according to bad words
-            banned_tokens = calc_banned_bad_words_ids(input_ids.tolist(), bad_words_ids)
-            # Modify the scores in place by setting the banned tokens logits to `-inf`
-            set_scores_to_inf_for_banned_tokens(scores, banned_tokens)
+            banned_tokens = calc_banned_bad_words_ids(input_ids, bad_words_ids)
+
+            for i, banned_tokens in enumerate(banned_tokens):
+                scores[i, banned_tokens] = -float("inf")
 
         return scores
 
@@ -226,7 +225,7 @@ class GenerationMixin:
             model = AutoModelWithLMHead.from_pretrained('distilgpt2')    # Download model and configuration from S3 and cache.
             input_context = 'The dog'
             input_ids = tokenizer.encode(input_context, return_tensors='pt')  # encode input context
-            outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3, do_sample=True)  # generate 3 candidates using sampling
+            outputs = model.generate(input_ids=input_ids, max_length=40, temperature=0.7, num_return_sequences=3)  # 3 generate sequences using by sampling
             for i in range(3): #  3 output sequences were generated
                 print('Generated {}: {}'.format(i, tokenizer.decode(outputs[i], skip_special_tokens=True)))
 
@@ -372,16 +371,11 @@ class GenerationMixin:
 
         if self.config.is_encoder_decoder:
             if decoder_start_token_id is None:
-                # see if BOS token can be used for decoder_start_token_id
-                if bos_token_id is not None:
-                    decoder_start_token_id = bos_token_id
-                elif hasattr(self.config, "decoder") and hasattr(self.config.decoder, "bos_token_id"):
-                    decoder_start_token_id = self.config.decoder.bos_token_id
-                else:
-                    raise ValueError(
-                        "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation"
-                    )
+                decoder_start_token_id = bos_token_id
 
+            assert (
+                decoder_start_token_id is not None
+            ), "decoder_start_token_id or bos_token_id has to be defined for encoder-decoder generation"
             assert hasattr(self, "get_encoder"), "{} should have a 'get_encoder' function defined".format(self)
             assert callable(self.get_encoder), "{} should be a method".format(self.get_encoder)
 
@@ -881,8 +875,8 @@ def calc_banned_bad_words_ids(prev_input_ids: Iterable[int], bad_words_ids: Iter
         if len(tokens) == 0:
             # if bad word tokens is just one token always ban it
             return True
-        if len(tokens) > len(prev_tokens):
-            # if bad word tokens are longer than prev tokens they can't be equal
+        if len(tokens) > len(prev_input_ids):
+            # if bad word tokens are longer then prev input_ids they can't be equal
             return False
 
         if prev_tokens[-len(tokens) :] == tokens:
@@ -899,7 +893,7 @@ def calc_banned_bad_words_ids(prev_input_ids: Iterable[int], bad_words_ids: Iter
                 bad_words_ids
             )
 
-            if _tokens_match(prev_input_ids_slice, banned_token_seq[:-1]) is False:
+            if _tokens_match(prev_input_ids_slice.tolist(), banned_token_seq[:-1]) is False:
                 # if tokens do not match continue
                 continue
 
@@ -910,30 +904,6 @@ def calc_banned_bad_words_ids(prev_input_ids: Iterable[int], bad_words_ids: Iter
     return banned_tokens
 
 
-def set_scores_to_inf_for_banned_tokens(scores: torch.Tensor, banned_tokens: List[List[int]]) -> None:
-    """ Modifies the scores in place by setting the banned token positions to `-inf`. Banned token is expected to be
-    a list of list of banned tokens to ban in the format [[batch index, vocabulary position],...]
-        Args:
-            scores: logits distribution of shape (batch size, vocabulary size)
-            banned_tokens: list of list of tokens to ban of length (batch_size)
-    """
-    banned_mask_list = []
-    for idx, batch_banned_tokens in enumerate(banned_tokens):
-        for token in batch_banned_tokens:
-            banned_mask_list.append([idx, token])
-    if not banned_mask_list:
-        return
-    banned_mask = torch.LongTensor(banned_mask_list)
-    indices = torch.ones(len(banned_mask))
-    # A sparse tensor is generated from a list of coordinates: [[0, 1], [0, 2], [2, 0]]. A conversion to dense tensor generates:
-    # [ 0  1  1 ]
-    # [ 0  0  0 ]
-    # [ 1  0  0 ]
-
-    banned_mask = torch.sparse.LongTensor(banned_mask.t(), indices, scores.size()).to(scores.device).to_dense().bool()
-    scores.masked_fill_(banned_mask, -float("inf"))
-
-
 def top_k_top_p_filtering(
     logits: Tensor,
     top_k: int = 0,

src/transformers/hf_argparser.py

@@ -4,7 +4,7 @@ import sys
 from argparse import ArgumentParser
 from enum import Enum
 from pathlib import Path
-from typing import Any, Iterable, List, NewType, Optional, Tuple, Union
+from typing import Any, Iterable, List, NewType, Tuple, Union
 
 
 DataClass = NewType("DataClass", Any)
@@ -64,7 +64,7 @@ class HfArgumentParser(ArgumentParser):
                 kwargs["type"] = field.type
                 if field.default is not dataclasses.MISSING:
                     kwargs["default"] = field.default
-            elif field.type is bool or field.type is Optional[bool]:
+            elif field.type is bool:
                 kwargs["action"] = "store_false" if field.default is True else "store_true"
                 if field.default is True:
                     field_name = f"--no-{field.name}"

src/transformers/modeling_albert.py

@@ -212,8 +212,7 @@ class AlbertAttention(BertSelfAttention):
         self.num_attention_heads = config.num_attention_heads
         self.hidden_size = config.hidden_size
         self.attention_head_size = config.hidden_size // config.num_attention_heads
-        self.attention_dropout = nn.Dropout(config.attention_probs_dropout_prob)
-        self.output_dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
         self.dense = nn.Linear(config.hidden_size, config.hidden_size)
         self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
         self.pruned_heads = set()
@@ -257,7 +256,7 @@ class AlbertAttention(BertSelfAttention):
 
         # This is actually dropping out entire tokens to attend to, which might
         # seem a bit unusual, but is taken from the original Transformer paper.
-        attention_probs = self.attention_dropout(attention_probs)
+        attention_probs = self.dropout(attention_probs)
 
         # Mask heads if we want to
         if head_mask is not None:
@@ -276,7 +275,7 @@ class AlbertAttention(BertSelfAttention):
         b = self.dense.bias.to(context_layer.dtype)
 
         projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b
-        projected_context_layer_dropout = self.output_dropout(projected_context_layer)
+        projected_context_layer_dropout = self.dropout(projected_context_layer)
         layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout)
         return (layernormed_context_layer, attention_probs) if output_attentions else (layernormed_context_layer,)
 
@@ -291,7 +290,6 @@ class AlbertLayer(nn.Module):
         self.ffn = nn.Linear(config.hidden_size, config.intermediate_size)
         self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size)
         self.activation = ACT2FN[config.hidden_act]
-        self.dropout = nn.Dropout(config.hidden_dropout_prob)
 
     def forward(
         self, hidden_states, attention_mask=None, head_mask=None, output_attentions=False, output_hidden_states=False
@@ -300,7 +298,6 @@ class AlbertLayer(nn.Module):
         ffn_output = self.ffn(attention_output[0])
         ffn_output = self.activation(ffn_output)
         ffn_output = self.ffn_output(ffn_output)
-        ffn_output = self.dropout(ffn_output)
         hidden_states = self.full_layer_layer_norm(ffn_output + attention_output[0])
 
         return (hidden_states,) + attention_output[1:]  # add attentions if we output them

src/transformers/modeling_auto.py

@@ -32,10 +32,8 @@ from .configuration_auto import (
     FlaubertConfig,
     GPT2Config,
     LongformerConfig,
-    MBartConfig,
     MobileBertConfig,
     OpenAIGPTConfig,
-    PegasusConfig,
     ReformerConfig,
     RetriBertConfig,
     RobertaConfig,
@@ -117,7 +115,6 @@ from .modeling_longformer import (
     LongformerModel,
 )
 from .modeling_marian import MarianMTModel
-from .modeling_mbart import MBartForConditionalGeneration
 from .modeling_mobilebert import (
     MobileBertForMaskedLM,
     MobileBertForMultipleChoice,
@@ -128,7 +125,6 @@ from .modeling_mobilebert import (
     MobileBertModel,
 )
 from .modeling_openai import OpenAIGPTLMHeadModel, OpenAIGPTModel
-from .modeling_pegasus import PegasusForConditionalGeneration
 from .modeling_reformer import (
     ReformerForMaskedLM,
     ReformerForQuestionAnswering,
@@ -137,7 +133,6 @@ from .modeling_reformer import (
 )
 from .modeling_retribert import RetriBertModel
 from .modeling_roberta import (
-    RobertaForCausalLM,
     RobertaForMaskedLM,
     RobertaForMultipleChoice,
     RobertaForQuestionAnswering,
@@ -253,7 +248,6 @@ MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
 
 MODEL_FOR_CAUSAL_LM_MAPPING = OrderedDict(
     [
-        (RobertaConfig, RobertaForCausalLM),
         (BertConfig, BertLMHeadModel),
         (OpenAIGPTConfig, OpenAIGPTLMHeadModel),
         (GPT2Config, GPT2LMHeadModel),
@@ -289,9 +283,7 @@ MODEL_FOR_MASKED_LM_MAPPING = OrderedDict(
 MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING = OrderedDict(
     [
         (T5Config, T5ForConditionalGeneration),
-        (PegasusConfig, PegasusForConditionalGeneration),
         (MarianConfig, MarianMTModel),
-        (MBartConfig, MBartForConditionalGeneration),
         (BartConfig, BartForConditionalGeneration),
         (EncoderDecoderConfig, EncoderDecoderModel),
     ]

src/transformers/modeling_bert.py

@@ -48,12 +48,7 @@ from .modeling_outputs import (
     SequenceClassifierOutput,
     TokenClassifierOutput,
 )
-from .modeling_utils import (
-    PreTrainedModel,
-    apply_chunking_to_forward,
-    find_pruneable_heads_and_indices,
-    prune_linear_layer,
-)
+from .modeling_utils import PreTrainedModel, find_pruneable_heads_and_indices, prune_linear_layer
 
 
 logger = logging.getLogger(__name__)
@@ -93,7 +88,6 @@ def load_tf_weights_in_bert(model, config, tf_checkpoint_path):
     """
     try:
         import re
-
         import numpy as np
         import tensorflow as tf
     except ImportError:
@@ -382,13 +376,9 @@ class BertOutput(nn.Module):
 class BertLayer(nn.Module):
     def __init__(self, config):
         super().__init__()
-        self.chunk_size_feed_forward = config.chunk_size_feed_forward
-        self.seq_len_dim = 1
         self.attention = BertAttention(config)
         self.is_decoder = config.is_decoder
-        self.add_cross_attention = config.add_cross_attention
-        if self.add_cross_attention:
-            assert self.is_decoder, f"{self} should be used as a decoder model if cross attention is added"
+        if self.is_decoder:
             self.crossattention = BertAttention(config)
         self.intermediate = BertIntermediate(config)
         self.output = BertOutput(config)
@@ -409,9 +399,6 @@ class BertLayer(nn.Module):
         outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights
 
         if self.is_decoder and encoder_hidden_states is not None:
-            assert hasattr(
-                self, "crossattention"
-            ), f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`"
             cross_attention_outputs = self.crossattention(
                 attention_output,
                 attention_mask,
@@ -423,16 +410,10 @@ class BertLayer(nn.Module):
             attention_output = cross_attention_outputs[0]
             outputs = outputs + cross_attention_outputs[1:]  # add cross attentions if we output attention weights
 
-        layer_output = apply_chunking_to_forward(
-            self.chunk_size_feed_forward, self.seq_len_dim, self.feed_forward_chunk, attention_output
-        )
-        outputs = (layer_output,) + outputs
-        return outputs
-
-    def feed_forward_chunk(self, attention_output):
         intermediate_output = self.intermediate(attention_output)
         layer_output = self.output(intermediate_output, attention_output)
-        return layer_output
+        outputs = (layer_output,) + outputs
+        return outputs
 
 
 class BertEncoder(nn.Module):
@@ -683,6 +664,14 @@ BERT_INPUTS_DOCSTRING = r"""
             Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation.
             This is useful if you want more control over how to convert `input_ids` indices into associated vectors
             than the model's internal embedding lookup matrix.
+        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`):
+            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
+            if the model is configured as a decoder.
+        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
+            Mask to avoid performing attention on the padding token indices of the encoder input. This mask
+            is used in the cross-attention if the model is configured as a decoder.
+            Mask values selected in ``[0, 1]``:
+            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
         output_attentions (:obj:`bool`, `optional`, defaults to :obj:`None`):
             If set to ``True``, the attentions tensors of all attention layers are returned. See ``attentions`` under returned tensors for more detail.
         output_hidden_states (:obj:`bool`, `optional`, defaults to :obj:`None`):
@@ -706,10 +695,8 @@ class BertModel(BertPreTrainedModel):
     Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
 
     To behave as an decoder the model needs to be initialized with the
-    :obj:`is_decoder` argument of the configuration set to :obj:`True`.
-    To be used in a Seq2Seq model, the model needs to initialized with both :obj:`is_decoder`
-    argument and :obj:`add_cross_attention` set to :obj:`True`; an
-    :obj:`encoder_hidden_states` is then expected as an input to the forward pass.
+    :obj:`is_decoder` argument of the configuration set to :obj:`True`; an
+    :obj:`encoder_hidden_states` is expected as an input to the forward pass.
 
     .. _`Attention is all you need`:
         https://arxiv.org/abs/1706.03762
@@ -761,16 +748,6 @@ class BertModel(BertPreTrainedModel):
         output_hidden_states=None,
         return_dict=None,
     ):
-        r"""
-        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`):
-            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
-            if the model is configured as a decoder.
-        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
-            Mask to avoid performing attention on the padding token indices of the encoder input. This mask
-            is used in the cross-attention if the model is configured as a decoder.
-            Mask values selected in ``[0, 1]``:
-            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
-        """
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
             output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
@@ -956,9 +933,7 @@ class BertForPreTraining(BertPreTrainedModel):
 class BertLMHeadModel(BertPreTrainedModel):
     def __init__(self, config):
         super().__init__(config)
-
-        if not config.is_decoder:
-            logger.warning("If you want to use `BertLMHeadModel` as a standalone, add `is_decoder=True.`")
+        assert config.is_decoder, "If you want to use `BertLMHeadModel` as a standalone, add `is_decoder=True`."
 
         self.bert = BertModel(config)
         self.cls = BertOnlyMLMHead(config)
@@ -978,27 +953,22 @@ class BertLMHeadModel(BertPreTrainedModel):
         position_ids=None,
         head_mask=None,
         inputs_embeds=None,
+        labels=None,
         encoder_hidden_states=None,
         encoder_attention_mask=None,
-        labels=None,
         output_attentions=None,
         output_hidden_states=None,
         return_dict=None,
+        **kwargs
     ):
         r"""
-        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`):
-            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
-            if the model is configured as a decoder.
-        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
-            Mask to avoid performing attention on the padding token indices of the encoder input. This mask
-            is used in the cross-attention if the model is configured as a decoder.
-            Mask values selected in ``[0, 1]``:
-            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
         labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
             Labels for computing the left-to-right language modeling loss (next word prediction).
             Indices should be in ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
             Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens with labels
             in ``[0, ..., config.vocab_size]``
+        kwargs (:obj:`Dict[str, any]`, optional, defaults to `{}`):
+            Used to hide legacy arguments that have been deprecated.
 
     Returns:
 
@@ -1066,12 +1036,9 @@ class BertLMHeadModel(BertPreTrainedModel):
 class BertForMaskedLM(BertPreTrainedModel):
     def __init__(self, config):
         super().__init__(config)
-
-        if config.is_decoder:
-            logger.warning(
-                "If you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for "
-                "bi-directional self-attention."
-            )
+        assert (
+            not config.is_decoder
+        ), "If you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention."
 
         self.bert = BertModel(config)
         self.cls = BertOnlyMLMHead(config)
@@ -1096,9 +1063,9 @@ class BertForMaskedLM(BertPreTrainedModel):
         position_ids=None,
         head_mask=None,
         inputs_embeds=None,
+        labels=None,
         encoder_hidden_states=None,
         encoder_attention_mask=None,
-        labels=None,
         output_attentions=None,
         output_hidden_states=None,
         return_dict=None,

src/transformers/modeling_encoder_decoder.py

@@ -168,18 +168,17 @@ class EncoderDecoderModel(PreTrainedModel):
                 from .configuration_auto import AutoConfig
 
                 decoder_config = AutoConfig.from_pretrained(decoder_pretrained_model_name_or_path)
-                if decoder_config.is_decoder is False or decoder_config.add_cross_attention is False:
+                if decoder_config.is_decoder is False:
                     logger.info(
                         f"Initializing {decoder_pretrained_model_name_or_path} as a decoder model. Cross attention layers are added to {decoder_pretrained_model_name_or_path} and randomly initialized if {decoder_pretrained_model_name_or_path}'s architecture allows for cross attention layers."
                     )
                     decoder_config.is_decoder = True
-                    decoder_config.add_cross_attention = True
 
                 kwargs_decoder["config"] = decoder_config
 
-            if kwargs_decoder["config"].is_decoder is False or decoder_config.add_cross_attention is False:
+            if kwargs_decoder["config"].is_decoder is False:
                 logger.warning(
-                    f"Decoder model {decoder_pretrained_model_name_or_path} is not initialized as a decoder. In order to initialize {decoder_pretrained_model_name_or_path} as a decoder, make sure that the attributes `is_decoder` and `add_cross_attention` of `decoder_config` passed to `.from_encoder_decoder_pretrained(...)` are set to `True` or do not pass a `decoder_config` to `.from_encoder_decoder_pretrained(...)`"
+                    f"Decoder model {decoder_pretrained_model_name_or_path} is not initialized as a decoder. In order to initialize {decoder_pretrained_model_name_or_path} as a decoder, make sure that the attribute `is_decoder` of `decoder_config` passed to `.from_encoder_decoder_pretrained(...)` is set to `True` or do not pass a `decoder_config` to `.from_encoder_decoder_pretrained(...)`"
                 )
 
             decoder = AutoModelForCausalLM.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs_decoder)
@@ -191,9 +190,11 @@ class EncoderDecoderModel(PreTrainedModel):
         input_ids=None,
         inputs_embeds=None,
         attention_mask=None,
+        head_mask=None,
         encoder_outputs=None,
         decoder_input_ids=None,
         decoder_attention_mask=None,
+        decoder_head_mask=None,
         decoder_inputs_embeds=None,
         labels=None,
         **kwargs,
@@ -214,6 +215,10 @@ class EncoderDecoderModel(PreTrainedModel):
                 Mask to avoid performing attention on padding token indices for the encoder.
                 Mask values selected in ``[0, 1]``:
                 ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
+            head_mask: (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`, defaults to :obj:`None`):
+                Mask to nullify selected heads of the self-attention modules for the encoder.
+                Mask values selected in ``[0, 1]``:
+                ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
             encoder_outputs (:obj:`tuple(tuple(torch.FloatTensor)`, `optional`, defaults to :obj:`None`):
                 Tuple consists of (`last_hidden_state`, `optional`: `hidden_states`, `optional`: `attentions`)
                 `last_hidden_state` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`) is a sequence of hidden-states at the output of the last layer of the encoder.
@@ -225,6 +230,10 @@ class EncoderDecoderModel(PreTrainedModel):
                 :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details.
             decoder_attention_mask (:obj:`torch.BoolTensor` of shape :obj:`(batch_size, tgt_seq_len)`, `optional`, defaults to :obj:`None`):
                 Default behavior: generate a tensor that ignores pad tokens in decoder_input_ids. Causal mask will also be used by default.
+            decoder_head_mask: (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`, defaults to :obj:`None`):
+                Mask to nullify selected heads of the self-attention modules for the decoder.
+                Mask values selected in ``[0, 1]``:
+                ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
             decoder_inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, target_sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`):
                 Optionally, instead of passing :obj:`decoder_input_ids` you can choose to directly pass an embedded representation.
                 This is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors
@@ -269,6 +278,7 @@ class EncoderDecoderModel(PreTrainedModel):
                 input_ids=input_ids,
                 attention_mask=attention_mask,
                 inputs_embeds=inputs_embeds,
+                head_mask=head_mask,
                 return_dict=False,
                 **kwargs_encoder,
             )
@@ -282,13 +292,12 @@ class EncoderDecoderModel(PreTrainedModel):
             attention_mask=decoder_attention_mask,
             encoder_hidden_states=hidden_states,
             encoder_attention_mask=attention_mask,
+            head_mask=decoder_head_mask,
             labels=labels,
             return_dict=False,
             **kwargs_decoder,
         )
 
-        # TODO(PVP): currently it is not possible to use `past`
-        # with the encoder/decoder framework -> should be implemented
         return decoder_outputs + encoder_outputs
 
     def prepare_inputs_for_generation(self, input_ids, past, attention_mask, **kwargs):
@@ -301,24 +310,15 @@ class EncoderDecoderModel(PreTrainedModel):
             encoder_outputs = (past,)
 
         decoder_inputs = self.decoder.prepare_inputs_for_generation(input_ids)
-        decoder_attention_mask = decoder_inputs["attention_mask"] if "attention_mask" in decoder_inputs else None
-        input_dict = {
+
+        return {
             "attention_mask": attention_mask,
-            "decoder_attention_mask": decoder_attention_mask,
+            "decoder_attention_mask": decoder_inputs["attention_mask"],
             "decoder_input_ids": decoder_inputs["input_ids"],
             "encoder_outputs": encoder_outputs,
         }
 
-        # Ideally all models should have a `use_cache`
-        # leave following to ifs until all have it implemented
-        if "use_cache" in decoder_inputs:
-            input_dict["decoder_use_cache"] = decoder_inputs["use_cache"]
-
-        if "past_key_values" in decoder_inputs:
-            input_dict["decoder_past_key_values"] = decoder_inputs["past_key_values"]
-
-        return input_dict
-
     def _reorder_cache(self, past, beam_idx):
-        # apply decoder cache reordering here
-        return self.decoder._reorder_cache(past, beam_idx)
+        # as a default encoder-decoder models do not re-order the past.
+        # TODO(PVP): might have to be updated, e.g. if GPT2 is to be used as a decoder
+        return past

src/transformers/modeling_gpt2.py

@@ -118,7 +118,7 @@ def load_tf_weights_in_gpt2(model, config, gpt2_checkpoint_path):
 
 
 class Attention(nn.Module):
-    def __init__(self, nx, n_ctx, config, scale=False, is_cross_attention=False):
+    def __init__(self, nx, n_ctx, config, scale=False):
         super().__init__()
 
         n_state = nx  # in Attention: n_state=768 (nx=n_embd)
@@ -131,12 +131,8 @@ class Attention(nn.Module):
         self.n_head = config.n_head
         self.split_size = n_state
         self.scale = scale
-        self.is_cross_attention = is_cross_attention
-        if self.is_cross_attention:
-            self.c_attn = Conv1D(2 * n_state, nx)
-            self.q_attn = Conv1D(n_state, nx)
-        else:
-            self.c_attn = Conv1D(3 * n_state, nx)
+
+        self.c_attn = Conv1D(n_state * 3, nx)
         self.c_proj = Conv1D(n_state, nx)
         self.attn_dropout = nn.Dropout(config.attn_pdrop)
         self.resid_dropout = nn.Dropout(config.resid_pdrop)
@@ -164,11 +160,8 @@ class Attention(nn.Module):
         if self.scale:
             w = w / (float(v.size(-1)) ** 0.5)
         nd, ns = w.size(-2), w.size(-1)
-
-        if not self.is_cross_attention:
-            # if only "normal" attention layer implements causal mask
-            mask = self.bias[:, :, ns - nd : ns, :ns]
-            w = torch.where(mask.bool(), w, self.masked_bias.to(w.dtype))
+        mask = self.bias[:, :, ns - nd : ns, :ns]
+        w = torch.where(mask.bool(), w, self.masked_bias.to(w.dtype))
 
         if attention_mask is not None:
             # Apply the attention mask
@@ -200,26 +193,10 @@ class Attention(nn.Module):
             return x.permute(0, 2, 1, 3)  # (batch, head, seq_length, head_features)
 
     def forward(
-        self,
-        hidden_states,
-        layer_past=None,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        use_cache=False,
-        output_attentions=False,
+        self, x, layer_past=None, attention_mask=None, head_mask=None, use_cache=False, output_attentions=False
     ):
-        if encoder_hidden_states is not None:
-            assert hasattr(
-                self, "q_attn"
-            ), "If class is used as cross attention, the weights `q_attn` have to be defined. Please make sure to instantiate class with `Attention(..., is_cross_attention=True)`."
-            query = self.q_attn(hidden_states)
-            key, value = self.c_attn(encoder_hidden_states).split(self.split_size, dim=2)
-            attention_mask = encoder_attention_mask
-        else:
-            query, key, value = self.c_attn(hidden_states).split(self.split_size, dim=2)
-
+        x = self.c_attn(x)
+        query, key, value = x.split(self.split_size, dim=2)
         query = self.split_heads(query)
         key = self.split_heads(key, k=True)
         value = self.split_heads(value)
@@ -262,64 +239,32 @@ class MLP(nn.Module):
 class Block(nn.Module):
     def __init__(self, n_ctx, config, scale=False):
         super().__init__()
-        hidden_size = config.n_embd
-        inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
-        self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
-        self.attn = Attention(hidden_size, n_ctx, config, scale)
-        self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
-        if config.add_cross_attention:
-            self.crossattention = Attention(hidden_size, n_ctx, config, scale, is_cross_attention=True)
-            self.ln_cross_attn = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
+        nx = config.n_embd
+        inner_dim = config.n_inner if config.n_inner is not None else 4 * nx
+        self.ln_1 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon)
+        self.attn = Attention(nx, n_ctx, config, scale)
+        self.ln_2 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon)
         self.mlp = MLP(inner_dim, config)
 
     def forward(
-        self,
-        hidden_states,
-        layer_past=None,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        use_cache=False,
-        output_attentions=False,
+        self, x, layer_past=None, attention_mask=None, head_mask=None, use_cache=False, output_attentions=False,
     ):
-        attn_outputs = self.attn(
-            self.ln_1(hidden_states),
+        output_attn = self.attn(
+            self.ln_1(x),
             layer_past=layer_past,
             attention_mask=attention_mask,
             head_mask=head_mask,
             use_cache=use_cache,
             output_attentions=output_attentions,
         )
-        attn_output = attn_outputs[0]  # output_attn: a, present, (attentions)
-        outputs = attn_outputs[1:]
-        # residual connection
-        hidden_states = attn_output + hidden_states
+        a = output_attn[0]  # output_attn: a, present, (attentions)
 
-        if encoder_hidden_states is not None:
-            # add one self-attention block for cross-attention
-            assert hasattr(
-                self, "crossattention"
-            ), f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`"
-            cross_attn_outputs = self.crossattention(
-                self.ln_cross_attn(hidden_states),
-                attention_mask=attention_mask,
-                head_mask=head_mask,
-                encoder_hidden_states=encoder_hidden_states,
-                encoder_attention_mask=encoder_attention_mask,
-                output_attentions=output_attentions,
-            )
-            attn_output = cross_attn_outputs[0]
-            # residual connection
-            hidden_states = hidden_states + attn_output
-            outputs = outputs + cross_attn_outputs[1:]  # add cross attentions if we output attention weights
+        x = x + a
+        m = self.mlp(self.ln_2(x))
+        x = x + m
 
-        feed_forward_hidden_states = self.mlp(self.ln_2(hidden_states))
-        # residual connection
-        hidden_states = hidden_states + feed_forward_hidden_states
-
-        outputs = [hidden_states] + outputs
-        return outputs  # hidden_states, present, (cross_attentions, attentions)
+        outputs = [x] + output_attn[1:]
+        return outputs  # x, present, (attentions)
 
 
 class GPT2PreTrainedModel(PreTrainedModel):
@@ -504,8 +449,6 @@ class GPT2Model(GPT2PreTrainedModel):
         position_ids=None,
         head_mask=None,
         inputs_embeds=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
         use_cache=None,
         output_attentions=None,
         output_hidden_states=None,
@@ -563,7 +506,7 @@ class GPT2Model(GPT2PreTrainedModel):
             # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
             # this attention mask is more simple than the triangular masking of causal attention
             # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
-            attention_mask = attention_mask[:, None, None, :]
+            attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
 
             # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
             # masked positions, this operation will create a tensor which is 0.0 for
@@ -573,17 +516,6 @@ class GPT2Model(GPT2PreTrainedModel):
             attention_mask = attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
             attention_mask = (1.0 - attention_mask) * -10000.0
 
-        # If a 2D ou 3D attention mask is provided for the cross-attention
-        # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length]
-        if self.config.add_cross_attention and encoder_hidden_states is not None:
-            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
-            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
-            if encoder_attention_mask is None:
-                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
-            encoder_attention_mask = self.invert_attention_mask(encoder_attention_mask)
-        else:
-            encoder_attention_mask = None
-
         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
         # attention_probs has shape bsz x n_heads x N x N
@@ -614,8 +546,6 @@ class GPT2Model(GPT2PreTrainedModel):
                 layer_past=layer_past,
                 attention_mask=attention_mask,
                 head_mask=head_mask[i],
-                encoder_hidden_states=encoder_hidden_states,
-                encoder_attention_mask=encoder_attention_mask,
                 use_cache=use_cache,
                 output_attentions=output_attentions,
             )
@@ -663,21 +593,17 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
     def get_output_embeddings(self):
         return self.lm_head
 
-    def prepare_inputs_for_generation(self, input_ids, past=None, **kwargs):
+    def prepare_inputs_for_generation(self, input_ids, past, **kwargs):
         # only last token for inputs_ids if past is defined in kwargs
         if past:
             input_ids = input_ids[:, -1].unsqueeze(-1)
 
-        return {
-            "input_ids": input_ids,
-            "past_key_values": past,
-            "use_cache": kwargs.get("use_cache"),
-        }
+        return {"input_ids": input_ids, "past_key_values": past, "use_cache": kwargs["use_cache"]}
 
     @add_start_docstrings_to_callable(GPT2_INPUTS_DOCSTRING)
     @add_code_sample_docstrings(
         tokenizer_class=_TOKENIZER_FOR_DOC,
-        checkpoint="gpt2",
+        checkpoint="ctrl",
         output_type=CausalLMOutputWithPast,
         config_class=_CONFIG_FOR_DOC,
     )
@@ -690,8 +616,6 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
         position_ids=None,
         head_mask=None,
         inputs_embeds=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
         labels=None,
         use_cache=None,
         output_attentions=None,
@@ -724,8 +648,6 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
             position_ids=position_ids,
             head_mask=head_mask,
             inputs_embeds=inputs_embeds,
-            encoder_hidden_states=encoder_hidden_states,
-            encoder_attention_mask=encoder_attention_mask,
             use_cache=use_cache,
             output_attentions=output_attentions,
             output_hidden_states=output_hidden_states,

src/transformers/modeling_longformer.py

@@ -71,6 +71,7 @@ def _get_question_end_index(input_ids, sep_token_id):
     assert (
         sep_token_indices.shape[0] == 3 * batch_size
     ), f"There should be exactly three separator tokens: {sep_token_id} in every sample for questions answering. You might also consider to set `global_attention_mask` manually in the forward function to avoid this error."
+
     return sep_token_indices.view(batch_size, 3, 2)[:, 0, 1]
 
 
@@ -80,6 +81,7 @@ def _compute_global_attention_mask(input_ids, sep_token_id, before_sep_token=Tru
         before `sep_token_id` if `before_sep_token is True` else after
         `sep_token_id`.
     """
+
     question_end_index = _get_question_end_index(input_ids, sep_token_id)
     question_end_index = question_end_index.unsqueeze(dim=1)  # size: batch_size x 1
     # bool attention mask with True in locations of global attention
@@ -129,172 +131,6 @@ class LongformerSelfAttention(nn.Module):
 
         self.one_sided_attn_window_size = attention_window // 2
 
-    def forward(
-        self, hidden_states, attention_mask=None, output_attentions=False,
-    ):
-        """
-        LongformerSelfAttention expects `len(hidden_states)` to be multiple of `attention_window`.
-        Padding to `attention_window` happens in LongformerModel.forward to avoid redoing the padding on each layer.
-
-        The `attention_mask` is changed in `BertModel.forward` from 0, 1, 2 to
-            -ve: no attention
-              0: local attention
-            +ve: global attention
-
-        """
-        attention_mask = attention_mask.squeeze(dim=2).squeeze(dim=1)
-
-        # is index masked or global attention
-        is_index_masked = attention_mask < 0
-        is_index_global_attn = attention_mask > 0
-        is_global_attn = is_index_global_attn.flatten().any().item()
-
-        hidden_states = hidden_states.transpose(0, 1)
-
-        # project hidden states
-        query_vectors = self.query(hidden_states)
-        key_vectors = self.key(hidden_states)
-        value_vectors = self.value(hidden_states)
-
-        seq_len, batch_size, embed_dim = hidden_states.size()
-        assert (
-            embed_dim == self.embed_dim
-        ), f"hidden_states should have embed_dim = {self.embed_dim}, but has {embed_dim}"
-
-        # normalize query
-        query_vectors /= math.sqrt(self.head_dim)
-
-        query_vectors = query_vectors.view(seq_len, batch_size, self.num_heads, self.head_dim).transpose(0, 1)
-        key_vectors = key_vectors.view(seq_len, batch_size, self.num_heads, self.head_dim).transpose(0, 1)
-
-        # attn_probs = (batch_size, seq_len, num_heads, window*2+1)
-        attn_scores = self._sliding_chunks_query_key_matmul(
-            query_vectors, key_vectors, self.one_sided_attn_window_size
-        )
-
-        # values to pad for attention probs
-        remove_from_windowed_attention_mask = (attention_mask != 0)[:, :, None, None]
-
-        # cast to fp32/fp16 then replace 1's with -inf
-        float_mask = remove_from_windowed_attention_mask.type_as(query_vectors).masked_fill(
-            remove_from_windowed_attention_mask, -10000.0
-        )
-        # diagonal mask with zeros everywhere and -inf inplace of padding
-        diagonal_mask = self._sliding_chunks_query_key_matmul(
-            float_mask.new_ones(size=float_mask.size()), float_mask, self.one_sided_attn_window_size
-        )
-
-        # pad local attention probs
-        attn_scores += diagonal_mask
-
-        assert list(attn_scores.size()) == [
-            batch_size,
-            seq_len,
-            self.num_heads,
-            self.one_sided_attn_window_size * 2 + 1,
-        ], f"attn_probs should be of size ({batch_size}, {seq_len}, {self.num_heads}, {self.one_sided_attn_window_size * 2 + 1}), but is of size {attn_scores.size()}"
-
-        # compute local attention probs from global attention keys and contact over window dim
-        if is_global_attn:
-            # compute global attn indices required through out forward fn
-            (
-                max_num_global_attn_indices,
-                is_index_global_attn_nonzero,
-                is_local_index_global_attn_nonzero,
-                is_local_index_no_global_attn_nonzero,
-            ) = self._get_global_attn_indices(is_index_global_attn)
-            # calculate global attn probs from global key
-
-            global_key_attn_scores = self._concat_with_global_key_attn_probs(
-                query_vectors=query_vectors,
-                key_vectors=key_vectors,
-                max_num_global_attn_indices=max_num_global_attn_indices,
-                is_index_global_attn_nonzero=is_index_global_attn_nonzero,
-                is_local_index_global_attn_nonzero=is_local_index_global_attn_nonzero,
-                is_local_index_no_global_attn_nonzero=is_local_index_no_global_attn_nonzero,
-            )
-            # concat to attn_probs
-            # (batch_size, seq_len, num_heads, extra attention count + 2*window+1)
-            attn_scores = torch.cat((global_key_attn_scores, attn_scores), dim=-1)
-
-            # free memory
-            del global_key_attn_scores
-
-        attn_probs_fp32 = F.softmax(attn_scores, dim=-1, dtype=torch.float32)  # use fp32 for numerical stability
-        attn_probs = attn_probs_fp32.type_as(attn_scores)
-
-        # free memory
-        del attn_probs_fp32
-
-        # softmax sometimes inserts NaN if all positions are masked, replace them with 0
-        attn_probs = torch.masked_fill(attn_probs, is_index_masked[:, :, None, None], 0.0)
-
-        # apply dropout
-        attn_probs = F.dropout(attn_probs, p=self.dropout, training=self.training)
-
-        value_vectors = value_vectors.view(seq_len, batch_size, self.num_heads, self.head_dim).transpose(0, 1)
-
-        # compute local attention output with global attention value and add
-        if is_global_attn:
-            # compute sum of global and local attn
-            attn_output = self._compute_attn_output_with_global_indices(
-                value_vectors=value_vectors,
-                attn_probs=attn_probs,
-                max_num_global_attn_indices=max_num_global_attn_indices,
-                is_index_global_attn_nonzero=is_index_global_attn_nonzero,
-                is_local_index_global_attn_nonzero=is_local_index_global_attn_nonzero,
-            )
-        else:
-            # compute local attn only
-            attn_output = self._sliding_chunks_matmul_attn_probs_value(
-                attn_probs, value_vectors, self.one_sided_attn_window_size
-            )
-
-        assert attn_output.size() == (batch_size, seq_len, self.num_heads, self.head_dim), "Unexpected size"
-        attn_output = attn_output.transpose(0, 1).reshape(seq_len, batch_size, embed_dim).contiguous()
-
-        # compute value for global attention and overwrite to attention output
-        # TODO: remove the redundant computation
-        if is_global_attn:
-            global_attn_output = self._compute_global_attn_output_from_hidden(
-                hidden_states=hidden_states,
-                max_num_global_attn_indices=max_num_global_attn_indices,
-                is_local_index_global_attn_nonzero=is_local_index_global_attn_nonzero,
-                is_index_global_attn_nonzero=is_index_global_attn_nonzero,
-                is_local_index_no_global_attn_nonzero=is_local_index_no_global_attn_nonzero,
-                is_index_masked=is_index_masked,
-            )
-
-            # get only non zero global attn output
-            nonzero_global_attn_output = global_attn_output[
-                is_local_index_global_attn_nonzero[0], :, is_local_index_global_attn_nonzero[1]
-            ]
-
-            # overwrite values with global attention
-            attn_output[is_index_global_attn_nonzero[::-1]] = nonzero_global_attn_output.view(
-                len(is_local_index_global_attn_nonzero[0]), -1
-            )
-
-        attn_output = attn_output.transpose(0, 1)
-
-        if output_attentions:
-            if is_global_attn:
-                # With global attention, return global attention probabilities only
-                # batch_size x num_heads x max_num_global_attention_tokens x sequence_length
-                # which is the attention weights from tokens with global attention to all tokens
-                # It doesn't not return local attention
-                # In case of variable number of global attantion in the rows of a batch,
-                # attn_probs are padded with -10000.0 attention scores
-                attn_probs = attn_probs.view(batch_size, self.num_heads, max_num_global_attn_indices, seq_len)
-            else:
-                # without global attention, return local attention probabilities
-                # batch_size x num_heads x sequence_length x window_size
-                # which is the attention weights of every token attending to its neighbours
-                attn_probs = attn_probs.permute(0, 2, 1, 3)
-
-        outputs = (attn_output, attn_probs) if output_attentions else (attn_output,)
-        return outputs
-
     @staticmethod
     def _pad_and_transpose_last_two_dims(hidden_states_padded, padding):
         """pads rows and then flips rows and columns"""
@@ -307,20 +143,8 @@ class LongformerSelfAttention(nn.Module):
         return hidden_states_padded
 
     @staticmethod
-    def _pad_and_diagonalize(chunked_hidden_states):
-        """shift every row 1 step right, converting columns into diagonals.
-           Example:
-                 chunked_hidden_states: [ 0.4983,  2.6918, -0.0071,  1.0492,
-                                          -1.8348,  0.7672,  0.2986,  0.0285,
-                                          -0.7584,  0.4206, -0.0405,  0.1599,
-                                          2.0514, -1.1600,  0.5372,  0.2629 ]
-                 window_overlap = num_rows = 4
-                (pad & diagonilize) =>
-                [ 0.4983,  2.6918, -0.0071,  1.0492, 0.0000,  0.0000,  0.0000
-                  0.0000,  -1.8348,  0.7672,  0.2986,  0.0285, 0.0000,  0.0000
-                  0.0000,  0.0000, -0.7584,  0.4206, -0.0405,  0.1599, 0.0000
-                  0.0000,  0.0000,  0.0000, 2.0514, -1.1600,  0.5372,  0.2629 ]
-        """
+    def _pad_by_window_overlap_except_last_row(chunked_hidden_states):
+        """shift every row 1 step right, converting columns into diagonals"""
         total_num_heads, num_chunks, window_overlap, hidden_dim = chunked_hidden_states.size()
         chunked_hidden_states = F.pad(
             chunked_hidden_states, (0, window_overlap + 1)
@@ -357,8 +181,7 @@ class LongformerSelfAttention(nn.Module):
         chunk_stride[1] = chunk_stride[1] // 2
         return hidden_states.as_strided(size=chunk_size, stride=chunk_stride)
 
-    @staticmethod
-    def _mask_invalid_locations(input_tensor, affected_seq_len) -> torch.Tensor:
+    def _mask_invalid_locations(self, input_tensor, affected_seq_len) -> torch.Tensor:
         beginning_mask_2d = input_tensor.new_ones(affected_seq_len, affected_seq_len + 1).tril().flip(dims=[0])
         beginning_mask = beginning_mask_2d[None, :, None, :]
         ending_mask = beginning_mask.flip(dims=(1, 3))
@@ -420,7 +243,6 @@ class LongformerSelfAttention(nn.Module):
         diagonal_attention_scores[:, 1:, :, :window_overlap] = diagonal_chunked_attention_scores[
             :, :, -(window_overlap + 1) : -1, window_overlap + 1 :
         ]
-
         diagonal_attention_scores[:, 0, 1:window_overlap, 1:window_overlap] = diagonal_chunked_attention_scores[
             :, 0, : window_overlap - 1, 1 - window_overlap :
         ]
@@ -439,13 +261,11 @@ class LongformerSelfAttention(nn.Module):
         """Same as _sliding_chunks_query_key_matmul but for attn_probs and value tensors.
            Returned tensor will be of the same shape as `attn_probs`"""
         batch_size, seq_len, num_heads, head_dim = value.size()
-
         assert seq_len % (window_overlap * 2) == 0
         assert attn_probs.size()[:3] == value.size()[:3]
         assert attn_probs.size(3) == 2 * window_overlap + 1
         chunks_count = seq_len // window_overlap - 1
         # group batch_size and num_heads dimensions into one, then chunk seq_len into chunks of size 2 window overlap
-
         chunked_attn_probs = attn_probs.transpose(1, 2).reshape(
             batch_size * num_heads, seq_len // window_overlap, window_overlap, 2 * window_overlap + 1
         )
@@ -467,11 +287,178 @@ class LongformerSelfAttention(nn.Module):
         )
         chunked_value = padded_value.as_strided(size=chunked_value_size, stride=chunked_value_stride)
 
-        chunked_attn_probs = self._pad_and_diagonalize(chunked_attn_probs)
+        chunked_attn_probs = self._pad_by_window_overlap_except_last_row(chunked_attn_probs)
 
         context = torch.einsum("bcwd,bcdh->bcwh", (chunked_attn_probs, chunked_value))
         return context.view(batch_size, num_heads, seq_len, head_dim).transpose(1, 2)
 
+    def forward(
+        self, hidden_states, attention_mask=None, output_attentions=False,
+    ):
+        """
+        LongformerSelfAttention expects `len(hidden_states)` to be multiple of `attention_window`.
+        Padding to `attention_window` happens in LongformerModel.forward to avoid redoing the padding on each layer.
+
+        The `attention_mask` is changed in `BertModel.forward` from 0, 1, 2 to
+            -ve: no attention
+              0: local attention
+            +ve: global attention
+
+        """
+
+        attention_mask = attention_mask.squeeze(dim=2).squeeze(dim=1)
+
+        # is index masked or global attention
+        is_index_masked = attention_mask < 0
+        is_index_global_attn = attention_mask > 0
+        is_global_attn = is_index_global_attn.flatten().any().item()
+
+        hidden_states = hidden_states.transpose(0, 1)
+
+        # project hidden states
+        query_vectors = self.query(hidden_states)
+        key_vectors = self.key(hidden_states)
+        value_vectors = self.value(hidden_states)
+
+        seq_len, batch_size, embed_dim = hidden_states.size()
+        assert (
+            embed_dim == self.embed_dim
+        ), f"hidden_states should have embed_dim = {self.embed_dim}, but has {embed_dim}"
+
+        # normalize query
+        query_vectors /= math.sqrt(self.head_dim)
+
+        query_vectors = query_vectors.view(seq_len, batch_size, self.num_heads, self.head_dim).transpose(0, 1)
+        key_vectors = key_vectors.view(seq_len, batch_size, self.num_heads, self.head_dim).transpose(0, 1)
+
+        # attn_probs = (batch_size, seq_len, num_heads, window*2+1)
+        attn_scores = self._sliding_chunks_query_key_matmul(
+            query_vectors, key_vectors, self.one_sided_attn_window_size
+        )
+
+        # values to pad for attention probs
+        remove_from_windowed_attention_mask = (attention_mask != 0).unsqueeze(dim=-1).unsqueeze(dim=-1)
+
+        # cast to fp32/fp16 then replace 1's with -inf
+        float_mask = remove_from_windowed_attention_mask.type_as(query_vectors).masked_fill(
+            remove_from_windowed_attention_mask, -10000.0
+        )
+        # diagonal mask with zeros everywhere and -inf inplace of padding
+        diagonal_mask = self._sliding_chunks_query_key_matmul(
+            float_mask.new_ones(size=float_mask.size()), float_mask, self.one_sided_attn_window_size
+        )
+
+        # pad local attention probs
+        attn_scores += diagonal_mask
+
+        assert list(attn_scores.size()) == [
+            batch_size,
+            seq_len,
+            self.num_heads,
+            self.one_sided_attn_window_size * 2 + 1,
+        ], f"attn_probs should be of size ({batch_size}, {seq_len}, {self.num_heads}, {self.one_sided_attn_window_size * 2 + 1}), but is of size {attn_scores.size()}"
+
+        # compute local attention probs from global attention keys and contact over window dim
+        if is_global_attn:
+            # compute global attn indices required through out forward fn
+            (
+                max_num_global_attn_indices,
+                is_index_global_attn_nonzero,
+                is_local_index_global_attn_nonzero,
+                is_local_index_no_global_attn_nonzero,
+            ) = self._get_global_attn_indices(is_index_global_attn)
+            # calculate global attn probs from global key
+            global_key_attn_scores = self._concat_with_global_key_attn_probs(
+                query_vectors=query_vectors,
+                key_vectors=key_vectors,
+                max_num_global_attn_indices=max_num_global_attn_indices,
+                is_index_global_attn_nonzero=is_index_global_attn_nonzero,
+                is_local_index_global_attn_nonzero=is_local_index_global_attn_nonzero,
+                is_local_index_no_global_attn_nonzero=is_local_index_no_global_attn_nonzero,
+            )
+            # concat to attn_probs
+            # (batch_size, seq_len, num_heads, extra attention count + 2*window+1)
+            attn_scores = torch.cat((global_key_attn_scores, attn_scores), dim=-1)
+
+            # free memory
+            del global_key_attn_scores
+
+        attn_probs_fp32 = F.softmax(attn_scores, dim=-1, dtype=torch.float32)  # use fp32 for numerical stability
+        attn_probs = attn_probs_fp32.type_as(attn_scores)
+
+        # free memory
+        del attn_probs_fp32
+
+        # softmax sometimes inserts NaN if all positions are masked, replace them with 0
+        attn_probs = torch.masked_fill(attn_probs, is_index_masked.unsqueeze(-1).unsqueeze(-1), 0.0)
+
+        # apply dropout
+        attn_probs = F.dropout(attn_probs, p=self.dropout, training=self.training)
+
+        value_vectors = value_vectors.view(seq_len, batch_size, self.num_heads, self.head_dim).transpose(0, 1)
+
+        # compute local attention output with global attention value and add
+        if is_global_attn:
+            # compute sum of global and local attn
+            attn_output = self._compute_attn_output_with_global_indices(
+                value_vectors=value_vectors,
+                attn_probs=attn_probs,
+                max_num_global_attn_indices=max_num_global_attn_indices,
+                is_index_global_attn_nonzero=is_index_global_attn_nonzero,
+                is_local_index_global_attn_nonzero=is_local_index_global_attn_nonzero,
+            )
+        else:
+            # compute local attn only
+            attn_output = self._sliding_chunks_matmul_attn_probs_value(
+                attn_probs, value_vectors, self.one_sided_attn_window_size
+            )
+
+        assert attn_output.size() == (batch_size, seq_len, self.num_heads, self.head_dim), "Unexpected size"
+        attn_output = attn_output.transpose(0, 1).reshape(seq_len, batch_size, embed_dim).contiguous()
+
+        # compute value for global attention and overwrite to attention output
+        # TODO: remove the redundant computation
+        if is_global_attn:
+            global_attn_output = self._compute_global_attn_output_from_hidden(
+                hidden_states=hidden_states,
+                max_num_global_attn_indices=max_num_global_attn_indices,
+                is_local_index_global_attn_nonzero=is_local_index_global_attn_nonzero,
+                is_index_global_attn_nonzero=is_index_global_attn_nonzero,
+                is_local_index_no_global_attn_nonzero=is_local_index_no_global_attn_nonzero,
+                is_index_masked=is_index_masked,
+            )
+
+            # get only non zero global attn output
+            nonzero_global_attn_output = global_attn_output[
+                is_local_index_global_attn_nonzero[0], :, is_local_index_global_attn_nonzero[1]
+            ]
+            # overwrite values with global attention
+            attn_output[is_index_global_attn_nonzero[::-1]] = nonzero_global_attn_output.view(
+                len(is_local_index_global_attn_nonzero[0]), -1
+            )
+
+        attn_output = attn_output.transpose(0, 1)
+
+        if output_attentions:
+            if is_global_attn:
+                # With global attention, return global attention probabilities only
+                # batch_size x num_heads x sequence_length x window_size
+                # which is the attention weights from all tokens to all tokens for global attention
+                # It doesn't not return local attention. Only tokens with global attention have values > 0.0
+                attn_probs = attn_probs[:, :, :, :max_num_global_attn_indices]
+                # pad attn_probs to max length with 0.0 since global attn did not attend there
+                window_size = self.one_sided_attn_window_size * 2 + 1
+                attn_probs = F.pad(attn_probs, (0, window_size - max_num_global_attn_indices), value=0.0,)
+                attn_probs = attn_probs.permute(0, 2, 1, 3)
+            else:
+                # without global attention, return local attention probabilities
+                # batch_size x num_heads x sequence_length x window_size
+                # which is the attention weights of every token attending to its neighbours
+                attn_probs = attn_probs.permute(0, 2, 1, 3)
+
+        outputs = (attn_output, attn_probs) if output_attentions else (attn_output,)
+        return outputs
+
     @staticmethod
     def _get_global_attn_indices(is_index_global_attn):
         """ compute global attn indices required throughout forward pass """
@@ -516,16 +503,12 @@ class LongformerSelfAttention(nn.Module):
         key_vectors_only_global = key_vectors.new_zeros(
             batch_size, max_num_global_attn_indices, self.num_heads, self.head_dim
         )
-
         key_vectors_only_global[is_local_index_global_attn_nonzero] = key_vectors[is_index_global_attn_nonzero]
-
         # (batch_size, seq_len, num_heads, max_num_global_attn_indices)
         attn_probs_from_global_key = torch.einsum("blhd,bshd->blhs", (query_vectors, key_vectors_only_global))
-
         attn_probs_from_global_key[
             is_local_index_no_global_attn_nonzero[0], :, :, is_local_index_no_global_attn_nonzero[1]
         ] = -10000.0
-
         return attn_probs_from_global_key
 
     def _compute_attn_output_with_global_indices(
@@ -617,7 +600,7 @@ class LongformerSelfAttention(nn.Module):
             is_local_index_no_global_attn_nonzero[0], :, is_local_index_no_global_attn_nonzero[1], :
         ] = -10000.0
 
-        global_attn_scores = global_attn_scores.masked_fill(is_index_masked[:, None, None, :], -10000.0,)
+        global_attn_scores = global_attn_scores.masked_fill(is_index_masked.unsqueeze(1).unsqueeze(2), -10000.0,)
 
         global_attn_scores = global_attn_scores.view(batch_size * self.num_heads, max_num_global_attn_indices, seq_len)
 
@@ -771,6 +754,7 @@ class LongformerPreTrainedModel(PreTrainedModel):
 
 
 LONGFORMER_START_DOCSTRING = r"""
+
     This model is a PyTorch `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__ sub-class.
     Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
     usage and behavior.
@@ -839,13 +823,13 @@ LONGFORMER_INPUTS_DOCSTRING = r"""
 )
 class LongformerModel(LongformerPreTrainedModel):
     """
-    This class copied code from :class:`~transformers.RobertaModel` and overwrote standard self-attention with longformer self-attention to provide the ability to process
-    long sequences following the self-attention approach described in `Longformer: the Long-Document Transformer
-    <https://arxiv.org/abs/2004.05150>`__ by Iz Beltagy, Matthew E. Peters, and Arman Cohan. Longformer self-attention
+    This class overrides :class:`~transformers.RobertaModel` to provide the ability to process
+    long sequences following the selfattention approach described in `Longformer: the Long-Document Transformer
+    <https://arxiv.org/abs/2004.05150>`__ by Iz Beltagy, Matthew E. Peters, and Arman Cohan. Longformer selfattention
     combines a local (sliding window) and global attention to extend to long documents without the O(n^2) increase in
     memory and compute.
 
-    The self-attention module `LongformerSelfAttention` implemented here supports the combination of local and
+    The selfattention module `LongformerSelfAttention` implemented here supports the combination of local and
     global attention but it lacks support for autoregressive attention and dilated attention. Autoregressive
     and dilated attention are more relevant for autoregressive language modeling than finetuning on downstream
     tasks. Future release will add support for autoregressive attention, but the support for dilated attention
@@ -899,7 +883,7 @@ class LongformerModel(LongformerPreTrainedModel):
         inputs_embeds: torch.Tensor,
         pad_token_id: int,
     ):
-        """A helper function to pad tokens and mask to work with implementation of Longformer self-attention."""
+        """A helper function to pad tokens and mask to work with implementation of Longformer selfattention."""
         # padding
         attention_window = (
             self.config.attention_window
@@ -1069,9 +1053,6 @@ class LongformerForMaskedLM(LongformerPreTrainedModel):
 
         self.init_weights()
 
-    def get_output_embeddings(self):
-        return self.lm_head.decoder
-
     @add_start_docstrings_to_callable(LONGFORMER_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
     @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC)
     def forward(
@@ -1334,14 +1315,11 @@ class LongformerForQuestionAnswering(BertPreTrainedModel):
         """
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
+        # set global attention on question tokens
         if global_attention_mask is None:
-            if input_ids is None:
-                logger.warning(
-                    "It is not possible to automatically generate the `global_attention_mask` because input_ids is None. Please make sure that it is correctly set."
-                )
-            else:
-                # set global attention on question tokens automatically
-                global_attention_mask = _compute_global_attention_mask(input_ids, self.config.sep_token_id)
+            logger.info("Initializing global attention on question tokens...")
+            # put global attention on all tokens until `config.sep_token_id` is reached
+            global_attention_mask = _compute_global_attention_mask(input_ids, self.config.sep_token_id)
 
         outputs = self.longformer(
             input_ids,
@@ -1526,7 +1504,7 @@ class LongformerForMultipleChoice(BertPreTrainedModel):
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
         # set global attention on question tokens
-        if global_attention_mask is None and input_ids is not None:
+        if global_attention_mask is None:
             logger.info("Initializing global attention on multiple choice...")
             # put global attention on all tokens after `config.sep_token_id`
             global_attention_mask = torch.stack(

src/transformers/modeling_marian.py

@@ -19,7 +19,9 @@ from .configuration_marian import MarianConfig
 from .modeling_bart import BartForConditionalGeneration
 
 
-# See all Marian models at https://huggingface.co/models?search=Helsinki-NLP
+MARIAN_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    # See all Marian models at https://huggingface.co/models?search=Helsinki-NLP
+]
 
 
 class MarianMTModel(BartForConditionalGeneration):
@@ -40,7 +42,7 @@ class MarianMTModel(BartForConditionalGeneration):
 
         >>> model = MarianMTModel.from_pretrained(mname)
         >>> tok = MarianTokenizer.from_pretrained(mname)
-        >>> batch = tok.prepare_seq2seq_batch(src_texts=[sample_text])  # don't need tgt_text for inference
+        >>> batch = tok.prepare_translation_batch(src_texts=[sample_text])  # don't need tgt_text for inference
         >>> gen = model.generate(**batch)  # for forward pass: model(**batch)
         >>> words: List[str] = tok.batch_decode(gen, skip_special_tokens=True)  # returns "Where is the the bus stop ?"
 

src/transformers/modeling_mbart.py

@@ -1,38 +0,0 @@
-from .configuration_mbart import MBartConfig
-from .file_utils import add_start_docstrings
-from .modeling_bart import BartForConditionalGeneration
-
-
-_CONFIG_FOR_DOC = "MBartConfig"
-_TOKENIZER_FOR_DOC = "MBartTokenizer"
-
-MBART_PRETRAINED_MODEL_ARCHIVE_LIST = [
-    "facebook/mbart-large-cc25",
-    "facebook/mbart-large-en-ro",
-    # See all multilingual BART models at https://huggingface.co/models?filter=mbart
-]
-
-MBART_START_DOCSTRING = r"""
-
-    This model is a PyTorch `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__ sub-class.
-    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
-    usage and behavior.
-
-    Parameters:
-        config (:class:`~transformers.MBartConfig`): Model configuration class with all the parameters of the
-            model. Initializing with a config file does not load the weights associated with the model, only the
-            configuration.
-            Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.
-"""
-
-
-@add_start_docstrings(
-    "The BART Model with a language modeling head. Can be used for machine translation.", MBART_START_DOCSTRING
-)
-class MBartForConditionalGeneration(BartForConditionalGeneration):
-    """
-    This class overrides :class:`~transformers.BartForConditionalGeneration`. Please check the
-    superclass for the appropriate documentation alongside usage examples.
-    """
-
-    config_class = MBartConfig

src/transformers/modeling_pegasus.py

@@ -1,46 +0,0 @@
-# coding=utf-8
-# Copyright 2020 Google and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""PyTorch Pegasus model, ported from https://github.com/google-research/pegasus"""
-
-
-from .configuration_pegasus import PegasusConfig
-from .file_utils import add_start_docstrings
-from .modeling_bart import BART_START_DOCSTRING, BartForConditionalGeneration
-
-
-@add_start_docstrings("The Pegasus Model for summarization ", BART_START_DOCSTRING)
-class PegasusForConditionalGeneration(BartForConditionalGeneration):
-    config_class = PegasusConfig
-    r"""
-    Pytorch version of google's pegasus model for summarization.
-    Model API is identical to BartForConditionalGeneration.
-    Available models are listed at `Model List <https://huggingface.co/models?search=pegasus>`__
-
-    Examples::
-
-        >>> from transformers import PegasusTokenizer, PegasusForConditionalGeneration
-        >>> from typing import List
-        >>> PGE_ARTICLE = "PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."
-        >>> mname = "google/pegasus-xsum"
-
-        >>> model = PegasusForConditionalGeneration.from_pretrained(mname)
-        >>> tok = PegasusTokenizer.from_pretrained(mname)
-        >>> batch = tok.prepare_seq2seq_batch(src_texts=[PGE_ARTICLE])  # don't need tgt_text for inference
-        >>> gen = model.generate(**batch)  # for forward pass: model(**batch)
-        >>> summary: List[str] = tok.batch_decode(gen, skip_special_tokens=True)
-        >>> assert summary == "California's largest electricity provider has turned off power to tens of thousands of customers."
-
-    """
-    # All the code is in src/transformers/modeling_bart.py

src/transformers/modeling_roberta.py

@@ -24,15 +24,9 @@ import torch.nn as nn
 from torch.nn import CrossEntropyLoss, MSELoss
 
 from .configuration_roberta import RobertaConfig
-from .file_utils import (
-    add_code_sample_docstrings,
-    add_start_docstrings,
-    add_start_docstrings_to_callable,
-    replace_return_docstrings,
-)
+from .file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_callable
 from .modeling_bert import BertEmbeddings, BertLayerNorm, BertModel, BertPreTrainedModel, gelu
 from .modeling_outputs import (
-    CausalLMOutput,
     MaskedLMOutput,
     MultipleChoiceModelOutput,
     QuestionAnsweringModelOutput,
@@ -181,121 +175,6 @@ class RobertaModel(BertModel):
         self.embeddings.word_embeddings = value
 
 
-@add_start_docstrings(
-    """RoBERTa Model with a `language modeling` head on top for CLM fine-tuning. """, ROBERTA_START_DOCSTRING
-)
-class RobertaForCausalLM(BertPreTrainedModel):
-    config_class = RobertaConfig
-    base_model_prefix = "roberta"
-
-    def __init__(self, config):
-        super().__init__(config)
-
-        if not config.is_decoder:
-            logger.warning("If you want to use `RobertaLMHeadModel` as a standalone, add `is_decoder=True.`")
-
-        self.roberta = RobertaModel(config)
-        self.lm_head = RobertaLMHead(config)
-
-        self.init_weights()
-
-    def get_output_embeddings(self):
-        return self.lm_head.decoder
-
-    @add_start_docstrings_to_callable(ROBERTA_INPUTS_DOCSTRING.format("(batch_size, sequence_length)"))
-    @replace_return_docstrings(output_type=CausalLMOutput, config_class=_CONFIG_FOR_DOC)
-    def forward(
-        self,
-        input_ids=None,
-        attention_mask=None,
-        token_type_ids=None,
-        position_ids=None,
-        head_mask=None,
-        inputs_embeds=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        labels=None,
-        output_attentions=None,
-        output_hidden_states=None,
-        return_dict=None,
-    ):
-        r"""
-        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`):
-            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
-            if the model is configured as a decoder.
-        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
-            Mask to avoid performing attention on the padding token indices of the encoder input. This mask
-            is used in the cross-attention if the model is configured as a decoder.
-            Mask values selected in ``[0, 1]``:
-            ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
-        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`):
-            Labels for computing the left-to-right language modeling loss (next word prediction).
-            Indices should be in ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring)
-            Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens with labels
-            in ``[0, ..., config.vocab_size]``
-
-    Returns:
-
-    Example::
-
-        >>> from transformers import RobertaTokenizer, RobertaLMHeadModel, RobertaConfig
-        >>> import torch
-
-        >>> tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
-        >>> config = RobertaConfig.from_pretrained("roberta-base")
-        >>> config.is_decoder = True
-        >>> model = RobertaLMHeadModel.from_pretrained('roberta-base', config=config, return_dict=True)
-
-        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-        >>> outputs = model(**inputs)
-
-        >>> prediction_logits = outputs.logits
-        """
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-        outputs = self.roberta(
-            input_ids,
-            attention_mask=attention_mask,
-            token_type_ids=token_type_ids,
-            position_ids=position_ids,
-            head_mask=head_mask,
-            inputs_embeds=inputs_embeds,
-            encoder_hidden_states=encoder_hidden_states,
-            encoder_attention_mask=encoder_attention_mask,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
-
-        sequence_output = outputs[0]
-        prediction_scores = self.lm_head(sequence_output)
-
-        lm_loss = None
-        if labels is not None:
-            # we are doing next-token prediction; shift prediction scores and input ids by one
-            shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous()
-            labels = labels[:, 1:].contiguous()
-            loss_fct = CrossEntropyLoss()
-            lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
-
-        if not return_dict:
-            output = (prediction_scores,) + outputs[2:]
-            return ((lm_loss,) + output) if lm_loss is not None else output
-
-        return CausalLMOutput(
-            loss=lm_loss, logits=prediction_scores, hidden_states=outputs.hidden_states, attentions=outputs.attentions,
-        )
-
-    def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **model_kwargs):
-        input_shape = input_ids.shape
-
-        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
-        if attention_mask is None:
-            attention_mask = input_ids.new_ones(input_shape)
-
-        return {"input_ids": input_ids, "attention_mask": attention_mask}
-
-
 @add_start_docstrings("""RoBERTa Model with a `language modeling` head on top. """, ROBERTA_START_DOCSTRING)
 class RobertaForMaskedLM(BertPreTrainedModel):
     config_class = RobertaConfig
@@ -304,12 +183,6 @@ class RobertaForMaskedLM(BertPreTrainedModel):
     def __init__(self, config):
         super().__init__(config)
 
-        if config.is_decoder:
-            logger.warning(
-                "If you want to use `RobertaForMaskedLM` make sure `config.is_decoder=False` for "
-                "bi-directional self-attention."
-            )
-
         self.roberta = RobertaModel(config)
         self.lm_head = RobertaLMHead(config)
 
@@ -333,8 +206,6 @@ class RobertaForMaskedLM(BertPreTrainedModel):
         position_ids=None,
         head_mask=None,
         inputs_embeds=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
         labels=None,
         output_attentions=None,
         output_hidden_states=None,
@@ -366,8 +237,6 @@ class RobertaForMaskedLM(BertPreTrainedModel):
             position_ids=position_ids,
             head_mask=head_mask,
             inputs_embeds=inputs_embeds,
-            encoder_hidden_states=encoder_hidden_states,
-            encoder_attention_mask=encoder_attention_mask,
             output_attentions=output_attentions,
             output_hidden_states=output_hidden_states,
             return_dict=return_dict,

src/transformers/modeling_tf_albert.py

@@ -274,8 +274,6 @@ class TFAlbertSelfOutput(tf.keras.layers.Layer):
 
 
 class TFAlbertAttention(TFBertSelfAttention):
-    """ Contains the complete attention sublayer, including both dropouts and layer norm. """
-
     def __init__(self, config, **kwargs):
         super().__init__(config, **kwargs)
 
@@ -286,9 +284,6 @@ class TFAlbertAttention(TFBertSelfAttention):
         )
         self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name="LayerNorm")
         self.pruned_heads = set()
-        # Two different dropout probabilities; see https://github.com/google-research/albert/blob/master/modeling.py#L971-L993
-        self.attention_dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob)
-        self.output_dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob)
 
     def prune_heads(self, heads):
         raise NotImplementedError
@@ -319,7 +314,7 @@ class TFAlbertAttention(TFBertSelfAttention):
 
         # This is actually dropping out entire tokens to attend to, which might
         # seem a bit unusual, but is taken from the original Transformer paper.
-        attention_probs = self.attention_dropout(attention_probs, training=training)
+        attention_probs = self.dropout(attention_probs, training=training)
 
         # Mask heads if we want to
         if head_mask is not None:
@@ -337,7 +332,7 @@ class TFAlbertAttention(TFBertSelfAttention):
         hidden_states = self_outputs[0]
 
         hidden_states = self.dense(hidden_states)
-        hidden_states = self.output_dropout(hidden_states, training=training)
+        hidden_states = self.dropout(hidden_states, training=training)
         attention_output = self.LayerNorm(hidden_states + input_tensor)
 
         # add attentions if we output them
@@ -374,8 +369,8 @@ class TFAlbertLayer(tf.keras.layers.Layer):
         ffn_output = self.ffn(attention_outputs[0])
         ffn_output = self.activation(ffn_output)
         ffn_output = self.ffn_output(ffn_output)
-        ffn_output = self.dropout(ffn_output, training=training)
 
+        hidden_states = self.dropout(hidden_states, training=training)
         hidden_states = self.full_layer_layer_norm(ffn_output + attention_outputs[0])
 
         # add attentions if we output them

src/transformers/modeling_tf_auto.py

@@ -29,7 +29,6 @@ from .configuration_auto import (
     ElectraConfig,
     FlaubertConfig,
     GPT2Config,
-    LongformerConfig,
     MobileBertConfig,
     OpenAIGPTConfig,
     RobertaConfig,
@@ -94,7 +93,6 @@ from .modeling_tf_flaubert import (
     TFFlaubertWithLMHeadModel,
 )
 from .modeling_tf_gpt2 import TFGPT2LMHeadModel, TFGPT2Model
-from .modeling_tf_longformer import TFLongformerForMaskedLM, TFLongformerForQuestionAnswering, TFLongformerModel
 from .modeling_tf_mobilebert import (
     TFMobileBertForMaskedLM,
     TFMobileBertForMultipleChoice,
@@ -151,7 +149,6 @@ TF_MODEL_MAPPING = OrderedDict(
         (AlbertConfig, TFAlbertModel),
         (CamembertConfig, TFCamembertModel),
         (XLMRobertaConfig, TFXLMRobertaModel),
-        (LongformerConfig, TFLongformerModel),
         (RobertaConfig, TFRobertaModel),
         (BertConfig, TFBertModel),
         (OpenAIGPTConfig, TFOpenAIGPTModel),
@@ -194,7 +191,6 @@ TF_MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
         (AlbertConfig, TFAlbertForMaskedLM),
         (CamembertConfig, TFCamembertForMaskedLM),
         (XLMRobertaConfig, TFXLMRobertaForMaskedLM),
-        (LongformerConfig, TFLongformerForMaskedLM),
         (RobertaConfig, TFRobertaForMaskedLM),
         (BertConfig, TFBertForMaskedLM),
         (OpenAIGPTConfig, TFOpenAIGPTLMHeadModel),
@@ -230,7 +226,6 @@ TF_MODEL_FOR_MASKED_LM_MAPPING = OrderedDict(
         (AlbertConfig, TFAlbertForMaskedLM),
         (CamembertConfig, TFCamembertForMaskedLM),
         (XLMRobertaConfig, TFXLMRobertaForMaskedLM),
-        (LongformerConfig, TFLongformerForMaskedLM),
         (RobertaConfig, TFRobertaForMaskedLM),
         (BertConfig, TFBertForMaskedLM),
         (MobileBertConfig, TFMobileBertForMaskedLM),
@@ -264,7 +259,6 @@ TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
         (AlbertConfig, TFAlbertForQuestionAnswering),
         (CamembertConfig, TFCamembertForQuestionAnswering),
         (XLMRobertaConfig, TFXLMRobertaForQuestionAnswering),
-        (LongformerConfig, TFLongformerForQuestionAnswering),
         (RobertaConfig, TFRobertaForQuestionAnswering),
         (BertConfig, TFBertForQuestionAnswering),
         (XLNetConfig, TFXLNetForQuestionAnsweringSimple),

src/transformers/modeling_tf_bert.py

@@ -860,12 +860,9 @@ class TFBertForPreTraining(TFBertPreTrainedModel):
 class TFBertForMaskedLM(TFBertPreTrainedModel, TFMaskedLanguageModelingLoss):
     def __init__(self, config, *inputs, **kwargs):
         super().__init__(config, *inputs, **kwargs)
-
-        if config.is_decoder:
-            logger.warning(
-                "If you want to use `TFBertForMaskedLM` make sure `config.is_decoder=False` for "
-                "bi-directional self-attention."
-            )
+        assert (
+            not config.is_decoder
+        ), "If you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention."
 
         self.bert = TFBertMainLayer(config, name="bert")
         self.mlm = TFBertMLMHead(config, self.bert.embeddings, name="mlm___cls")
@@ -939,9 +936,7 @@ class TFBertForMaskedLM(TFBertPreTrainedModel, TFMaskedLanguageModelingLoss):
 class TFBertLMHeadModel(TFBertPreTrainedModel, TFCausalLanguageModelingLoss):
     def __init__(self, config, *inputs, **kwargs):
         super().__init__(config, *inputs, **kwargs)
-
-        if not config.is_decoder:
-            logger.warning("If you want to use `TFBertLMHeadModel` as a standalone, add `is_decoder=True.`")
+        assert config.is_decoder, "If you want to use `TFBertLMHeadModel` as a standalone, add `is_decoder=True.`"
 
         self.bert = TFBertMainLayer(config, name="bert")
         self.mlm = TFBertMLMHead(config, self.bert.embeddings, name="mlm___cls")

src/transformers/modeling_utils.py

@@ -885,7 +885,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin):
             else:
                 logger.info(
                     f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at {pretrained_model_name_or_path}.\n"
-                    f"If your task is similar to the task the model of the checkpoint was trained on, "
+                    f"If your task is similar to the task the model of the ckeckpoint was trained on, "
                     f"you can already use {model.__class__.__name__} for predictions without further training."
                 )
             if len(error_msgs) > 0:

src/transformers/optimization.py

@@ -165,52 +165,6 @@ def get_cosine_with_hard_restarts_schedule_with_warmup(
     return LambdaLR(optimizer, lr_lambda, last_epoch)
 
 
-def get_polynomial_decay_schedule_with_warmup(
-    optimizer, num_warmup_steps, num_training_steps, lr_end=1e-7, power=2.0, last_epoch=-1
-):
-    """
-    Create a schedule with a learning rate that decreases as a polynomial decay
-    from the initial lr set in the optimizer to end lr defined by `lr_end`,
-    after a warmup period during which it increases linearly from 0 to the
-    initial lr set in the optimizer.
-
-    Args:
-        optimizer (:class:`~torch.optim.Optimizer`):
-            The optimizer for which to schedule the learning rate.
-        num_warmup_steps (:obj:`int`):
-            The number of steps for the warmup phase.
-        num_training_steps (:obj:`int`):
-            The total number of training steps.
-        lr_end (:obj:`float`, `optional`, defaults to 1e-7):
-            The end LR.
-        power (:obj:`float`, `optional`, defaults to 1.0):
-            Power factor.
-        last_epoch (:obj:`int`, `optional`, defaults to -1):
-            The index of the last epoch when resuming training.
-
-    Return:
-        :obj:`torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule.
-
-    """
-
-    lr_init = optimizer.defaults["lr"]
-    assert lr_init > lr_end, f"lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})"
-
-    def lr_lambda(current_step: int):
-        if current_step < num_warmup_steps:
-            return float(current_step) / float(max(1, num_warmup_steps))
-        elif current_step > num_training_steps:
-            return lr_end / lr_init  # as LambdaLR multiplies by lr_init
-        else:
-            lr_range = lr_init - lr_end
-            decay_steps = num_training_steps - num_warmup_steps
-            pct_remaining = 1 - (current_step - num_warmup_steps) / decay_steps
-            decay = lr_range * pct_remaining ** power + lr_end
-            return decay / lr_init  # as LambdaLR multiplies by lr_init
-
-    return LambdaLR(optimizer, lr_lambda, last_epoch)
-
-
 class AdamW(Optimizer):
     """
     Implements Adam algorithm with weight decay fix as introduced in

src/transformers/pipelines.py

@@ -1158,17 +1158,12 @@ class FillMaskPipeline(Pipeline):
                 f"No mask_token ({self.tokenizer.mask_token}) found on the input",
             )
 
-    def __call__(self, *args, targets=None, **kwargs):
+    def __call__(self, *args, **kwargs):
         """
         Fill the masked token in the text(s) given as inputs.
 
         Args:
-            args (:obj:`str` or :obj:`List[str]`):
-                One or several texts (or one list of prompts) with masked tokens.
-            targets (:obj:`str` or :obj:`List[str]`, `optional`):
-                When passed, the model will return the scores for the passed token or tokens rather than the top k
-                predictions in the entire vocabulary. If the provided targets are not in the model vocab, they will
-                be tokenized and the first resulting token will be used (with a warning).
+            args (:obj:`str` or :obj:`List[str]`): One or several texts (or one list of prompts) with masked tokens.
 
         Return:
             A list or a list of list of :obj:`dict`: Each result comes as list of dictionaries with the
@@ -1185,24 +1180,6 @@ class FillMaskPipeline(Pipeline):
         results = []
         batch_size = outputs.shape[0] if self.framework == "tf" else outputs.size(0)
 
-        if targets is not None:
-            if len(targets) == 0 or len(targets[0]) == 0:
-                raise ValueError("At least one target must be provided when passed.")
-            if isinstance(targets, str):
-                targets = [targets]
-
-            targets_proc = []
-            for target in targets:
-                target_enc = self.tokenizer.tokenize(target)
-                if len(target_enc) > 1 or target_enc[0] == self.tokenizer.unk_token:
-                    logger.warning(
-                        "The specified target token `{}` does not exist in the model vocabulary. Replacing with `{}`.".format(
-                            target, target_enc[0]
-                        )
-                    )
-                targets_proc.append(target_enc[0])
-            target_inds = np.array(self.tokenizer.convert_tokens_to_ids(targets_proc))
-
         for i in range(batch_size):
             input_ids = inputs["input_ids"][i]
             result = []
@@ -1215,14 +1192,8 @@ class FillMaskPipeline(Pipeline):
 
                 logits = outputs[i, masked_index.item(), :]
                 probs = tf.nn.softmax(logits)
-                if targets is None:
-                    topk = tf.math.top_k(probs, k=self.topk)
-                    values, predictions = topk.values.numpy(), topk.indices.numpy()
-                else:
-                    values = tf.gather_nd(probs, tf.reshape(target_inds, (-1, 1)))
-                    sort_inds = tf.reverse(tf.argsort(values), [0])
-                    values = tf.gather_nd(values, tf.reshape(sort_inds, (-1, 1))).numpy()
-                    predictions = target_inds[sort_inds.numpy()]
+                topk = tf.math.top_k(probs, k=self.topk)
+                values, predictions = topk.values.numpy(), topk.indices.numpy()
             else:
                 masked_index = (input_ids == self.tokenizer.mask_token_id).nonzero()
 
@@ -1231,13 +1202,7 @@ class FillMaskPipeline(Pipeline):
 
                 logits = outputs[i, masked_index.item(), :]
                 probs = logits.softmax(dim=0)
-                if targets is None:
-                    values, predictions = probs.topk(self.topk)
-                else:
-                    values = probs[..., target_inds]
-                    sort_inds = list(reversed(values.argsort(dim=-1)))
-                    values = values[..., sort_inds]
-                    predictions = target_inds[sort_inds]
+                values, predictions = probs.topk(self.topk)
 
             for v, p in zip(values.tolist(), predictions.tolist()):
                 tokens = input_ids.numpy()
@@ -2353,7 +2318,7 @@ class ConversationalPipeline(Pipeline):
         max_len = max([len(item) for item in outputs])
         outputs = [output + [self.pad_token_id] * (max_len - len(output)) for output in outputs]
         outputs = BatchEncoding(
-            {"input_ids": outputs, "attention_mask": [[1] * len(outputs)]}, tensor_type=self.framework,
+            {"input_ids": outputs, "attention_mask": [1] * len(outputs)}, tensor_type=self.framework
         )
         return outputs
 

src/transformers/testing_utils.py

@@ -1,9 +1,6 @@
 import os
-import re
-import sys
 import unittest
 from distutils.util import strtobool
-from io import StringIO
 
 from .file_utils import _tf_available, _torch_available, _torch_tpu_available
 
@@ -139,119 +136,3 @@ def require_torch_and_cuda(test_case):
         return unittest.skip("test requires CUDA")
     else:
         return test_case
-
-
-#
-# Helper functions for dealing with testing text outputs
-# The original code came from:
-# https://github.com/fastai/fastai/blob/master/tests/utils/text.py
-
-# When any function contains print() calls that get overwritten, like progress bars,
-# a special care needs to be applied, since under pytest -s captured output (capsys
-# or contextlib.redirect_stdout) contains any temporary printed strings, followed by
-# \r's. This helper function ensures that the buffer will contain the same output
-# with and without -s in pytest, by turning:
-# foo bar\r tar mar\r final message
-# into:
-# final message
-# it can handle a single string or a multiline buffer
-def apply_print_resets(buf):
-    return re.sub(r"^.*\r", "", buf, 0, re.M)
-
-
-def assert_screenout(out, what):
-    out_pr = apply_print_resets(out).lower()
-    match_str = out_pr.find(what.lower())
-    assert match_str != -1, f"expecting to find {what} in output: f{out_pr}"
-
-
-class CaptureStd:
-    """ Context manager to capture:
-    stdout, clean it up and make it available via obj.out
-    stderr, and make it available via obj.err
-
-    init arguments:
-    - out - capture stdout: True/False, default True
-    - err - capture stdout: True/False, default True
-
-    Examples:
-
-    with CaptureStdout() as cs:
-        print("Secret message")
-    print(f"captured: {cs.out}")
-
-    import sys
-    with CaptureStderr() as cs:
-        print("Warning: ", file=sys.stderr)
-    print(f"captured: {cs.err}")
-
-    # to capture just one of the streams, but not the other
-    with CaptureStd(err=False) as cs:
-        print("Secret message")
-    print(f"captured: {cs.out}")
-    # but best use the stream-specific subclasses
-
-    """
-
-    def __init__(self, out=True, err=True):
-        if out:
-            self.out_buf = StringIO()
-            self.out = "error: CaptureStd context is unfinished yet, called too early"
-        else:
-            self.out_buf = None
-            self.out = "not capturing stdout"
-
-        if err:
-            self.err_buf = StringIO()
-            self.err = "error: CaptureStd context is unfinished yet, called too early"
-        else:
-            self.err_buf = None
-            self.err = "not capturing stderr"
-
-    def __enter__(self):
-        if self.out_buf:
-            self.out_old = sys.stdout
-            sys.stdout = self.out_buf
-
-        if self.err_buf:
-            self.err_old = sys.stderr
-            sys.stderr = self.err_buf
-
-        return self
-
-    def __exit__(self, *exc):
-        if self.out_buf:
-            sys.stdout = self.out_old
-            self.out = apply_print_resets(self.out_buf.getvalue())
-
-        if self.err_buf:
-            sys.stderr = self.err_old
-            self.err = self.err_buf.getvalue()
-
-    def __repr__(self):
-        msg = ""
-        if self.out_buf:
-            msg += f"stdout: {self.out}\n"
-        if self.err_buf:
-            msg += f"stderr: {self.err}\n"
-        return msg
-
-
-# in tests it's the best to capture only the stream that's wanted, otherwise
-# it's easy to miss things, so unless you need to capture both streams, use the
-# subclasses below (less typing). Or alternatively, configure `CaptureStd` to
-# disable the stream you don't need to test.
-
-
-class CaptureStdout(CaptureStd):
-    """ Same as CaptureStd but captures only stdout """
-
-    def __init__(self):
-        super().__init__(err=False)
-
-
-class CaptureStderr(CaptureStd):
-    """ Same as CaptureStd but captures only stderr """
-
-    def __init__(self):
-        super().__init__(out=False)

src/transformers/tokenization_auto.py

@@ -30,11 +30,8 @@ from .configuration_auto import (
     FlaubertConfig,
     GPT2Config,
     LongformerConfig,
-    MarianConfig,
     MBartConfig,
-    MobileBertConfig,
     OpenAIGPTConfig,
-    PegasusConfig,
     ReformerConfig,
     RetriBertConfig,
     RobertaConfig,
@@ -44,9 +41,11 @@ from .configuration_auto import (
     XLMRobertaConfig,
     XLNetConfig,
 )
+from .configuration_marian import MarianConfig
+from .configuration_mobilebert import MobileBertConfig
 from .configuration_utils import PretrainedConfig
 from .tokenization_albert import AlbertTokenizer
-from .tokenization_bart import BartTokenizer, BartTokenizerFast
+from .tokenization_bart import BartTokenizer, MBartTokenizer
 from .tokenization_bert import BertTokenizer, BertTokenizerFast
 from .tokenization_bert_japanese import BertJapaneseTokenizer
 from .tokenization_camembert import CamembertTokenizer
@@ -55,12 +54,10 @@ from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFas
 from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast
 from .tokenization_flaubert import FlaubertTokenizer
 from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
-from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast
+from .tokenization_longformer import LongformerTokenizer
 from .tokenization_marian import MarianTokenizer
-from .tokenization_mbart import MBartTokenizer
 from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFast
 from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
-from .tokenization_pegasus import PegasusTokenizer
 from .tokenization_reformer import ReformerTokenizer
 from .tokenization_retribert import RetriBertTokenizer, RetriBertTokenizerFast
 from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
@@ -82,12 +79,11 @@ TOKENIZER_MAPPING = OrderedDict(
         (DistilBertConfig, (DistilBertTokenizer, DistilBertTokenizerFast)),
         (AlbertConfig, (AlbertTokenizer, None)),
         (CamembertConfig, (CamembertTokenizer, None)),
-        (PegasusConfig, (PegasusTokenizer, None)),
         (MBartConfig, (MBartTokenizer, None)),
         (XLMRobertaConfig, (XLMRobertaTokenizer, None)),
         (MarianConfig, (MarianTokenizer, None)),
-        (BartConfig, (BartTokenizer, BartTokenizerFast)),
-        (LongformerConfig, (LongformerTokenizer, LongformerTokenizerFast)),
+        (BartConfig, (BartTokenizer, None)),
+        (LongformerConfig, (LongformerTokenizer, None)),
         (RobertaConfig, (RobertaTokenizer, RobertaTokenizerFast)),
         (ReformerConfig, (ReformerTokenizer, None)),
         (ElectraConfig, (ElectraTokenizer, ElectraTokenizerFast)),

src/transformers/tokenization_bart.py

@@ -14,8 +14,11 @@
 # limitations under the License.
 
 import logging
+from typing import List, Optional
 
 from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
+from .tokenization_utils import BatchEncoding
+from .tokenization_xlm_roberta import XLMRobertaTokenizer
 
 
 logger = logging.getLogger(__name__)
@@ -50,3 +53,206 @@ class BartTokenizerFast(RobertaTokenizerFast):
         "vocab_file": {m: vocab_url for m in _all_bart_models},
         "merges_file": {m: merges_url for m in _all_bart_models},
     }
+
+
+_all_mbart_models = ["facebook/mbart-large-en-ro", "facebook/mbart-large-cc25"]
+SPM_URL = "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/mbart-large-en-ro/sentence.bpe.model"
+
+FAIRSEQ_LANGUAGE_CODES = [
+    "ar_AR",
+    "cs_CZ",
+    "de_DE",
+    "en_XX",
+    "es_XX",
+    "et_EE",
+    "fi_FI",
+    "fr_XX",
+    "gu_IN",
+    "hi_IN",
+    "it_IT",
+    "ja_XX",
+    "kk_KZ",
+    "ko_KR",
+    "lt_LT",
+    "lv_LV",
+    "my_MM",
+    "ne_NP",
+    "nl_XX",
+    "ro_RO",
+    "ru_RU",
+    "si_LK",
+    "tr_TR",
+    "vi_VN",
+    "zh_CN",
+]
+
+
+class MBartTokenizer(XLMRobertaTokenizer):
+    """
+    This inherits from XLMRobertaTokenizer. ``prepare_translation_batch`` should be used to encode inputs.
+    Other tokenizer methods like ``encode`` do not work properly.
+    The tokenization method is ``<tokens> <eos> <language code>`` for source language documents, and
+    ``<language code> <tokens> <eos>``` for target language documents.
+
+    Examples::
+
+        >>> from transformers import MBartTokenizer
+        >>> tokenizer = MBartTokenizer.from_pretrained('facebook/mbart-large-en-ro')
+        >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
+        >>> expected_translation_romanian = "Şeful ONU declară că nu există o soluţie militară în Siria"
+        >>> batch: dict = tokenizer.prepare_translation_batch(
+        ...     example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian
+        ... )
+
+    """
+
+    vocab_files_names = {"vocab_file": "sentencepiece.bpe.model"}
+    max_model_input_sizes = {m: 1024 for m in _all_mbart_models}
+    pretrained_vocab_files_map = {"vocab_file": {m: SPM_URL for m in _all_mbart_models}}
+
+    prefix_tokens: List[int] = []
+    suffix_tokens: List[int] = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.sp_model_size = len(self.sp_model)
+        self.lang_code_to_id = {
+            code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(FAIRSEQ_LANGUAGE_CODES)
+        }
+        self.id_to_lang_code = {v: k for k, v in self.lang_code_to_id.items()}
+        self.cur_lang_code = self.lang_code_to_id["en_XX"]
+        self.fairseq_tokens_to_ids["<mask>"] = len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset
+
+        self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
+        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}
+        self._additional_special_tokens = list(self.lang_code_to_id.keys())
+        self.set_src_lang_special_tokens(kwargs.get("src_lang", "en_XX"))
+
+    def build_inputs_with_special_tokens(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks
+        by concatenating and adding special tokens. The special tokens depend on calling set_lang.
+        An MBART sequence has the following format, where ``X`` represents the sequence:
+        - ``input_ids`` (for encoder) ``X [eos, src_lang_code]``
+        - ``decoder_input_ids``: (for decoder) ``[tgt_lang_code] X [eos]``
+        BOS is never used.
+        Pairs of sequences are not the expected use case, but they will be handled without a separator.
+
+        Args:
+            token_ids_0 (:obj:`List[int]`):
+                List of IDs to which the special tokens will be added
+            token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
+        """
+        if token_ids_1 is None:
+            return self.prefix_tokens + token_ids_0 + self.suffix_tokens
+        # We don't expect to process pairs, but leave the pair logic for API consistency
+        return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """
+        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer ``prepare_for_model`` methods.
+
+        Args:
+            token_ids_0 (:obj:`List[int]`):
+                List of ids.
+            token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                Set to True if the token list is already formatted with special tokens for the model
+
+        Returns:
+            :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+
+        if already_has_special_tokens:
+            if token_ids_1 is not None:
+                raise ValueError(
+                    "You should not supply a second sequence if the provided sequence of "
+                    "ids is already formated with special tokens for the model."
+                )
+            return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))
+        prefix_ones = [1] * len(self.prefix_tokens)
+        suffix_ones = [1] * len(self.suffix_tokens)
+        if token_ids_1 is None:
+            return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
+        return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones
+
+    def prepare_translation_batch(
+        self,
+        src_texts: List[str],
+        src_lang: str = "en_XX",
+        tgt_texts: Optional[List[str]] = None,
+        tgt_lang: str = "ro_RO",
+        max_length: Optional[int] = None,
+        max_target_length: Optional[int] = None,
+        padding: str = "longest",
+        return_tensors: str = "pt",
+        **kwargs,
+    ) -> BatchEncoding:
+        """Prepare a batch that can be passed directly to an instance of MBartModel.
+        Arguments:
+            src_texts: list of src language texts
+            src_lang: default en_XX (english), the language we are translating from
+            tgt_texts: list of tgt language texts
+            tgt_lang: default ro_RO (romanian), the language we are translating to
+            max_length: (default=None, which defers to the config value of 1024 for facebook/mbart-large*
+            padding: strategy for padding input_ids and decoder_input_ids. Should be max_length or longest.
+            **kwargs: passed to self.__call__
+
+        Returns:
+            :obj:`BatchEncoding`: with keys input_ids, attention_mask, decoder_input_ids, decoder_attention_mask.
+        """
+        if max_length is None:
+            max_length = self.max_len
+        self.set_src_lang_special_tokens(src_lang)
+        model_inputs: BatchEncoding = self(
+            src_texts,
+            add_special_tokens=True,
+            return_tensors=return_tensors,
+            max_length=max_length,
+            padding=padding,
+            truncation=True,
+            **kwargs,
+        )
+        if tgt_texts is None:
+            return model_inputs
+        # Process tgt_texts
+        if max_target_length is None:
+            max_target_length = max_length
+        self.set_tgt_lang_special_tokens(tgt_lang)
+        decoder_inputs: BatchEncoding = self(
+            tgt_texts,
+            add_special_tokens=True,
+            return_tensors=return_tensors,
+            padding=padding,
+            max_length=max_target_length,
+            truncation=True,
+            **kwargs,
+        )
+        for k, v in decoder_inputs.items():
+            model_inputs[f"decoder_{k}"] = v
+
+        self.set_src_lang_special_tokens(src_lang)  # sets to src_lang
+        return model_inputs
+
+    def set_src_lang_special_tokens(self, src_lang) -> None:
+        """Reset the special tokens to the source lang setting. No prefix and suffix=[eos, cur_lang_code]."""
+        self.cur_lang_code = self.lang_code_to_id[src_lang]
+        self.prefix_tokens = []
+        self.suffix_tokens = [self.eos_token_id, self.cur_lang_code]
+
+    def set_tgt_lang_special_tokens(self, lang: str) -> None:
+        """Reset the special tokens to the target language setting. Prefix [tgt_lang_code], suffix =[eos]."""
+        self.cur_lang_code = self.lang_code_to_id[lang]
+        self.prefix_tokens = [self.cur_lang_code]
+        self.suffix_tokens = [self.eos_token_id]

src/transformers/tokenization_marian.py

@@ -7,9 +7,7 @@ from typing import Dict, List, Optional, Tuple, Union
 
 import sentencepiece
 
-from .file_utils import add_start_docstrings_to_callable
 from .tokenization_utils import BatchEncoding, PreTrainedTokenizer
-from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING
 
 
 vocab_files_names = {
@@ -23,8 +21,7 @@ vocab_files_names = {
 
 class MarianTokenizer(PreTrainedTokenizer):
     """Sentencepiece tokenizer for marian. Source and target languages have different SPM models.
-    The logic is use the relevant source_spm or target_spm to encode txt as pieces, then look up each piece in a
-    vocab dictionary.
+    The logic is use the relevant source_spm or target_spm to encode txt as pieces, then look up each piece in a vocab dictionary.
 
     Examples::
 
@@ -32,7 +29,7 @@ class MarianTokenizer(PreTrainedTokenizer):
         >>> tok = MarianTokenizer.from_pretrained('Helsinki-NLP/opus-mt-en-de')
         >>> src_texts = [ "I am a small frog.", "Tom asked his teacher for advice."]
         >>> tgt_texts = ["Ich bin ein kleiner Frosch.", "Tom bat seinen Lehrer um Rat."]  # optional
-        >>> batch_enc: BatchEncoding = tok.prepare_seq2seq_batch(src_texts, tgt_texts=tgt_texts)
+        >>> batch_enc: BatchEncoding = tok.prepare_translation_batch(src_texts, tgt_texts=tgt_texts)
         >>> # keys  [input_ids, attention_mask, decoder_input_ids,  decoder_attention_mask].
         >>> # model(**batch) should work
     """
@@ -125,20 +122,30 @@ class MarianTokenizer(PreTrainedTokenizer):
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return token_ids_0 + token_ids_1 + [self.eos_token_id]
 
-    @add_start_docstrings_to_callable(PREPARE_SEQ2SEQ_BATCH_DOCSTRING)
-    def prepare_seq2seq_batch(
+    def prepare_translation_batch(
         self,
         src_texts: List[str],
         tgt_texts: Optional[List[str]] = None,
         max_length: Optional[int] = None,
         max_target_length: Optional[int] = None,
+        pad_to_max_length: bool = True,
         return_tensors: str = "pt",
-        truncation=True,
+        truncation_strategy="only_first",
         padding="longest",
         **unused,
     ) -> BatchEncoding:
         """Prepare model inputs for translation. For best performance, translate one sentence at a time.
+        Arguments:
+            src_texts: list of src language texts
+            tgt_texts: list of tgt language texts
+            max_length: (None) defer to config (1024 for mbart-large-en-ro)
+            pad_to_max_length: (bool)
+            return_tensors: (str) default "pt" returns pytorch tensors, pass None to return lists.
 
+        Returns:
+            BatchEncoding: with keys [input_ids, attention_mask, decoder_input_ids,  decoder_attention_mask]
+            all shaped bs, seq_len. (BatchEncoding is a dict of string -> tensor or lists).
+            If no tgt_text is specified, the only keys will be input_ids and attention_mask.
         """
         if "" in src_texts:
             raise ValueError(f"found empty string in src_texts: {src_texts}")
@@ -148,15 +155,14 @@ class MarianTokenizer(PreTrainedTokenizer):
             add_special_tokens=True,
             return_tensors=return_tensors,
             max_length=max_length,
-            truncation=truncation,
+            pad_to_max_length=pad_to_max_length,
+            truncation_strategy=truncation_strategy,
             padding=padding,
         )
         model_inputs: BatchEncoding = self(src_texts, **tokenizer_kwargs)
 
         if tgt_texts is None:
             return model_inputs
-        if max_target_length is not None:
-            tokenizer_kwargs["max_length"] = max_target_length
 
         if max_target_length is not None:
             tokenizer_kwargs["max_length"] = max_target_length

src/transformers/tokenization_mbart.py

@@ -1,279 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from typing import List, Optional
-
-from .file_utils import add_start_docstrings_to_callable
-from .tokenization_utils import BatchEncoding
-from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING
-from .tokenization_xlm_roberta import XLMRobertaTokenizer
-
-
-logger = logging.getLogger(__name__)
-
-_all_mbart_models = ["facebook/mbart-large-en-ro", "facebook/mbart-large-cc25"]
-SPM_URL = "https://s3.amazonaws.com/models.huggingface.co/bert/facebook/mbart-large-en-ro/sentence.bpe.model"
-
-FAIRSEQ_LANGUAGE_CODES = [
-    "ar_AR",
-    "cs_CZ",
-    "de_DE",
-    "en_XX",
-    "es_XX",
-    "et_EE",
-    "fi_FI",
-    "fr_XX",
-    "gu_IN",
-    "hi_IN",
-    "it_IT",
-    "ja_XX",
-    "kk_KZ",
-    "ko_KR",
-    "lt_LT",
-    "lv_LV",
-    "my_MM",
-    "ne_NP",
-    "nl_XX",
-    "ro_RO",
-    "ru_RU",
-    "si_LK",
-    "tr_TR",
-    "vi_VN",
-    "zh_CN",
-]
-
-
-class MBartTokenizer(XLMRobertaTokenizer):
-    """
-    This inherits from XLMRobertaTokenizer. ``prepare_seq2seq_batch`` should be used to encode inputs.
-    Other tokenizer methods like ``encode`` do not work properly.
-    The tokenization method is ``<tokens> <eos> <language code>`` for source language documents, and
-    ``<language code> <tokens> <eos>``` for target language documents.
-
-    Examples::
-
-        >>> from transformers import MBartTokenizer
-        >>> tokenizer = MBartTokenizer.from_pretrained('facebook/mbart-large-en-ro')
-        >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
-        >>> expected_translation_romanian = "Şeful ONU declară că nu există o soluţie militară în Siria"
-        >>> batch: dict = tokenizer.prepare_seq2seq_batch(
-        ...     example_english_phrase, src_lang="en_XX", tgt_lang="ro_RO", tgt_texts=expected_translation_romanian
-        ... )
-
-    """
-
-    vocab_files_names = {"vocab_file": "sentencepiece.bpe.model"}
-    max_model_input_sizes = {m: 1024 for m in _all_mbart_models}
-    pretrained_vocab_files_map = {"vocab_file": {m: SPM_URL for m in _all_mbart_models}}
-
-    prefix_tokens: List[int] = []
-    suffix_tokens: List[int] = []
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        self.sp_model_size = len(self.sp_model)
-        self.lang_code_to_id = {
-            code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(FAIRSEQ_LANGUAGE_CODES)
-        }
-        self.id_to_lang_code = {v: k for k, v in self.lang_code_to_id.items()}
-        self.cur_lang_code = self.lang_code_to_id["en_XX"]
-        self.fairseq_tokens_to_ids["<mask>"] = len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset
-
-        self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
-        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}
-        self._additional_special_tokens = list(self.lang_code_to_id.keys())
-        self.set_src_lang_special_tokens(kwargs.get("src_lang", "en_XX"))
-
-    def build_inputs_with_special_tokens(
-        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
-    ) -> List[int]:
-        """
-        Build model inputs from a sequence or a pair of sequence for sequence classification tasks
-        by concatenating and adding special tokens. The special tokens depend on calling set_lang.
-        An MBART sequence has the following format, where ``X`` represents the sequence:
-        - ``input_ids`` (for encoder) ``X [eos, src_lang_code]``
-        - ``decoder_input_ids``: (for decoder) ``[tgt_lang_code] X [eos]``
-        BOS is never used.
-        Pairs of sequences are not the expected use case, but they will be handled without a separator.
-
-        Args:
-            token_ids_0 (:obj:`List[int]`):
-                List of IDs to which the special tokens will be added
-            token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`):
-                Optional second list of IDs for sequence pairs.
-
-        Returns:
-            :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
-        """
-        if token_ids_1 is None:
-            return self.prefix_tokens + token_ids_0 + self.suffix_tokens
-        # We don't expect to process pairs, but leave the pair logic for API consistency
-        return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens
-
-    def get_special_tokens_mask(
-        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
-    ) -> List[int]:
-        """
-        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
-        special tokens using the tokenizer ``prepare_for_model`` methods.
-
-        Args:
-            token_ids_0 (:obj:`List[int]`):
-                List of ids.
-            token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`):
-                Optional second list of IDs for sequence pairs.
-            already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Set to True if the token list is already formatted with special tokens for the model
-
-        Returns:
-            :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
-        """
-
-        if already_has_special_tokens:
-            if token_ids_1 is not None:
-                raise ValueError(
-                    "You should not supply a second sequence if the provided sequence of "
-                    "ids is already formated with special tokens for the model."
-                )
-            return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))
-        prefix_ones = [1] * len(self.prefix_tokens)
-        suffix_ones = [1] * len(self.suffix_tokens)
-        if token_ids_1 is None:
-            return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
-        return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones
-
-    @add_start_docstrings_to_callable(PREPARE_SEQ2SEQ_BATCH_DOCSTRING)
-    def prepare_seq2seq_batch(
-        self,
-        src_texts: List[str],
-        src_lang: str = "en_XX",
-        tgt_texts: Optional[List[str]] = None,
-        tgt_lang: str = "ro_RO",
-        max_length: Optional[int] = None,
-        max_target_length: Optional[int] = None,
-        truncation: bool = True,
-        padding: str = "longest",
-        return_tensors: str = "pt",
-        **kwargs,
-    ) -> BatchEncoding:
-        """Prepare a batch that can be passed directly to an instance of MBartModel.
-
-        Arguments:
-            src_texts: (:obj:`list`):
-                list of documents to summarize or source language texts
-            src_lang: (:obj:`str`, `optional`, default='en_XX'):
-                default en_XX (english), the language we are translating from
-            tgt_texts: (:obj:`list`, `optional`):
-                list of tgt language texts or summaries.
-            tgt_lang: (:obj:`str`, `optional`, default='ro_RO'):
-                default ro_RO (romanian), the language we are translating to
-            max_length (:obj:`int`, `optional`):
-                Controls the maximum length for encoder inputs (documents to summarize or source language texts)
-                If left unset or set to :obj:`None`, this will use the predefined model maximum length if a maximum
-                length is required by one of the truncation/padding parameters. If the model has no specific maximum
-                input length (like XLNet) truncation/padding to a maximum length will be deactivated.
-            max_target_length (:obj:`int`, `optional`):
-                Controls the maximum length of decoder inputs (target language texts or summaries)
-                If left unset or set to :obj:`None`, this will use the max_length value.
-            padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`False`):
-                Activates and controls padding. Accepts the following values:
-
-                * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a
-                  single sequence if provided).
-                * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
-                  maximum acceptable input length for the model if that argument is not provided.
-                * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
-                  different lengths).
-            return_tensors (:obj:`str` or :class:`~transformers.tokenization_utils_base.TensorType`, `optional`, defaults to "pt"):
-                If set, will return tensors instead of list of python integers. Acceptable values are:
-
-                * :obj:`'tf'`: Return TensorFlow :obj:`tf.constant` objects.
-                * :obj:`'pt'`: Return PyTorch :obj:`torch.Tensor` objects.
-                * :obj:`'np'`: Return Numpy :obj:`np.ndarray` objects.
-            truncation (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.TruncationStrategy`, `optional`, defaults to :obj:`True`):
-                Activates and controls truncation. Accepts the following values:
-
-                * :obj:`True` or :obj:`'longest_first'`: Truncate to a maximum length specified with the argument
-                  :obj:`max_length` or to the maximum acceptable input length for the model if that argument is not
-                  provided. This will truncate token by token, removing a token from the longest sequence in the pair
-                  if a pair of sequences (or a batch of pairs) is provided.
-                * :obj:`'only_first'`: Truncate to a maximum length specified with the argument :obj:`max_length` or to
-                  the maximum acceptable input length for the model if that argument is not provided. This will only
-                  truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
-                * :obj:`'only_second'`: Truncate to a maximum length specified with the argument :obj:`max_length` or
-                  to the maximum acceptable input length for the model if that argument is not provided. This will only
-                  truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
-                * :obj:`False` or :obj:`'do_not_truncate'` (default): No truncation (i.e., can output batch with
-                  sequence lengths greater than the model maximum admissible input size).
-
-        Return:
-            :class:`~transformers.BatchEncoding`: A :class:`~transformers.BatchEncoding` with the following fields:
-
-            - **input_ids** -- List of token ids to be fed to the encoder.
-            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model.
-            - **decoder_input_ids** -- List of token ids to be fed to the decoder.
-            - **decoder_attention_mask** -- List of indices specifying which tokens should be attended to by the decoder.
-                This does not include causal mask, which is built by the model.
-
-            The full set of keys ``[input_ids, attention_mask, decoder_input_ids,  decoder_attention_mask]``,
-            will only be returned if tgt_texts is passed. Otherwise, input_ids, attention_mask will be the only keys.
-
-        """
-        if max_length is None:
-            max_length = self.max_len
-        self.set_src_lang_special_tokens(src_lang)
-        model_inputs: BatchEncoding = self(
-            src_texts,
-            add_special_tokens=True,
-            return_tensors=return_tensors,
-            max_length=max_length,
-            padding=padding,
-            truncation=truncation,
-            **kwargs,
-        )
-        if tgt_texts is None:
-            return model_inputs
-        # Process tgt_texts
-        if max_target_length is None:
-            max_target_length = max_length
-        self.set_tgt_lang_special_tokens(tgt_lang)
-        decoder_inputs: BatchEncoding = self(
-            tgt_texts,
-            add_special_tokens=True,
-            return_tensors=return_tensors,
-            padding=padding,
-            max_length=max_target_length,
-            truncation=True,
-            **kwargs,
-        )
-        for k, v in decoder_inputs.items():
-            model_inputs[f"decoder_{k}"] = v
-
-        self.set_src_lang_special_tokens(src_lang)  # sets to src_lang
-        return model_inputs
-
-    def set_src_lang_special_tokens(self, src_lang) -> None:
-        """Reset the special tokens to the source lang setting. No prefix and suffix=[eos, cur_lang_code]."""
-        self.cur_lang_code = self.lang_code_to_id[src_lang]
-        self.prefix_tokens = []
-        self.suffix_tokens = [self.eos_token_id, self.cur_lang_code]
-
-    def set_tgt_lang_special_tokens(self, lang: str) -> None:
-        """Reset the special tokens to the target language setting. Prefix [tgt_lang_code], suffix =[eos]."""
-        self.cur_lang_code = self.lang_code_to_id[lang]
-        self.prefix_tokens = [self.cur_lang_code]
-        self.suffix_tokens = [self.eos_token_id]

src/transformers/tokenization_pegasus.py

@@ -1,139 +0,0 @@
-# coding=utf-8
-# Copyright 2020 Google and The HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from typing import Dict, List, Optional
-
-from transformers.tokenization_reformer import ReformerTokenizer
-
-from .file_utils import add_start_docstrings_to_callable
-from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING, BatchEncoding
-
-
-class PegasusTokenizer(ReformerTokenizer):
-    offset = 103  # entries 2-104 are only used for pretraining
-    vocab_files_names = {"vocab_file": "spiece.model"}
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # Dont use reserved words added_token_encoder, added_tokens_decoder because of
-        # AssertionError: Non-consecutive added token '1' found. in from_pretrained
-        assert len(self.added_tokens_decoder) == 0
-        self.encoder: Dict[int, str] = {0: self.pad_token, 1: self.eos_token}
-        # entries 2-104 are only used for pretraining and called unk_2, ...unk_104
-        self.encoder.update({i: f"unk_{i}" for i in range(2, self.offset + 2)})
-        self.decoder: Dict[str, int] = {v: k for k, v in self.encoder.items()}
-
-    def _convert_token_to_id(self, token: str) -> int:
-        """ Converts a token (str) in an id using the vocab. """
-        if token in self.decoder:
-            return self.decoder[token]
-        elif token in self.added_tokens_decoder:
-            return self.added_tokens_decoder[token]
-        sp_id = self.sp_model.piece_to_id(token)
-        return sp_id + self.offset
-
-    def _convert_id_to_token(self, index: int) -> str:
-        """Converts an index (integer) in a token (str) using the vocab."""
-        if index in self.encoder:
-            return self.encoder[index]
-        elif index in self.added_tokens_encoder:
-            return self.added_tokens_encoder[index]
-        else:
-            # assert index > self.offset, f"cannot decode ids between 2 and {self.offset}. Got {index}"
-            token = self.sp_model.IdToPiece(index - self.offset)
-        return token
-
-    @property
-    def vocab_size(self) -> int:
-        return len(self.sp_model) + self.offset
-
-    def get_vocab(self) -> Dict[str, int]:
-        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
-        vocab.update(self.added_tokens_encoder)
-        return vocab
-
-    def num_special_tokens_to_add(self, pair=False):
-        """Just EOS"""
-        return 1
-
-    def _special_token_mask(self, seq):
-        all_special_ids = set(self.all_special_ids)  # call it once instead of inside list comp
-        all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special
-        assert all_special_ids == set([0, 1])
-        return [1 if x in all_special_ids else 0 for x in seq]
-
-    def get_special_tokens_mask(
-        self, token_ids_0: List, token_ids_1: Optional[List] = None, already_has_special_tokens: bool = False
-    ) -> List[int]:
-        """Get list where entries are [1] if a token is [eos] or [pad] else 0."""
-        if already_has_special_tokens:
-            return self._special_token_mask(token_ids_0)
-        elif token_ids_1 is None:
-            return self._special_token_mask(token_ids_0) + [1]
-        else:
-            return self._special_token_mask(token_ids_0 + token_ids_1) + [1]
-
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
-        """
-        Build model inputs from a sequence by adding eos to the end. no bos token is added to the front.
-        - single sequence: ``X </s>``
-        - pair of sequences: ``A B </s>``  (not intended use)
-
-        Args:
-            token_ids_0 (:obj:`List[int]`):
-                List of IDs to which the special tokens will be added
-            token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`):
-                Optional second list of IDs for sequence pairs.
-
-        Returns:
-            :obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
-        """
-        if token_ids_1 is None:
-            return token_ids_0 + [self.eos_token_id]
-        # We don't expect to process pairs, but leave the pair logic for API consistency
-        return token_ids_0 + token_ids_1 + [self.eos_token_id]
-
-    @add_start_docstrings_to_callable(PREPARE_SEQ2SEQ_BATCH_DOCSTRING)
-    def prepare_seq2seq_batch(
-        self,
-        src_texts: List[str],
-        tgt_texts: Optional[List[str]] = None,
-        max_length: Optional[int] = None,
-        max_target_length: Optional[int] = None,
-        return_tensors: str = "pt",
-        truncation=True,
-        padding="longest",
-    ) -> BatchEncoding:
-        """
-        Prepare model inputs for summarization or translation.
-
-        """
-        if "" in src_texts:
-            raise ValueError(f"found empty string in src_texts: {src_texts}")
-        tokenizer_kwargs = dict(
-            add_special_tokens=True,
-            return_tensors=return_tensors,
-            max_length=max_length,
-            truncation=truncation,
-            padding=padding,
-        )
-        model_inputs: BatchEncoding = self(src_texts, **tokenizer_kwargs)
-        if tgt_texts is None:
-            return model_inputs
-        if max_target_length is not None:
-            tokenizer_kwargs["max_length"] = max_target_length
-        decoder_inputs: BatchEncoding = self(tgt_texts, **tokenizer_kwargs)
-        for k, v in decoder_inputs.items():
-            model_inputs[f"decoder_{k}"] = v
-        return model_inputs

src/transformers/tokenization_utils_base.py

@@ -553,12 +553,11 @@ class BatchEncoding(UserDict):
 
                 tensor = as_tensor(value)
 
-                # Removing this for now in favor of controling the shape with `prepend_batch_axis`
-                # # at-least2d
-                # if tensor.ndim > 2:
-                #     tensor = tensor.squeeze(0)
-                # elif tensor.ndim < 2:
-                #     tensor = tensor[None, :]
+                # at-least2d
+                if tensor.ndim > 2:
+                    tensor = tensor.squeeze(0)
+                elif tensor.ndim < 2:
+                    tensor = tensor[None, :]
 
                 self[key] = tensor
             except:  # noqa E722
@@ -590,6 +589,43 @@ class BatchEncoding(UserDict):
         return self
 
 
+# class AddedToken(UserString):
+#     """ AddedToken represents a token to be added to a Tokenizer
+
+#         An AddedToken can have special options defining the way it should behave.
+
+#         Args:
+#             content: str:
+#                 The content of the token
+
+#             single_word: bool
+#                 Whether this token should only match against single word. If True,
+#                 this token will never match inside of a word.
+
+#             lstrip: bool
+#                 Whether this token should strip all potential whitespaces on the left side.
+#                 If True, this token will greedily match any whitespace on the left and then strip
+#                 them out.
+
+#             rstrip: bool
+#                 Whether this token should strip all potential whitespaces on the right side.
+#                 If True, this token will greedily match any whitespace on the right and then strip
+#                 them out.
+#     """
+
+#     def __init__(
+#         self, data: str, single_word: bool = False, lstrip: bool = False, rstrip: bool = False,
+#     ):
+#         super().__init__(data)
+
+#         self._single_word = single_word
+#         self._lstrip = lstrip
+#         self._rstrip = rstrip
+
+#     def lower(self):
+#         return AddedToken(self.data.lower(), self._single_word, self._lstrip, self._rstrip)
+
+
 class SpecialTokensMixin:
     """
     A mixin derived by :class:`~transformers.PreTrainedTokenizer` and :class:`~transformers.PreTrainedTokenizerFast`
@@ -1213,67 +1249,6 @@ INIT_TOKENIZER_DOCSTRING = r"""
 """
 
 
-PREPARE_SEQ2SEQ_BATCH_DOCSTRING = """
-
-        Arguments:
-            src_texts: (:obj:`list`):
-                list of documents to summarize or source language texts
-            tgt_texts: (:obj:`list`, `optional`):
-                list of tgt language texts or summaries.
-            max_length (:obj:`int`, `optional`):
-                Controls the maximum length for encoder inputs (documents to summarize or source language texts)
-                If left unset or set to :obj:`None`, this will use the predefined model maximum length if a maximum
-                length is required by one of the truncation/padding parameters. If the model has no specific maximum
-                input length (like XLNet) truncation/padding to a maximum length will be deactivated.
-            max_target_length (:obj:`int`, `optional`):
-                Controls the maximum length of decoder inputs (target language texts or summaries)
-                If left unset or set to :obj:`None`, this will use the max_length value.
-            padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`False`):
-                Activates and controls padding. Accepts the following values:
-
-                * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a
-                  single sequence if provided).
-                * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
-                  maximum acceptable input length for the model if that argument is not provided.
-                * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
-                  different lengths).
-            return_tensors (:obj:`str` or :class:`~transformers.tokenization_utils_base.TensorType`, `optional`, defaults to "pt"):
-                If set, will return tensors instead of list of python integers. Acceptable values are:
-
-                * :obj:`'tf'`: Return TensorFlow :obj:`tf.constant` objects.
-                * :obj:`'pt'`: Return PyTorch :obj:`torch.Tensor` objects.
-                * :obj:`'np'`: Return Numpy :obj:`np.ndarray` objects.
-            truncation (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.TruncationStrategy`, `optional`, defaults to :obj:`True`):
-                Activates and controls truncation. Accepts the following values:
-
-                * :obj:`True` or :obj:`'longest_first'`: Truncate to a maximum length specified with the argument
-                  :obj:`max_length` or to the maximum acceptable input length for the model if that argument is not
-                  provided. This will truncate token by token, removing a token from the longest sequence in the pair
-                  if a pair of sequences (or a batch of pairs) is provided.
-                * :obj:`'only_first'`: Truncate to a maximum length specified with the argument :obj:`max_length` or to
-                  the maximum acceptable input length for the model if that argument is not provided. This will only
-                  truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
-                * :obj:`'only_second'`: Truncate to a maximum length specified with the argument :obj:`max_length` or
-                  to the maximum acceptable input length for the model if that argument is not provided. This will only
-                  truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
-                * :obj:`False` or :obj:`'do_not_truncate'` (default): No truncation (i.e., can output batch with
-                  sequence lengths greater than the model maximum admissible input size).
-
-        Return:
-            :class:`~transformers.BatchEncoding`: A :class:`~transformers.BatchEncoding` with the following fields:
-
-            - **input_ids** -- List of token ids to be fed to the encoder.
-            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model.
-            - **decoder_input_ids** -- List of token ids to be fed to the decoder.
-            - **decoder_attention_mask** -- List of indices specifying which tokens should be attended to by the decoder.
-                This does not include causal mask, which is built by the model.
-
-            The full set of keys ``[input_ids, attention_mask, decoder_input_ids,  decoder_attention_mask]``,
-            will only be returned if tgt_texts is passed. Otherwise, input_ids, attention_mask will be the only keys.
-
-"""
-
-
 @add_end_docstrings(INIT_TOKENIZER_DOCSTRING)
 class PreTrainedTokenizerBase(SpecialTokensMixin):
     """
@@ -1587,8 +1562,6 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
             for key, value in special_tokens_map.items():
                 if isinstance(value, dict):
                     value = AddedToken(**value)
-                elif isinstance(value, list):
-                    value = [AddedToken(**token) if isinstance(token, dict) else token for token in value]
                 setattr(tokenizer, key, value)
 
         # Add supplementary tokens.
@@ -1660,10 +1633,6 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
             for key, value in self.special_tokens_map_extended.items():
                 if isinstance(value, AddedToken):
                     write_dict[key] = value.__getstate__()
-                elif isinstance(value, list):
-                    write_dict[key] = [
-                        token.__getstate__() if isinstance(token, AddedToken) else token for token in value
-                    ]
                 else:
                     write_dict[key] = value
             f.write(json.dumps(write_dict, ensure_ascii=False))
@@ -2189,21 +2158,12 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
         Padding side (left/right) padding token ids are defined at the tokenizer level
         (with ``self.padding_side``, ``self.pad_token_id`` and ``self.pad_token_type_id``)
 
-        .. note::
-
-            If the ``encoded_inputs`` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the
-            result will use the same type unless you provide a different tensor type with ``return_tensors``. In the
-            case of PyTorch tensors, you will lose the specific device of your tensors however.
-
         Args:
             encoded_inputs (:class:`~transformers.BatchEncoding`, list of :class:`~transformers.BatchEncoding`, :obj:`Dict[str, List[int]]`, :obj:`Dict[str, List[List[int]]` or :obj:`List[Dict[str, List[int]]]`):
                 Tokenized inputs. Can represent one input (:class:`~transformers.BatchEncoding` or
                 :obj:`Dict[str, List[int]]`) or a batch of tokenized inputs (list of
                 :class:`~transformers.BatchEncoding`, `Dict[str, List[List[int]]]` or `List[Dict[str, List[int]]]`) so
                 you can use this method during preprocessing as well as in a PyTorch Dataloader collate function.
-
-                Instead of :obj:`List[int]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow tensors),
-                see the note above for the return type.
             padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`False`):
                  Select a strategy to pad the returned sequences (according to the model's padding side and padding
                  index) among:
@@ -2236,7 +2196,6 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
                 Whether or not to print informations and warnings.
         """
         # If we have a list of dicts, let's convert it in a dict of lists
-        # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
         if isinstance(encoded_inputs, (list, tuple)) and isinstance(encoded_inputs[0], (dict, BatchEncoding)):
             encoded_inputs = {key: [example[key] for example in encoded_inputs] for key in encoded_inputs[0].keys()}
 
@@ -2251,40 +2210,6 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
                 encoded_inputs["attention_mask"] = []
             return encoded_inputs
 
-        # If we have PyTorch/TF/NumPy tensors/arrays as inputs, we cast them as python objects
-        # and rebuild them afterwards if no return_tensors is specified
-        # Note that we lose the specific device the tensor may be on for PyTorch
-        first_element = encoded_inputs["input_ids"][0]
-        if isinstance(first_element, (list, tuple)) and first_element:
-            first_element = first_element[0]
-        if not isinstance(first_element, int):
-            if is_tf_available() and isinstance(first_element, tf.Tensor):
-                return_tensors = "tf" if return_tensors is None else return_tensors
-            elif is_torch_available() and isinstance(first_element, torch.Tensor):
-                return_tensors = "pt" if return_tensors is None else return_tensors
-            elif isinstance(first_element, np.ndarray):
-                return_tensors = "np" if return_tensors is None else return_tensors
-            else:
-                raise ValueError(
-                    f"type of {first_element} unknown: {type(first_element)}. "
-                    f"Should be one of a python, numpy, pytorch or tensorflow object."
-                )
-
-            def to_py_obj(obj):
-                if isinstance(obj, (list, tuple)):
-                    return [to_py_obj(o) for o in obj]
-                elif is_tf_available() and isinstance(obj, tf.Tensor):
-                    return obj.numpy().tolist()
-                elif is_torch_available() and isinstance(obj, torch.Tensor):
-                    return obj.cpu().tolist()
-                elif isinstance(obj, np.ndarray):
-                    return obj.tolist()
-                else:
-                    return obj
-
-            for key, value in encoded_inputs.items():
-                encoded_inputs[key] = to_py_obj(value)
-
         # Convert padding_strategy in PaddingStrategy
         padding_strategy, _, max_length, _ = self._get_padding_truncation_strategies(
             padding=padding, max_length=max_length, verbose=verbose

src/transformers/trainer.py

@@ -159,8 +159,6 @@ class Trainer:
             A tuple containing the optimizer and the scheduler to use. Will default to an instance of
             :class:`~transformers.AdamW` on your model and a scheduler given by
             :func:`~transformers.get_linear_schedule_with_warmup` controlled by :obj:`args`.
-        kwargs:
-            Deprecated keyword arguments.
     """
 
     def __init__(
@@ -171,9 +169,9 @@ class Trainer:
         train_dataset: Optional[Dataset] = None,
         eval_dataset: Optional[Dataset] = None,
         compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None,
+        prediction_loss_only=False,
         tb_writer: Optional["SummaryWriter"] = None,
         optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (None, None),
-        **kwargs,
     ):
         self.model = model.to(args.device)
         self.args = args
@@ -181,16 +179,9 @@ class Trainer:
         self.train_dataset = train_dataset
         self.eval_dataset = eval_dataset
         self.compute_metrics = compute_metrics
+        self.prediction_loss_only = prediction_loss_only
         self.optimizer, self.lr_scheduler = optimizers
         self.tb_writer = tb_writer
-        if "prediction_loss_only" in kwargs:
-            warnings.warn(
-                "Passing `prediction_loss_only` as a keyword argument is deprecated and won't be possible in a future version. Use `args.prediction_loss_only` instead.",
-                FutureWarning,
-            )
-            self.args.prediction_loss_only = kwargs.pop("prediction_loss_only")
-        assert kwargs == {}, f"Unexpected keyword arguments: {list(kwargs.keys())}."
-
         if tb_writer is None and is_tensorboard_available() and self.is_world_process_zero():
             self.tb_writer = SummaryWriter(log_dir=self.args.logging_dir)
         if not is_tensorboard_available():
@@ -633,14 +624,8 @@ class Trainer:
                 train_iterator.close()
                 break
             if self.args.tpu_metrics_debug or self.args.debug:
-                if is_torch_tpu_available():
-                    # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
-                    xm.master_print(met.metrics_report())
-                else:
-                    logger.warning(
-                        "You enabled PyTorch/XLA debug metrics but you don't have a TPU "
-                        "configured. Check your training configuration if this is unexpected."
-                    )
+                # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
+                xm.master_print(met.metrics_report())
 
         if self.tb_writer:
             self.tb_writer.close()
@@ -960,9 +945,7 @@ class Trainer:
             )
             return self._prediction_loop(dataloader, description, prediction_loss_only=prediction_loss_only)
 
-        prediction_loss_only = (
-            prediction_loss_only if prediction_loss_only is not None else self.args.prediction_loss_only
-        )
+        prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else self.prediction_loss_only
 
         model = self.model
         # multi-gpu eval

src/transformers/trainer_tf.py

@@ -44,6 +44,8 @@ class TFTrainer:
         compute_metrics (:obj:`Callable[[EvalPrediction], Dict]`, `optional`):
             The function that will be used to compute metrics at evaluation. Must take a
             :class:`~transformers.EvalPrediction` and return a dictionary string to metric values.
+        prediction_loss_only (:obj:`bool`, `optional`, defaults to `False`):
+            When performing evaluation and predictions, only returns the loss.
         tb_writer (:obj:`tf.summary.SummaryWriter`, `optional`):
             Object to write to TensorBoard.
         optimizers (:obj:`Tuple[tf.keras.optimizers.Optimizer, tf.keras.optimizers.schedules.LearningRateSchedule]`, `optional`):
@@ -52,8 +54,6 @@ class TFTrainer:
             :class:`~transformers.AdamWeightDecay`. The scheduler will default to an instance of
             :class:`tf.keras.optimizers.schedules.PolynomialDecay` if :obj:`args.num_warmup_steps` is 0 else
             an instance of :class:`~transformers.WarmUp`.
-        kwargs:
-            Deprecated keyword arguments.
     """
 
     def __init__(
@@ -63,12 +63,12 @@ class TFTrainer:
         train_dataset: Optional[tf.data.Dataset] = None,
         eval_dataset: Optional[tf.data.Dataset] = None,
         compute_metrics: Optional[Callable[[EvalPrediction], Dict]] = None,
+        prediction_loss_only=False,
         tb_writer: Optional[tf.summary.SummaryWriter] = None,
         optimizers: Tuple[tf.keras.optimizers.Optimizer, tf.keras.optimizers.schedules.LearningRateSchedule] = (
             None,
             None,
         ),
-        **kwargs,
     ):
         assert parse(tf.__version__).release >= (2, 2, 0), (
             "You need to run the TensorFlow trainer with at least the version 2.2.0, your version is %r "
@@ -80,17 +80,11 @@ class TFTrainer:
         self.train_dataset = train_dataset
         self.eval_dataset = eval_dataset
         self.compute_metrics = compute_metrics
+        self.prediction_loss_only = prediction_loss_only
         self.optimizer, self.lr_scheduler = optimizers
         self.gradient_accumulator = GradientAccumulator()
         self.global_step = 0
         self.epoch_logging = 0
-        if "prediction_loss_only" in kwargs:
-            warnings.warn(
-                "Passing `prediction_loss_only` as a keyword argument is deprecated and won't be possible in a future version. Use `args.prediction_loss_only` instead.",
-                FutureWarning,
-            )
-            self.args.prediction_loss_only = kwargs.pop("prediction_loss_only")
-        assert kwargs == {}, f"Unexpected keyword arguments: {list(kwargs.keys())}."
 
         if tb_writer is not None:
             self.tb_writer = tb_writer
@@ -288,9 +282,7 @@ class TFTrainer:
                 dataset, steps, num_examples, description, prediction_loss_only=prediction_loss_only
             )
 
-        prediction_loss_only = (
-            prediction_loss_only if prediction_loss_only is not None else self.args.prediction_loss_only
-        )
+        prediction_loss_only = prediction_loss_only if prediction_loss_only is not None else self.prediction_loss_only
 
         logger.info("***** Running %s *****", description)
         logger.info("  Num examples = %d", num_examples)

src/transformers/training_args.py

@@ -52,8 +52,6 @@ class TrainingArguments:
             Whether to run predictions on the test set or not.
         evaluate_during_training (:obj:`bool`, `optional`, defaults to :obj:`False`):
             Whether to run evaluation during training at each logging step or not.
-        prediction_loss_only (:obj:`bool`, `optional`, defaults to `False`):
-            When performing evaluation and predictions, only returns the loss.
         per_device_train_batch_size (:obj:`int`, `optional`, defaults to 8):
             The batch size per GPU/TPU core/CPU for training.
         per_device_eval_batch_size (:obj:`int`, `optional`, defaults to 8):
@@ -134,9 +132,6 @@ class TrainingArguments:
     evaluate_during_training: bool = field(
         default=False, metadata={"help": "Run evaluation during training at each logging step."},
     )
-    prediction_loss_only: bool = field(
-        default=False, metadata={"help": "When performing evaluation and predictions, only returns the loss."},
-    )
 
     per_device_train_batch_size: int = field(
         default=8, metadata={"help": "Batch size per GPU/TPU core/CPU for training."}

templates/adding_a_new_model/tests/test_modeling_tf_xxx.py

@@ -148,10 +148,10 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
 
             result = model(input_ids)
 
-            self.parent.assertEqual(
-                result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)
+            self.parent.assertListEqual(
+                list(result["last_hidden_state"].shape), [self.batch_size, self.seq_length, self.hidden_size]
             )
-            self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))
+            self.parent.assertListEqual(list(result["pooler_output"].shape), [self.batch_size, self.hidden_size])
 
         def create_and_check_xxx_for_masked_lm(
             self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
@@ -159,7 +159,9 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForMaskedLM(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             result = model(inputs)
-            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
+            self.parent.assertListEqual(
+                list(result["logits"].shape), [self.batch_size, self.seq_length, self.vocab_size]
+            )
 
         def create_and_check_xxx_for_sequence_classification(
             self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
@@ -168,7 +170,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForSequenceClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             result = model(inputs)
-            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))
+            self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_labels])
 
         def create_and_check_bert_for_multiple_choice(
             self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
@@ -184,7 +186,7 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
                 "token_type_ids": multiple_choice_token_type_ids,
             }
             result = model(inputs)
-            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))
+            self.parent.assertListEqual(list(result["logits"].shape), [self.batch_size, self.num_choices])
 
         def create_and_check_xxx_for_token_classification(
             self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
@@ -193,7 +195,9 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForTokenClassification(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             result = model(inputs)
-            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
+            self.parent.assertListEqual(
+                list(result["logits"].shape), [self.batch_size, self.seq_length, self.num_labels]
+            )
 
         def create_and_check_xxx_for_question_answering(
             self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
@@ -201,8 +205,8 @@ class TFXxxModelTest(TFModelTesterMixin, unittest.TestCase):
             model = TFXxxForQuestionAnswering(config=config)
             inputs = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
             result = model(inputs)
-            self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
-            self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))
+            self.parent.assertListEqual(list(result["start_logits"].shape), [self.batch_size, self.seq_length])
+            self.parent.assertListEqual(list(result["end_logits"].shape), [self.batch_size, self.seq_length])
 
         def prepare_config_and_inputs_for_common(self):
             config_and_inputs = self.prepare_config_and_inputs()

tests/test_modeling_bart.py

@@ -45,7 +45,6 @@ if is_torch_available():
         _prepare_bart_decoder_inputs,
         SinusoidalPositionalEmbedding,
     )
-PGE_ARTICLE = """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."""
 
 
 @require_torch
@@ -479,6 +478,7 @@ class BartModelIntegrationTests(unittest.TestCase):
         self.assertFalse(model.config.is_valid_mbart())
         tok = BartTokenizer.from_pretrained("facebook/bart-large")
 
+        PGE_ARTICLE = """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."""
         EXPECTED_SUMMARY = "California's largest power company has begun shutting off power to tens of thousands of homes and businesses in the state."
         dct = tok.batch_encode_plus(
             [PGE_ARTICLE], max_length=1024, padding="max_length", truncation=True, return_tensors="pt",

tests/test_modeling_bert.py

@@ -152,7 +152,7 @@ class BertModelTester:
             encoder_attention_mask,
         )
 
-    def create_and_check_model(
+    def create_and_check_bert_model(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):
         model = BertModel(config=config)
@@ -164,7 +164,7 @@ class BertModelTester:
         self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
         self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))
 
-    def create_and_check_model_as_decoder(
+    def create_and_check_bert_model_as_decoder(
         self,
         config,
         input_ids,
@@ -176,7 +176,6 @@ class BertModelTester:
         encoder_hidden_states,
         encoder_attention_mask,
     ):
-        config.add_cross_attention = True
         model = BertModel(config)
         model.to(torch_device)
         model.eval()
@@ -197,7 +196,7 @@ class BertModelTester:
         self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
         self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))
 
-    def create_and_check_for_causal_lm(
+    def create_and_check_bert_for_causal_lm(
         self,
         config,
         input_ids,
@@ -215,7 +214,7 @@ class BertModelTester:
         result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
         self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
 
-    def create_and_check_for_masked_lm(
+    def create_and_check_bert_for_masked_lm(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):
         model = BertForMaskedLM(config=config)
@@ -224,7 +223,7 @@ class BertModelTester:
         result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
         self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
 
-    def create_and_check_model_for_causal_lm_as_decoder(
+    def create_and_check_bert_model_for_causal_lm_as_decoder(
         self,
         config,
         input_ids,
@@ -236,7 +235,6 @@ class BertModelTester:
         encoder_hidden_states,
         encoder_attention_mask,
     ):
-        config.add_cross_attention = True
         model = BertLMHeadModel(config=config)
         model.to(torch_device)
         model.eval()
@@ -257,7 +255,7 @@ class BertModelTester:
         )
         self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
 
-    def create_and_check_for_next_sequence_prediction(
+    def create_and_check_bert_for_next_sequence_prediction(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):
         model = BertForNextSentencePrediction(config=config)
@@ -268,7 +266,7 @@ class BertModelTester:
         )
         self.parent.assertEqual(result.logits.shape, (self.batch_size, 2))
 
-    def create_and_check_for_pretraining(
+    def create_and_check_bert_for_pretraining(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):
         model = BertForPreTraining(config=config)
@@ -284,7 +282,7 @@ class BertModelTester:
         self.parent.assertEqual(result.prediction_logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
         self.parent.assertEqual(result.seq_relationship_logits.shape, (self.batch_size, 2))
 
-    def create_and_check_for_question_answering(
+    def create_and_check_bert_for_question_answering(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):
         model = BertForQuestionAnswering(config=config)
@@ -300,7 +298,7 @@ class BertModelTester:
         self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
         self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))
 
-    def create_and_check_for_sequence_classification(
+    def create_and_check_bert_for_sequence_classification(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):
         config.num_labels = self.num_labels
@@ -310,7 +308,7 @@ class BertModelTester:
         result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels)
         self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))
 
-    def create_and_check_for_token_classification(
+    def create_and_check_bert_for_token_classification(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):
         config.num_labels = self.num_labels
@@ -320,7 +318,7 @@ class BertModelTester:
         result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
         self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
 
-    def create_and_check_for_multiple_choice(
+    def create_and_check_bert_for_multiple_choice(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):
         config.num_choices = self.num_choices
@@ -370,7 +368,6 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
         if is_torch_available()
         else ()
     )
-    test_chunking = True
 
     def setUp(self):
         self.model_tester = BertModelTester(self)
@@ -379,15 +376,15 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
     def test_config(self):
         self.config_tester.run_common_tests()
 
-    def test_model(self):
+    def test_bert_model(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_model(*config_and_inputs)
+        self.model_tester.create_and_check_bert_model(*config_and_inputs)
 
-    def test_model_as_decoder(self):
+    def test_bert_model_as_decoder(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
-        self.model_tester.create_and_check_model_as_decoder(*config_and_inputs)
+        self.model_tester.create_and_check_bert_model_as_decoder(*config_and_inputs)
 
-    def test_model_as_decoder_with_default_input_mask(self):
+    def test_bert_model_as_decoder_with_default_input_mask(self):
         # This regression test was failing with PyTorch < 1.3
         (
             config,
@@ -403,7 +400,7 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
 
         input_mask = None
 
-        self.model_tester.create_and_check_model_as_decoder(
+        self.model_tester.create_and_check_bert_model_as_decoder(
             config,
             input_ids,
             token_type_ids,
@@ -417,39 +414,39 @@ class BertModelTest(ModelTesterMixin, unittest.TestCase):
 
     def test_for_causal_lm(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
-        self.model_tester.create_and_check_for_causal_lm(*config_and_inputs)
+        self.model_tester.create_and_check_bert_for_causal_lm(*config_and_inputs)
 
     def test_for_masked_lm(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_masked_lm(*config_and_inputs)
+        self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs)
 
     def test_for_causal_lm_decoder(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
-        self.model_tester.create_and_check_model_for_causal_lm_as_decoder(*config_and_inputs)
+        self.model_tester.create_and_check_bert_model_for_causal_lm_as_decoder(*config_and_inputs)
 
     def test_for_multiple_choice(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs)
+        self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs)
 
     def test_for_next_sequence_prediction(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_next_sequence_prediction(*config_and_inputs)
+        self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs)
 
     def test_for_pretraining(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_pretraining(*config_and_inputs)
+        self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs)
 
     def test_for_question_answering(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_question_answering(*config_and_inputs)
+        self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs)
 
     def test_for_sequence_classification(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs)
+        self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs)
 
     def test_for_token_classification(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        self.model_tester.create_and_check_for_token_classification(*config_and_inputs)
+        self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs)
 
     @slow
     def test_model_from_pretrained(self):

tests/test_modeling_common.py

@@ -18,7 +18,7 @@ import os.path
 import random
 import tempfile
 import unittest
-from typing import List, Tuple
+from typing import List
 
 from transformers import is_torch_available
 from transformers.testing_utils import require_multigpu, require_torch, slow, torch_device
@@ -37,11 +37,6 @@ if is_torch_available():
         BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
         MODEL_FOR_QUESTION_ANSWERING_MAPPING,
-        MODEL_FOR_CAUSAL_LM_MAPPING,
-        MODEL_FOR_MASKED_LM_MAPPING,
-        MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
-        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
-        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
         top_k_top_p_filtering,
     )
 
@@ -65,42 +60,16 @@ class ModelTesterMixin:
     test_resize_embeddings = True
     test_head_masking = True
     test_missing_keys = True
-    test_chunking = False
     is_encoder_decoder = False
 
-    def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
-        inputs_dict = copy.deepcopy(inputs_dict)
+    def _prepare_for_class(self, inputs_dict, model_class):
         if model_class in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.values():
-            inputs_dict = {
+            return {
                 k: v.unsqueeze(1).expand(-1, self.model_tester.num_choices, -1).contiguous()
                 if isinstance(v, torch.Tensor) and v.ndim > 1
                 else v
                 for k, v in inputs_dict.items()
             }
-
-        if return_labels:
-            if model_class in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.values():
-                inputs_dict["labels"] = torch.ones(self.model_tester.batch_size, dtype=torch.long, device=torch_device)
-            elif model_class in MODEL_FOR_QUESTION_ANSWERING_MAPPING.values():
-                inputs_dict["start_positions"] = torch.zeros(
-                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
-                )
-                inputs_dict["end_positions"] = torch.zeros(
-                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
-                )
-            elif model_class in MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.values():
-                inputs_dict["labels"] = torch.zeros(
-                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
-                )
-            elif model_class in [
-                *MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.values(),
-                *MODEL_FOR_CAUSAL_LM_MAPPING.values(),
-                *MODEL_FOR_MASKED_LM_MAPPING.values(),
-                *MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.values(),
-            ]:
-                inputs_dict["labels"] = torch.zeros(
-                    (self.model_tester.batch_size, self.model_tester.seq_length), dtype=torch.long, device=torch_device
-                )
         return inputs_dict
 
     def test_save_load(self):
@@ -550,29 +519,6 @@ class ModelTesterMixin:
 
             check_hidden_states_output(inputs_dict, config, model_class)
 
-    def test_feed_forward_chunking(self):
-        (original_config, inputs_dict,) = self.model_tester.prepare_config_and_inputs_for_common()
-        if not self.test_chunking:
-            return
-
-        for model_class in self.all_model_classes:
-            torch.manual_seed(0)
-            config = copy.deepcopy(original_config)
-            model = model_class(config)
-            model.to(torch_device)
-            model.eval()
-
-            hidden_states_no_chunk = model(**self._prepare_for_class(inputs_dict, model_class))[0]
-
-            torch.manual_seed(0)
-            config.chunk_size_feed_forward = 1
-            model = model_class(config)
-            model.to(torch_device)
-            model.eval()
-
-            hidden_states_with_chunk = model(**self._prepare_for_class(inputs_dict, model_class))[0]
-            self.assertTrue(torch.allclose(hidden_states_no_chunk, hidden_states_with_chunk, atol=1e-3))
-
     def test_resize_tokens_embeddings(self):
         (original_config, inputs_dict,) = self.model_tester.prepare_config_and_inputs_for_common()
         if not self.test_resize_embeddings:
@@ -693,66 +639,6 @@ class ModelTesterMixin:
             # self.assertTrue(model.transformer.wte.weight.shape, model.lm_head.weight.shape)
             # self.assertTrue(check_same_values(model.transformer.wte, model.lm_head))
 
-    def test_model_outputs_equivalence(self):
-
-        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
-
-        def check_equivalence(model, tuple_inputs, dict_inputs, additional_kwargs={}):
-            with torch.no_grad():
-                tuple_output = model(**tuple_inputs, return_dict=False, **additional_kwargs)
-                dict_output = model(**dict_inputs, return_dict=True, **additional_kwargs).to_tuple()
-
-                def recursive_check(tuple_object, dict_object):
-                    if isinstance(tuple_object, (List, Tuple)):
-                        for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object):
-                            recursive_check(tuple_iterable_value, dict_iterable_value)
-                    elif tuple_object is None:
-                        return
-                    else:
-                        tuple_object.clamp_(-1e12, 1e12)
-                        dict_object.clamp_(-1e12, 1e12)
-                        self.assertTrue(
-                            torch.allclose(tuple_object, dict_object, atol=1e-5),
-                            msg=f"Tuple and dict output are not equal. Difference: {torch.max(torch.abs(tuple_object - dict_object))}",
-                        )
-
-                recursive_check(tuple_output, dict_output)
-
-        for model_class in self.all_model_classes:
-            model = model_class(config)
-            model.to(torch_device)
-            model.eval()
-
-            tuple_inputs = self._prepare_for_class(inputs_dict, model_class)
-            dict_inputs = self._prepare_for_class(inputs_dict, model_class)
-            check_equivalence(model, tuple_inputs, dict_inputs)
-
-            tuple_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
-            dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
-            check_equivalence(model, tuple_inputs, dict_inputs)
-
-            tuple_inputs = self._prepare_for_class(inputs_dict, model_class)
-            dict_inputs = self._prepare_for_class(inputs_dict, model_class)
-            check_equivalence(model, tuple_inputs, dict_inputs, {"output_hidden_states": True})
-
-            tuple_inputs = self._prepare_for_class(inputs_dict, model_class)
-            dict_inputs = self._prepare_for_class(inputs_dict, model_class)
-            check_equivalence(model, tuple_inputs, dict_inputs, {"output_attentions": True})
-
-            tuple_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
-            dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
-            check_equivalence(model, tuple_inputs, dict_inputs, {"output_hidden_states": True})
-
-            tuple_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
-            dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
-            check_equivalence(model, tuple_inputs, dict_inputs, {"output_attentions": True})
-
-            tuple_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
-            dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
-            check_equivalence(
-                model, tuple_inputs, dict_inputs, {"output_hidden_states": True, "output_attentions": True}
-            )
-
     def test_inputs_embeds(self):
 
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

tests/test_modeling_ctrl.py

@@ -117,7 +117,7 @@ class CTRLModelTester:
         model(input_ids, token_type_ids=token_type_ids)
         result = model(input_ids)
         self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
-        self.parent.assertEqual(len(result.past_key_values), config.n_layer)
+        self.parent.assertEqual(len(result["past_key_values"]), config.n_layer)
 
     def create_and_check_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args):
         model = CTRLLMHeadModel(config)

tests/test_modeling_encoder_decoder.py

@@ -20,38 +20,61 @@ import unittest
 from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 
+# TODO(PVP): this line reruns all the tests in BertModelTest; not sure whether this can be prevented
+# for now only run module with pytest tests/test_modeling_encoder_decoder.py::EncoderDecoderModelTest
 from .test_modeling_bert import BertModelTester
 from .test_modeling_common import ids_tensor
-from .test_modeling_gpt2 import GPT2ModelTester
-from .test_modeling_roberta import RobertaModelTester
 
 
 if is_torch_available():
-    from transformers import (
-        BertModel,
-        BertLMHeadModel,
-        GPT2LMHeadModel,
-        RobertaModel,
-        RobertaForCausalLM,
-        EncoderDecoderModel,
-        EncoderDecoderConfig,
-    )
+    from transformers import BertModel, EncoderDecoderModel, EncoderDecoderConfig
+    from transformers.modeling_bert import BertLMHeadModel
     import numpy as np
     import torch
 
 
 @require_torch
-class EncoderDecoderMixin:
-    def get_encoder_decoder_model(self, config, decoder_config):
-        pass
+class EncoderDecoderModelTest(unittest.TestCase):
+    def prepare_config_and_inputs_bert(self):
+        bert_model_tester = BertModelTester(self)
+        encoder_config_and_inputs = bert_model_tester.prepare_config_and_inputs()
+        decoder_config_and_inputs = bert_model_tester.prepare_config_and_inputs_for_decoder()
+        (
+            config,
+            input_ids,
+            token_type_ids,
+            input_mask,
+            sequence_labels,
+            token_labels,
+            choice_labels,
+        ) = encoder_config_and_inputs
+        (
+            decoder_config,
+            decoder_input_ids,
+            decoder_token_type_ids,
+            decoder_input_mask,
+            decoder_sequence_labels,
+            decoder_token_labels,
+            decoder_choice_labels,
+            encoder_hidden_states,
+            encoder_attention_mask,
+        ) = decoder_config_and_inputs
+        return {
+            "config": config,
+            "input_ids": input_ids,
+            "attention_mask": input_mask,
+            "decoder_config": decoder_config,
+            "decoder_input_ids": decoder_input_ids,
+            "decoder_token_type_ids": decoder_token_type_ids,
+            "decoder_attention_mask": decoder_input_mask,
+            "decoder_sequence_labels": decoder_sequence_labels,
+            "decoder_token_labels": decoder_token_labels,
+            "decoder_choice_labels": decoder_choice_labels,
+            "encoder_hidden_states": encoder_hidden_states,
+            "labels": decoder_token_labels,
+        }
 
-    def prepare_config_and_inputs(self):
-        pass
-
-    def get_pretrained_model(self):
-        pass
-
-    def check_encoder_decoder_model_from_pretrained_configs(
+    def create_and_check_bert_encoder_decoder_model_from_pretrained_configs(
         self,
         config,
         input_ids,
@@ -81,7 +104,7 @@ class EncoderDecoderMixin:
         self.assertEqual(outputs_encoder_decoder[0].shape, (decoder_input_ids.shape + (decoder_config.vocab_size,)))
         self.assertEqual(outputs_encoder_decoder[1].shape, (input_ids.shape + (config.hidden_size,)))
 
-    def check_encoder_decoder_model(
+    def create_and_check_bert_encoder_decoder_model(
         self,
         config,
         input_ids,
@@ -92,10 +115,10 @@ class EncoderDecoderMixin:
         decoder_attention_mask,
         **kwargs
     ):
-        encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config)
+        encoder_model = BertModel(config)
+        decoder_model = BertLMHeadModel(decoder_config)
         enc_dec_model = EncoderDecoderModel(encoder=encoder_model, decoder=decoder_model)
         self.assertTrue(enc_dec_model.config.decoder.is_decoder)
-        self.assertTrue(enc_dec_model.config.decoder.add_cross_attention)
         self.assertTrue(enc_dec_model.config.is_encoder_decoder)
         enc_dec_model.to(torch_device)
         outputs_encoder_decoder = enc_dec_model(
@@ -118,7 +141,7 @@ class EncoderDecoderMixin:
         self.assertEqual(outputs_encoder_decoder[0].shape, (decoder_input_ids.shape + (decoder_config.vocab_size,)))
         self.assertEqual(outputs_encoder_decoder[1].shape, (input_ids.shape + (config.hidden_size,)))
 
-    def check_encoder_decoder_model_from_pretrained(
+    def create_and_check_bert_encoder_decoder_model_from_pretrained(
         self,
         config,
         input_ids,
@@ -129,7 +152,8 @@ class EncoderDecoderMixin:
         decoder_attention_mask,
         **kwargs
     ):
-        encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config)
+        encoder_model = BertModel(config)
+        decoder_model = BertLMHeadModel(decoder_config)
         kwargs = {"encoder_model": encoder_model, "decoder_model": decoder_model}
         enc_dec_model = EncoderDecoderModel.from_encoder_decoder_pretrained(**kwargs)
         enc_dec_model.to(torch_device)
@@ -143,7 +167,7 @@ class EncoderDecoderMixin:
         self.assertEqual(outputs_encoder_decoder[0].shape, (decoder_input_ids.shape + (decoder_config.vocab_size,)))
         self.assertEqual(outputs_encoder_decoder[1].shape, (input_ids.shape + (config.hidden_size,)))
 
-    def check_save_and_load(
+    def create_and_check_save_and_load(
         self,
         config,
         input_ids,
@@ -154,7 +178,8 @@ class EncoderDecoderMixin:
         decoder_attention_mask,
         **kwargs
     ):
-        encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config)
+        encoder_model = BertModel(config)
+        decoder_model = BertLMHeadModel(decoder_config)
         enc_dec_model = EncoderDecoderModel(encoder=encoder_model, decoder=decoder_model)
         enc_dec_model.to(torch_device)
         enc_dec_model.eval()
@@ -183,7 +208,7 @@ class EncoderDecoderMixin:
                 max_diff = np.amax(np.abs(out_1 - out_2))
                 self.assertLessEqual(max_diff, 1e-5)
 
-    def check_save_and_load_encoder_decoder_model(
+    def create_and_check_save_and_load_encoder_decoder_model(
         self,
         config,
         input_ids,
@@ -194,7 +219,8 @@ class EncoderDecoderMixin:
         decoder_attention_mask,
         **kwargs
     ):
-        encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config)
+        encoder_model = BertModel(config)
+        decoder_model = BertLMHeadModel(decoder_config)
         enc_dec_model = EncoderDecoderModel(encoder=encoder_model, decoder=decoder_model)
         enc_dec_model.to(torch_device)
         enc_dec_model.eval()
@@ -227,7 +253,7 @@ class EncoderDecoderMixin:
                 max_diff = np.amax(np.abs(out_1 - out_2))
                 self.assertLessEqual(max_diff, 1e-5)
 
-    def check_encoder_decoder_model_labels(
+    def create_and_check_bert_encoder_decoder_model_labels(
         self,
         config,
         input_ids,
@@ -239,7 +265,8 @@ class EncoderDecoderMixin:
         labels,
         **kwargs
     ):
-        encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config)
+        encoder_model = BertModel(config)
+        decoder_model = BertLMHeadModel(decoder_config)
         enc_dec_model = EncoderDecoderModel(encoder=encoder_model, decoder=decoder_model)
         enc_dec_model.to(torch_device)
         outputs_encoder_decoder = enc_dec_model(
@@ -257,8 +284,9 @@ class EncoderDecoderMixin:
         self.assertEqual(outputs_encoder_decoder[1].shape, (decoder_input_ids.shape + (decoder_config.vocab_size,)))
         self.assertEqual(outputs_encoder_decoder[2].shape, (input_ids.shape + (config.hidden_size,)))
 
-    def check_encoder_decoder_model_generate(self, input_ids, config, decoder_config, **kwargs):
-        encoder_model, decoder_model = self.get_encoder_decoder_model(config, decoder_config)
+    def create_and_check_bert_encoder_decoder_model_generate(self, input_ids, config, decoder_config, **kwargs):
+        encoder_model = BertModel(config)
+        decoder_model = BertLMHeadModel(decoder_config)
         enc_dec_model = EncoderDecoderModel(encoder=encoder_model, decoder=decoder_model)
         enc_dec_model.to(torch_device)
 
@@ -268,37 +296,47 @@ class EncoderDecoderMixin:
         )
         self.assertEqual(generated_output.shape, (input_ids.shape[0],) + (decoder_config.max_length,))
 
-    def test_encoder_decoder_model(self):
-        input_ids_dict = self.prepare_config_and_inputs()
-        self.check_encoder_decoder_model(**input_ids_dict)
+    def test_bert_encoder_decoder_model(self):
+        input_ids_dict = self.prepare_config_and_inputs_bert()
+        self.create_and_check_bert_encoder_decoder_model(**input_ids_dict)
 
-    def test_encoder_decoder_model_from_pretrained_configs(self):
-        input_ids_dict = self.prepare_config_and_inputs()
-        self.check_encoder_decoder_model_from_pretrained_configs(**input_ids_dict)
+    def test_bert_encoder_decoder_model_from_pretrained_configs(self):
+        input_ids_dict = self.prepare_config_and_inputs_bert()
+        self.create_and_check_bert_encoder_decoder_model_from_pretrained_configs(**input_ids_dict)
 
-    def test_encoder_decoder_model_from_pretrained(self):
-        input_ids_dict = self.prepare_config_and_inputs()
-        self.check_encoder_decoder_model_from_pretrained(**input_ids_dict)
+    def test_bert_encoder_decoder_model_from_pretrained(self):
+        input_ids_dict = self.prepare_config_and_inputs_bert()
+        self.create_and_check_bert_encoder_decoder_model_from_pretrained(**input_ids_dict)
 
     def test_save_and_load_from_pretrained(self):
-        input_ids_dict = self.prepare_config_and_inputs()
-        self.check_save_and_load(**input_ids_dict)
+        input_ids_dict = self.prepare_config_and_inputs_bert()
+        self.create_and_check_save_and_load(**input_ids_dict)
 
     def test_save_and_load_from_encoder_decoder_pretrained(self):
-        input_ids_dict = self.prepare_config_and_inputs()
-        self.check_save_and_load_encoder_decoder_model(**input_ids_dict)
+        input_ids_dict = self.prepare_config_and_inputs_bert()
+        self.create_and_check_save_and_load_encoder_decoder_model(**input_ids_dict)
 
-    def test_encoder_decoder_model_labels(self):
-        input_ids_dict = self.prepare_config_and_inputs()
-        self.check_encoder_decoder_model_labels(**input_ids_dict)
+    def test_bert_encoder_decoder_model_labels(self):
+        input_ids_dict = self.prepare_config_and_inputs_bert()
+        self.create_and_check_bert_encoder_decoder_model_labels(**input_ids_dict)
 
-    def test_encoder_decoder_model_generate(self):
-        input_ids_dict = self.prepare_config_and_inputs()
-        self.check_encoder_decoder_model_generate(**input_ids_dict)
+    def test_bert_encoder_decoder_model_generate(self):
+        input_ids_dict = self.prepare_config_and_inputs_bert()
+        self.create_and_check_bert_encoder_decoder_model_generate(**input_ids_dict)
 
     @slow
-    def test_real_model_save_load_from_pretrained(self):
-        model_2 = self.get_pretrained_model()
+    def test_real_bert_model_from_pretrained(self):
+        model = EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-uncased", "bert-base-uncased")
+        self.assertIsNotNone(model)
+
+    @slow
+    def test_real_bert_model_from_pretrained_has_cross_attention(self):
+        model = EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-uncased", "bert-base-uncased")
+        self.assertTrue(hasattr(model.decoder.bert.encoder.layer[0], "crossattention"))
+
+    @slow
+    def test_real_bert_model_save_load_from_pretrained(self):
+        model_2 = EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-uncased", "bert-base-uncased")
         model_2.to(torch_device)
         input_ids = ids_tensor([13, 5], model_2.config.encoder.vocab_size)
         decoder_input_ids = ids_tensor([13, 1], model_2.config.encoder.vocab_size)
@@ -320,163 +358,3 @@ class EncoderDecoderMixin:
                 out_1[np.isnan(out_1)] = 0
                 max_diff = np.amax(np.abs(out_1 - out_2))
                 self.assertLessEqual(max_diff, 1e-5)
-
-
-class BertEncoderDecoderModelTest(EncoderDecoderMixin, unittest.TestCase):
-    def get_pretrained_model(self):
-        return EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-cased", "bert-base-cased")
-
-    def get_encoder_decoder_model(self, config, decoder_config):
-        encoder_model = BertModel(config)
-        decoder_model = BertLMHeadModel(decoder_config)
-        return encoder_model, decoder_model
-
-    def prepare_config_and_inputs(self):
-        model_tester = BertModelTester(self)
-        encoder_config_and_inputs = model_tester.prepare_config_and_inputs()
-        decoder_config_and_inputs = model_tester.prepare_config_and_inputs_for_decoder()
-        (
-            config,
-            input_ids,
-            token_type_ids,
-            input_mask,
-            sequence_labels,
-            token_labels,
-            choice_labels,
-        ) = encoder_config_and_inputs
-        (
-            decoder_config,
-            decoder_input_ids,
-            decoder_token_type_ids,
-            decoder_input_mask,
-            decoder_sequence_labels,
-            decoder_token_labels,
-            decoder_choice_labels,
-            encoder_hidden_states,
-            encoder_attention_mask,
-        ) = decoder_config_and_inputs
-
-        # make sure that cross attention layers are added
-        decoder_config.add_cross_attention = True
-        return {
-            "config": config,
-            "input_ids": input_ids,
-            "attention_mask": input_mask,
-            "decoder_config": decoder_config,
-            "decoder_input_ids": decoder_input_ids,
-            "decoder_token_type_ids": decoder_token_type_ids,
-            "decoder_attention_mask": decoder_input_mask,
-            "decoder_sequence_labels": decoder_sequence_labels,
-            "decoder_token_labels": decoder_token_labels,
-            "decoder_choice_labels": decoder_choice_labels,
-            "encoder_hidden_states": encoder_hidden_states,
-            "labels": decoder_token_labels,
-        }
-
-
-class RoBertaEncoderDecoderModelTest(EncoderDecoderMixin, unittest.TestCase):
-    def get_encoder_decoder_model(self, config, decoder_config):
-        encoder_model = RobertaModel(config)
-        decoder_model = RobertaForCausalLM(decoder_config)
-        return encoder_model, decoder_model
-
-    def prepare_config_and_inputs(self):
-        model_tester = RobertaModelTester(self)
-        encoder_config_and_inputs = model_tester.prepare_config_and_inputs()
-        decoder_config_and_inputs = model_tester.prepare_config_and_inputs_for_decoder()
-        (
-            config,
-            input_ids,
-            token_type_ids,
-            input_mask,
-            sequence_labels,
-            token_labels,
-            choice_labels,
-        ) = encoder_config_and_inputs
-        (
-            decoder_config,
-            decoder_input_ids,
-            decoder_token_type_ids,
-            decoder_input_mask,
-            decoder_sequence_labels,
-            decoder_token_labels,
-            decoder_choice_labels,
-            encoder_hidden_states,
-            encoder_attention_mask,
-        ) = decoder_config_and_inputs
-
-        # make sure that cross attention layers are added
-        decoder_config.add_cross_attention = True
-        return {
-            "config": config,
-            "input_ids": input_ids,
-            "attention_mask": input_mask,
-            "decoder_config": decoder_config,
-            "decoder_input_ids": decoder_input_ids,
-            "decoder_token_type_ids": decoder_token_type_ids,
-            "decoder_attention_mask": decoder_input_mask,
-            "decoder_sequence_labels": decoder_sequence_labels,
-            "decoder_token_labels": decoder_token_labels,
-            "decoder_choice_labels": decoder_choice_labels,
-            "encoder_hidden_states": encoder_hidden_states,
-            "labels": decoder_token_labels,
-        }
-
-    def get_pretrained_model(self):
-        return EncoderDecoderModel.from_encoder_decoder_pretrained("roberta-base", "roberta-base")
-
-
-class GPT2EncoderDecoderModelTest(EncoderDecoderMixin, unittest.TestCase):
-    def get_encoder_decoder_model(self, config, decoder_config):
-        encoder_model = BertModel(config)
-        decoder_model = GPT2LMHeadModel(decoder_config)
-        return encoder_model, decoder_model
-
-    def prepare_config_and_inputs(self):
-        model_tester_encoder = BertModelTester(self, batch_size=13)
-        model_tester_decoder = GPT2ModelTester(self, batch_size=13)
-        encoder_config_and_inputs = model_tester_encoder.prepare_config_and_inputs()
-        decoder_config_and_inputs = model_tester_decoder.prepare_config_and_inputs_for_decoder()
-        (
-            config,
-            input_ids,
-            token_type_ids,
-            input_mask,
-            sequence_labels,
-            token_labels,
-            choice_labels,
-        ) = encoder_config_and_inputs
-        (
-            decoder_config,
-            decoder_input_ids,
-            decoder_input_mask,
-            decoder_head_mask,
-            decoder_token_type_ids,
-            decoder_sequence_labels,
-            decoder_token_labels,
-            decoder_choice_labels,
-            encoder_hidden_states,
-            encoder_attention_mask,
-        ) = decoder_config_and_inputs
-
-        # make sure that cross attention layers are added
-        decoder_config.add_cross_attention = True
-        #  disable cache for now
-        decoder_config.use_cache = False
-        return {
-            "config": config,
-            "input_ids": input_ids,
-            "attention_mask": input_mask,
-            "decoder_config": decoder_config,
-            "decoder_input_ids": decoder_input_ids,
-            "decoder_token_type_ids": decoder_token_type_ids,
-            "decoder_attention_mask": decoder_input_mask,
-            "decoder_sequence_labels": decoder_sequence_labels,
-            "decoder_token_labels": decoder_token_labels,
-            "decoder_choice_labels": decoder_choice_labels,
-            "encoder_hidden_states": encoder_hidden_states,
-            "labels": decoder_token_labels,
-        }
-
-    def get_pretrained_model(self):
-        return EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-cased", "gpt2")

tests/test_modeling_gpt2.py

@@ -20,7 +20,7 @@ from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 
 from .test_configuration_common import ConfigTester
-from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
+from .test_modeling_common import ModelTesterMixin, ids_tensor
 
 
 if is_torch_available():
@@ -62,27 +62,27 @@ class GPT2ModelTester:
         scope=None,
     ):
         self.parent = parent
-        self.batch_size = batch_size
-        self.seq_length = seq_length
-        self.is_training = is_training
-        self.use_token_type_ids = use_token_type_ids
-        self.use_input_mask = use_input_mask
-        self.use_labels = use_labels
-        self.use_mc_token_ids = use_mc_token_ids
-        self.vocab_size = vocab_size
-        self.hidden_size = hidden_size
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.intermediate_size = intermediate_size
-        self.hidden_act = hidden_act
-        self.hidden_dropout_prob = hidden_dropout_prob
-        self.attention_probs_dropout_prob = attention_probs_dropout_prob
-        self.max_position_embeddings = max_position_embeddings
-        self.type_vocab_size = type_vocab_size
-        self.type_sequence_label_size = type_sequence_label_size
-        self.initializer_range = initializer_range
-        self.num_labels = num_labels
-        self.num_choices = num_choices
+        self.batch_size = 14
+        self.seq_length = 7
+        self.is_training = True
+        self.use_token_type_ids = True
+        self.use_input_mask = True
+        self.use_labels = True
+        self.use_mc_token_ids = True
+        self.vocab_size = 99
+        self.hidden_size = 32
+        self.num_hidden_layers = 5
+        self.num_attention_heads = 4
+        self.intermediate_size = 37
+        self.hidden_act = "gelu"
+        self.hidden_dropout_prob = 0.1
+        self.attention_probs_dropout_prob = 0, 1
+        self.max_position_embeddings = 512
+        self.type_vocab_size = 16
+        self.type_sequence_label_size = 2
+        self.initializer_range = 0.02
+        self.num_labels = 3
+        self.num_choices = 4
         self.scope = None
         self.bos_token_id = vocab_size - 1
         self.eos_token_id = vocab_size - 1
@@ -142,35 +142,6 @@ class GPT2ModelTester:
             choice_labels,
         )
 
-    def prepare_config_and_inputs_for_decoder(self):
-        (
-            config,
-            input_ids,
-            input_mask,
-            head_mask,
-            token_type_ids,
-            mc_token_ids,
-            sequence_labels,
-            token_labels,
-            choice_labels,
-        ) = self.prepare_config_and_inputs()
-
-        encoder_hidden_states = floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
-        encoder_attention_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
-
-        return (
-            config,
-            input_ids,
-            input_mask,
-            head_mask,
-            token_type_ids,
-            sequence_labels,
-            token_labels,
-            choice_labels,
-            encoder_hidden_states,
-            encoder_attention_mask,
-        )
-
     def create_and_check_gpt2_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args):
         model = GPT2Model(config=config)
         model.to(torch_device)
@@ -181,7 +152,7 @@ class GPT2ModelTester:
         result = model(input_ids)
 
         self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
-        self.parent.assertEqual(len(result.past_key_values), config.n_layer)
+        self.parent.assertEqual(len(result["past_key_values"]), config.n_layer)
 
     def create_and_check_gpt2_model_past(self, config, input_ids, input_mask, head_mask, token_type_ids, *args):
         model = GPT2Model(config=config)

tests/test_modeling_longformer.py

@@ -33,7 +33,6 @@ if is_torch_available():
         LongformerForTokenClassification,
         LongformerForQuestionAnswering,
         LongformerForMultipleChoice,
-        LongformerSelfAttention,
     )
 
 
@@ -326,209 +325,7 @@ class LongformerModelTest(ModelTesterMixin, unittest.TestCase):
         self.model_tester.create_and_check_longformer_for_multiple_choice(*config_and_inputs)
 
 
-@require_torch
 class LongformerModelIntegrationTest(unittest.TestCase):
-    def _get_hidden_states(self):
-        return torch.tensor(
-            [
-                [
-                    [
-                        4.98332758e-01,
-                        2.69175139e00,
-                        -7.08081422e-03,
-                        1.04915401e00,
-                        -1.83476661e00,
-                        7.67220476e-01,
-                        2.98580543e-01,
-                        2.84803992e-02,
-                    ],
-                    [
-                        -7.58357372e-01,
-                        4.20635998e-01,
-                        -4.04739919e-02,
-                        1.59924145e-01,
-                        2.05135748e00,
-                        -1.15997978e00,
-                        5.37166397e-01,
-                        2.62873606e-01,
-                    ],
-                    [
-                        -1.69438001e00,
-                        4.17574660e-01,
-                        -1.49196962e00,
-                        -1.76483717e00,
-                        -1.94566312e-01,
-                        -1.71183858e00,
-                        7.72903565e-01,
-                        -1.11557056e00,
-                    ],
-                    [
-                        5.44028163e-01,
-                        2.05466114e-01,
-                        -3.63045868e-01,
-                        2.41865062e-01,
-                        3.20348382e-01,
-                        -9.05611176e-01,
-                        -1.92690727e-01,
-                        -1.19917547e00,
-                    ],
-                ]
-            ],
-            dtype=torch.float32,
-            device=torch_device,
-        )
-
-    def test_diagonalize(self):
-        hidden_states = self._get_hidden_states()
-        hidden_states = hidden_states.reshape((1, 8, 4))  # set seq length = 8, hidden dim = 4
-        chunked_hidden_states = LongformerSelfAttention._chunk(hidden_states, window_overlap=2)
-        window_overlap_size = chunked_hidden_states.shape[2]
-        self.assertTrue(window_overlap_size == 4)
-
-        padded_hidden_states = LongformerSelfAttention._pad_and_diagonalize(chunked_hidden_states)
-
-        self.assertTrue(padded_hidden_states.shape[-1] == chunked_hidden_states.shape[-1] + window_overlap_size - 1)
-
-        # first row => [0.4983,  2.6918, -0.0071,  1.0492, 0.0000,  0.0000,  0.0000]
-        self.assertTrue(torch.allclose(padded_hidden_states[0, 0, 0, :4], chunked_hidden_states[0, 0, 0], atol=1e-3))
-        self.assertTrue(
-            torch.allclose(
-                padded_hidden_states[0, 0, 0, 4:],
-                torch.zeros((3,), device=torch_device, dtype=torch.float32),
-                atol=1e-3,
-            )
-        )
-        # last row => [0.0000,  0.0000,  0.0000, 2.0514, -1.1600,  0.5372,  0.2629]
-        self.assertTrue(torch.allclose(padded_hidden_states[0, 0, -1, 3:], chunked_hidden_states[0, 0, -1], atol=1e-3))
-        self.assertTrue(
-            torch.allclose(
-                padded_hidden_states[0, 0, -1, :3],
-                torch.zeros((3,), device=torch_device, dtype=torch.float32),
-                atol=1e-3,
-            )
-        )
-
-    def test_pad_and_transpose_last_two_dims(self):
-        hidden_states = self._get_hidden_states()
-        self.assertTrue(hidden_states.shape, (1, 8, 4))
-        padding = (0, 0, 0, 1)
-
-        padded_hidden_states = LongformerSelfAttention._pad_and_transpose_last_two_dims(hidden_states, padding)
-        self.assertTrue(padded_hidden_states.shape, (1, 8, 5))
-
-        expected_added_dim = torch.zeros((5,), device=torch_device, dtype=torch.float32)
-        self.assertTrue(torch.allclose(expected_added_dim, padded_hidden_states[0, -1, :], atol=1e-6))
-        self.assertTrue(torch.allclose(hidden_states[0, -1, :], padded_hidden_states.view(1, -1)[0, 24:32], atol=1e-6))
-
-    def test_chunk(self):
-        hidden_states = self._get_hidden_states()
-        batch_size = 1
-        seq_length = 8
-        hidden_size = 4
-        hidden_states = hidden_states.reshape((batch_size, seq_length, hidden_size))
-
-        chunked_hidden_states = LongformerSelfAttention._chunk(hidden_states, window_overlap=2)
-
-        # expected slices across chunk and seq length dim
-        expected_slice_along_seq_length = torch.tensor(
-            [0.4983, -0.7584, -1.6944], device=torch_device, dtype=torch.float32
-        )
-        expected_slice_along_chunk = torch.tensor(
-            [0.4983, -1.8348, -0.7584, 2.0514], device=torch_device, dtype=torch.float32
-        )
-
-        self.assertTrue(torch.allclose(chunked_hidden_states[0, :, 0, 0], expected_slice_along_seq_length, atol=1e-3))
-        self.assertTrue(torch.allclose(chunked_hidden_states[0, 0, :, 0], expected_slice_along_chunk, atol=1e-3))
-        self.assertTrue(chunked_hidden_states.shape, (1, 3, 4, 4))
-
-    def test_mask_invalid_locations(self):
-        hidden_states = self._get_hidden_states()
-
-        batch_size = 1
-        seq_length = 8
-        hidden_size = 4
-        hidden_states = hidden_states.reshape((batch_size, seq_length, hidden_size))
-        chunked_hidden_states = LongformerSelfAttention._chunk(hidden_states, window_overlap=2)
-
-        hid_states_1 = chunked_hidden_states.clone()
-        LongformerSelfAttention._mask_invalid_locations(hid_states_1, 1)
-        self.assertTrue(torch.isinf(hid_states_1).sum().item() == 8)
-
-        hid_states_2 = chunked_hidden_states.clone()
-        LongformerSelfAttention._mask_invalid_locations(hid_states_2, 2)
-        self.assertTrue(torch.isinf(hid_states_2).sum().item() == 24)
-
-        hid_states_3 = chunked_hidden_states.clone()[:, :, :, :3]
-        LongformerSelfAttention._mask_invalid_locations(hid_states_3, 2)
-        self.assertTrue(torch.isinf(hid_states_3).sum().item() == 24)
-
-        hid_states_4 = chunked_hidden_states.clone()[:, :, 2:, :]
-        LongformerSelfAttention._mask_invalid_locations(hid_states_4, 2)
-        self.assertTrue(torch.isinf(hid_states_4).sum().item() == 12)
-
-    def test_layer_local_attn(self):
-        model = LongformerModel.from_pretrained("patrickvonplaten/longformer-random-tiny")
-        model.eval()
-        layer = model.encoder.layer[0].attention.self.to(torch_device)
-        hidden_states = self._get_hidden_states()
-        batch_size, seq_length, hidden_size = hidden_states.size()
-        attention_mask = torch.zeros((batch_size, 1, 1, seq_length), dtype=torch.float32, device=torch_device)
-        attention_mask[:, :, :, -2:] = -10000
-        output_hidden_states = layer(hidden_states, attention_mask)[0]
-
-        self.assertTrue(output_hidden_states.shape, (1, 4, 8))
-        self.assertTrue(
-            torch.allclose(
-                output_hidden_states[0, 1],
-                torch.tensor(
-                    [0.0019, 0.0122, -0.0171, -0.0256, -0.0300, 0.0173, -0.0115, 0.0048],
-                    dtype=torch.float32,
-                    device=torch_device,
-                ),
-                atol=1e-3,
-            )
-        )
-
-    def test_layer_global_attn(self):
-        model = LongformerModel.from_pretrained("patrickvonplaten/longformer-random-tiny")
-        model.eval()
-        layer = model.encoder.layer[0].attention.self.to(torch_device)
-        hidden_states = torch.cat([self._get_hidden_states(), self._get_hidden_states() - 0.5], dim=0)
-        batch_size, seq_length, hidden_size = hidden_states.size()
-        attention_mask = torch.zeros((batch_size, 1, 1, seq_length), dtype=torch.float32, device=torch_device)
-
-        # create attn mask
-        attention_mask[0, :, :, -2:] = 10000.0
-        attention_mask[0, :, :, -1:] = -10000.0
-        attention_mask[1, :, :, 1:] = 10000.0
-        output_hidden_states = layer(hidden_states, attention_mask)[0]
-
-        self.assertTrue(output_hidden_states.shape, (2, 4, 8))
-
-        self.assertTrue(
-            torch.allclose(
-                output_hidden_states[0, 2],
-                torch.tensor(
-                    [-0.0651, -0.0393, 0.0309, -0.0342, -0.0066, -0.0155, -0.0209, -0.0494],
-                    dtype=torch.float32,
-                    device=torch_device,
-                ),
-                atol=1e-3,
-            )
-        )
-
-        self.assertTrue(
-            torch.allclose(
-                output_hidden_states[1, -2],
-                torch.tensor(
-                    [-0.0405, -0.0384, 0.0396, -0.0374, -0.0341, 0.0136, 0.0014, -0.0571],
-                    dtype=torch.float32,
-                    device=torch_device,
-                ),
-                atol=1e-3,
-            )
-        )
-
     @slow
     def test_inference_no_head(self):
         model = LongformerModel.from_pretrained("allenai/longformer-base-4096")
@@ -574,13 +371,13 @@ class LongformerModelIntegrationTest(unittest.TestCase):
         input_ids = torch.tensor(
             [[0] + [20920, 232, 328, 1437] * 1000 + [2]], dtype=torch.long, device=torch_device
         )  # long input
-        input_ids = input_ids.to(torch_device)
 
         loss, prediction_scores = model(input_ids, labels=input_ids)
 
         expected_loss = torch.tensor(0.0074, device=torch_device)
         expected_prediction_scores_sum = torch.tensor(-6.1048e08, device=torch_device)
         expected_prediction_scores_mean = torch.tensor(-3.0348, device=torch_device)
+        input_ids = input_ids.to(torch_device)
 
         self.assertTrue(torch.allclose(loss, expected_loss, atol=1e-4))
         self.assertTrue(torch.allclose(prediction_scores.sum(), expected_prediction_scores_sum, atol=1e-4))