Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 LysandreJik
|
e77d866a87 | ||
|
Rogge Niels
|
f543c46e65 | ||
|
Rogge Niels
|
c28edcaf14 | ||
|
Rogge Niels
|
c2eae65e36 | ||
|
Rogge Niels
|
31fa6ca155 | ||
|
Rogge Niels
|
7a0b5200bf | ||
|
Rogge Niels
|
2ae412db07 | ||
|
Rogge Niels
|
1bd7499f9a | ||
|
Rogge Niels
|
3cd567a241 | ||
|
Rogge Niels
|
f700094ca1 | ||
|
Rogge Niels
|
6d871ed4da | ||
|
Rogge Niels
|
6457896328 |
No files matched your search
@@ -102,7 +102,6 @@ jobs:
|
||||
- v0.4-{{ checksum "setup.py" }}
|
||||
- run: pip install --upgrade pip
|
||||
- run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece]
|
||||
- run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html
|
||||
- save_cache:
|
||||
key: v0.4-{{ checksum "setup.py" }}
|
||||
paths:
|
||||
@@ -130,7 +129,6 @@ jobs:
|
||||
- v0.4-{{ checksum "setup.py" }}
|
||||
- run: pip install --upgrade pip
|
||||
- run: pip install .[sklearn,torch,testing,sentencepiece]
|
||||
- run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html
|
||||
- save_cache:
|
||||
key: v0.4-torch-{{ checksum "setup.py" }}
|
||||
paths:
|
||||
@@ -212,7 +210,6 @@ jobs:
|
||||
- v0.4-{{ checksum "setup.py" }}
|
||||
- run: pip install --upgrade pip
|
||||
- run: pip install .[sklearn,torch,testing,sentencepiece]
|
||||
- run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html
|
||||
- save_cache:
|
||||
key: v0.4-torch-{{ checksum "setup.py" }}
|
||||
paths:
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
name: Model templates runner
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "src/**"
|
||||
- "tests/**"
|
||||
- ".github/**"
|
||||
- "templates/**"
|
||||
|
||||
jobs:
|
||||
run_tests_templates:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v1
|
||||
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.6
|
||||
|
||||
- name: Loading cache.
|
||||
uses: actions/cache@v2
|
||||
id: cache
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: v1.2-tests_templates
|
||||
restore-keys: |
|
||||
v1.2-tests_templates-${{ hashFiles('setup.py') }}
|
||||
v1.2-tests_templates
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install .[dev]
|
||||
- name: Create model files
|
||||
run: |
|
||||
transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/encoder-bert-tokenizer.json --path=templates/adding_a_new_model
|
||||
transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/pt-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
|
||||
transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/standalone.json --path=templates/adding_a_new_model
|
||||
transformers-cli add-new-model --testing --testing_file=templates/adding_a_new_model/tests/tf-encoder-bert-tokenizer.json --path=templates/adding_a_new_model
|
||||
make style
|
||||
python utils/check_table.py --fix_and_overwrite
|
||||
python utils/check_dummies.py --fix_and_overwrite
|
||||
|
||||
- name: Run all non-slow tests
|
||||
run: |
|
||||
python -m pytest -n 2 --dist=loadfile -s --make-reports=tests_templates tests/*template*
|
||||
|
||||
- name: Run style changes
|
||||
run: |
|
||||
git fetch origin master:master
|
||||
make fixup
|
||||
|
||||
- name: Failure short reports
|
||||
if: ${{ always() }}
|
||||
run: cat reports/tests_templates_failures_short.txt
|
||||
|
||||
- name: Test suite reports artifacts
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: run_all_tests_templates_test_reports
|
||||
path: reports
|
||||
@@ -159,6 +159,3 @@ tags
|
||||
|
||||
# pre-commit
|
||||
.pre-commit*
|
||||
|
||||
# .lock
|
||||
*.lock
|
||||
@@ -213,7 +213,6 @@ Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih.
|
||||
1. **[LXMERT](https://huggingface.co/transformers/model_doc/lxmert.html)** (from UNC Chapel Hill) released with the paper [LXMERT: Learning Cross-Modality Encoder Representations from Transformers for Open-Domain Question Answering](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal.
|
||||
1. **[MarianMT](https://huggingface.co/transformers/model_doc/marian.html)** Machine translation models trained using [OPUS](http://opus.nlpl.eu/) data by Jörg Tiedemann. The [Marian Framework](https://marian-nmt.github.io/) is being developed by the Microsoft Translator Team.
|
||||
1. **[MBart](https://huggingface.co/transformers/model_doc/mbart.html)** (from Facebook) released with the paper [Multilingual Denoising Pre-training for Neural Machine Translation](https://arxiv.org/abs/2001.08210) by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
|
||||
1. **[MPNet](https://huggingface.co/transformers/model_doc/mpnet.html)** (from Microsoft Research) released with the paper [MPNet: Masked and Permuted Pre-training for Language Understanding](https://arxiv.org/abs/2004.09297) by Kaitao Song, Xu Tan, Tao Qin, Jianfeng Lu, Tie-Yan Liu.
|
||||
1. **[MT5](https://huggingface.co/transformers/model_doc/mt5.html)** (from Google AI) released with the paper [mT5: A massively multilingual pre-trained text-to-text transformer](https://arxiv.org/abs/2010.11934) by Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
|
||||
1. **[Pegasus](https://huggingface.co/transformers/model_doc/pegasus.html)** (from Google) released with the paper [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive Summarization](https://arxiv.org/abs/1912.08777)> by Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu.
|
||||
1. **[ProphetNet](https://huggingface.co/transformers/model_doc/prophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// These two things need to be updated at each release for the version selector.
|
||||
// Last stable version
|
||||
const stableVersion = "v4.0.0"
|
||||
// Dictionary doc folder to label. The last stable version should have an empty key.
|
||||
// Dictionary doc folder to label
|
||||
const versionMapping = {
|
||||
"master": "master",
|
||||
"": "v4.0.0 (stable)",
|
||||
"v4.0.0": "v4.0.0",
|
||||
"v3.5.1": "v3.5.0/v3.5.1",
|
||||
"v3.4.0": "v3.4.0",
|
||||
"v3.3.1": "v3.3.0/v3.3.1",
|
||||
|
||||
+16
-23
@@ -151,47 +151,44 @@ and conversion utilities for the following models:
|
||||
22. :doc:`MBart <model_doc/mbart>` (from Facebook) released with the paper `Multilingual Denoising Pre-training for
|
||||
Neural Machine Translation <https://arxiv.org/abs/2001.08210>`__ by Yinhan Liu, Jiatao Gu, Naman Goyal, Xian Li,
|
||||
Sergey Edunov, Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.
|
||||
23. :doc:`MPNet <model_doc/mpnet>` (from Microsoft Research) released with the paper `MPNet: Masked and Permuted
|
||||
Pre-training for Language Understanding <https://arxiv.org/abs/2004.09297>`__ by Kaitao Song, Xu Tan, Tao Qin,
|
||||
Jianfeng Lu, Tie-Yan Liu.
|
||||
24. :doc:`MT5 <model_doc/mt5>` (from Google AI) released with the paper `mT5: A massively multilingual pre-trained
|
||||
23. :doc:`MT5 <model_doc/mt5>` (from Google AI) released with the paper `mT5: A massively multilingual pre-trained
|
||||
text-to-text transformer <https://arxiv.org/abs/2010.11934>`__ by Linting Xue, Noah Constant, Adam Roberts, Mihir
|
||||
Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel.
|
||||
25. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
|
||||
24. :doc:`Pegasus <model_doc/pegasus>` (from Google) released with the paper `PEGASUS: Pre-training with Extracted
|
||||
Gap-sentences for Abstractive Summarization <https://arxiv.org/abs/1912.08777>`__> by Jingqing Zhang, Yao Zhao,
|
||||
Mohammad Saleh and Peter J. Liu.
|
||||
26. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
|
||||
25. :doc:`ProphetNet <model_doc/prophetnet>` (from Microsoft Research) released with the paper `ProphetNet: Predicting
|
||||
Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan, Weizhen Qi,
|
||||
Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
|
||||
27. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
|
||||
26. :doc:`Reformer <model_doc/reformer>` (from Google Research) released with the paper `Reformer: The Efficient
|
||||
Transformer <https://arxiv.org/abs/2001.04451>`__ by Nikita Kitaev, Łukasz Kaiser, Anselm Levskaya.
|
||||
28. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
|
||||
27. :doc:`RoBERTa <model_doc/roberta>` (from Facebook), released together with the paper a `Robustly Optimized BERT
|
||||
Pretraining Approach <https://arxiv.org/abs/1907.11692>`__ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar
|
||||
Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. ultilingual BERT into `DistilmBERT
|
||||
<https://github.com/huggingface/transformers/tree/master/examples/distillation>`__ and a German version of
|
||||
DistilBERT.
|
||||
29. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
|
||||
28. :doc:`SqueezeBert <model_doc/squeezebert>` released with the paper `SqueezeBERT: What can computer vision teach NLP
|
||||
about efficient neural networks? <https://arxiv.org/abs/2006.11316>`__ by Forrest N. Iandola, Albert E. Shaw, Ravi
|
||||
Krishna, and Kurt W. Keutzer.
|
||||
30. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
|
||||
29. :doc:`T5 <model_doc/t5>` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a
|
||||
Unified Text-to-Text Transformer <https://arxiv.org/abs/1910.10683>`__ by Colin Raffel and Noam Shazeer and Adam
|
||||
Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu.
|
||||
31. `TAPAS <https://huggingface.co/transformers/master/model_doc/tapas.html>`__ released with the paper `TAPAS: Weakly
|
||||
Supervised Table Parsing via Pre-training <https://arxiv.org/abs/2004.02349>`__ by Jonathan Herzig, Paweł Krzysztof
|
||||
Nowak, Thomas Müller, Francesco Piccinno and Julian Martin Eisenschlos.
|
||||
32. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
|
||||
30. :doc:`TAPAS <model_doc/tapas>` (from Google AI) released with the paper `TAPAS: Weakly Supervised Table Parsing via
|
||||
Pre-training <https://arxiv.org/abs/2004.02349>`__ by Jonathan Herzig, Paweł Krzysztof Nowak, Thomas Müller,
|
||||
Francesco Piccinno and Julian Martin Eisenschlos.
|
||||
31. :doc:`Transformer-XL <model_doc/transformerxl>` (from Google/CMU) released with the paper `Transformer-XL:
|
||||
Attentive Language Models Beyond a Fixed-Length Context <https://arxiv.org/abs/1901.02860>`__ by Zihang Dai*,
|
||||
Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov.
|
||||
33. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
|
||||
32. :doc:`XLM <model_doc/xlm>` (from Facebook) released together with the paper `Cross-lingual Language Model
|
||||
Pretraining <https://arxiv.org/abs/1901.07291>`__ by Guillaume Lample and Alexis Conneau.
|
||||
34. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
|
||||
33. :doc:`XLM-ProphetNet <model_doc/xlmprophetnet>` (from Microsoft Research) released with the paper `ProphetNet:
|
||||
Predicting Future N-gram for Sequence-to-Sequence Pre-training <https://arxiv.org/abs/2001.04063>`__ by Yu Yan,
|
||||
Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou.
|
||||
35. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
|
||||
34. :doc:`XLM-RoBERTa <model_doc/xlmroberta>` (from Facebook AI), released together with the paper `Unsupervised
|
||||
Cross-lingual Representation Learning at Scale <https://arxiv.org/abs/1911.02116>`__ by Alexis Conneau*, Kartikay
|
||||
Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke
|
||||
Zettlemoyer and Veselin Stoyanov.
|
||||
36. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `XLNet: Generalized Autoregressive
|
||||
35. :doc:`XLNet <model_doc/xlnet>` (from Google/CMU) released with the paper `XLNet: Generalized Autoregressive
|
||||
Pretraining for Language Understanding <https://arxiv.org/abs/1906.08237>`__ by Zhilin Yang*, Zihang Dai*, Yiming
|
||||
Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le.
|
||||
|
||||
@@ -246,8 +243,6 @@ TensorFlow and/or Flax.
|
||||
+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
|
||||
| Longformer | ✅ | ✅ | ✅ | ✅ | ❌ |
|
||||
+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
|
||||
| MPNet | ✅ | ✅ | ✅ | ✅ | ❌ |
|
||||
+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
|
||||
| Marian | ✅ | ❌ | ✅ | ✅ | ❌ |
|
||||
+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
|
||||
| MobileBERT | ✅ | ✅ | ✅ | ✅ | ❌ |
|
||||
@@ -272,8 +267,6 @@ TensorFlow and/or Flax.
|
||||
+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
|
||||
| T5 | ✅ | ✅ | ✅ | ✅ | ❌ |
|
||||
+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
|
||||
| TAPAS | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
|
||||
| Transformer-XL | ✅ | ❌ | ✅ | ✅ | ❌ |
|
||||
+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
|
||||
| XLM | ✅ | ❌ | ✅ | ✅ | ❌ |
|
||||
@@ -289,6 +282,7 @@ TensorFlow and/or Flax.
|
||||
| mT5 | ✅ | ✅ | ✅ | ✅ | ❌ |
|
||||
+-----------------------------+----------------+----------------+-----------------+--------------------+--------------+
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Get started
|
||||
@@ -375,7 +369,6 @@ TensorFlow and/or Flax.
|
||||
model_doc/marian
|
||||
model_doc/mbart
|
||||
model_doc/mobilebert
|
||||
model_doc/mpnet
|
||||
model_doc/mt5
|
||||
model_doc/gpt
|
||||
model_doc/gpt2
|
||||
|
||||
@@ -15,8 +15,7 @@ Utilities for Generation
|
||||
|
||||
This page lists all the utility functions used by :meth:`~transformers.PretrainedModel.generate`,
|
||||
:meth:`~transformers.PretrainedModel.greedy_search`, :meth:`~transformers.PretrainedModel.sample`,
|
||||
:meth:`~transformers.PretrainedModel.beam_search`, :meth:`~transformers.PretrainedModel.beam_sample`, and
|
||||
:meth:`~transformers.PretrainedModel.group_beam_search`.
|
||||
:meth:`~transformers.PretrainedModel.beam_search`, and :meth:`~transformers.PretrainedModel.beam_sample`.
|
||||
|
||||
Most of those are only useful if you are studying the code of the generate methods in the library.
|
||||
|
||||
@@ -53,12 +52,6 @@ generation.
|
||||
.. autoclass:: transformers.NoBadWordsLogitsProcessor
|
||||
:members: __call__
|
||||
|
||||
.. autoclass:: transformers.PrefixConstrainedLogitsProcessor
|
||||
:members: __call__
|
||||
|
||||
.. autoclass:: transformers.HammingDiversityLogitsProcessor
|
||||
:members: __call__
|
||||
|
||||
BeamSearch
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
||||
@@ -105,6 +105,8 @@ BartModel
|
||||
.. autoclass:: transformers.BartModel
|
||||
:members: forward
|
||||
|
||||
.. autofunction:: transformers.models.bart.modeling_bart._prepare_bart_decoder_inputs
|
||||
|
||||
|
||||
BartForConditionalGeneration
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
MPNet
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
Overview
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The MPNet model was proposed in `MPNet: Masked and Permuted Pre-training for Language Understanding
|
||||
<https://arxiv.org/abs/2004.09297>`__ by Kaitao Song, Xu Tan, Tao Qin, Jianfeng Lu, Tie-Yan Liu.
|
||||
|
||||
MPNet adopts a novel pre-training method, named masked and permuted language modeling, to inherit the advantages of
|
||||
masked language modeling and permuted language modeling for natural language understanding.
|
||||
|
||||
The abstract from the paper is the following:
|
||||
|
||||
*BERT adopts masked language modeling (MLM) for pre-training and is one of the most successful pre-training models.
|
||||
Since BERT neglects dependency among predicted tokens, XLNet introduces permuted language modeling (PLM) for
|
||||
pre-training to address this problem. However, XLNet does not leverage the full position information of a sentence and
|
||||
thus suffers from position discrepancy between pre-training and fine-tuning. In this paper, we propose MPNet, a novel
|
||||
pre-training method that inherits the advantages of BERT and XLNet and avoids their limitations. MPNet leverages the
|
||||
dependency among predicted tokens through permuted language modeling (vs. MLM in BERT), and takes auxiliary position
|
||||
information as input to make the model see a full sentence and thus reducing the position discrepancy (vs. PLM in
|
||||
XLNet). We pre-train MPNet on a large-scale dataset (over 160GB text corpora) and fine-tune on a variety of
|
||||
down-streaming tasks (GLUE, SQuAD, etc). Experimental results show that MPNet outperforms MLM and PLM by a large
|
||||
margin, and achieves better results on these tasks compared with previous state-of-the-art pre-trained methods (e.g.,
|
||||
BERT, XLNet, RoBERTa) under the same model setting.*
|
||||
|
||||
Tips:
|
||||
|
||||
- MPNet doesn't have :obj:`token_type_ids`, you don't need to indicate which token belongs to which segment. just
|
||||
separate your segments with the separation token :obj:`tokenizer.sep_token` (or :obj:`[sep]`).
|
||||
|
||||
The original code can be found `here <https://github.com/microsoft/MPNet>`__.
|
||||
|
||||
MPNetConfig
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.MPNetConfig
|
||||
:members:
|
||||
|
||||
|
||||
MPNetTokenizer
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.MPNetTokenizer
|
||||
:members: build_inputs_with_special_tokens, get_special_tokens_mask,
|
||||
create_token_type_ids_from_sequences, save_vocabulary
|
||||
|
||||
|
||||
MPNetTokenizerFast
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.MPNetTokenizerFast
|
||||
:members:
|
||||
|
||||
|
||||
MPNetModel
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.MPNetModel
|
||||
:members: forward
|
||||
|
||||
|
||||
MPNetForMaskedLM
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.MPNetForMaskedLM
|
||||
:members: forward
|
||||
|
||||
|
||||
MPNetForSequenceClassification
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.MPNetForSequenceClassification
|
||||
:members: forward
|
||||
|
||||
|
||||
MPNetForMultipleChoice
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.MPNetForMultipleChoice
|
||||
:members: forward
|
||||
|
||||
|
||||
MPNetForTokenClassification
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.MPNetForTokenClassification
|
||||
:members: forward
|
||||
|
||||
|
||||
MPNetForQuestionAnswering
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.MPNetForQuestionAnswering
|
||||
:members: forward
|
||||
|
||||
|
||||
TFMPNetModel
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.TFMPNetModel
|
||||
:members: call
|
||||
|
||||
|
||||
TFMPNetForMaskedLM
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.TFMPNetForMaskedLM
|
||||
:members: call
|
||||
|
||||
|
||||
TFMPNetForSequenceClassification
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.TFMPNetForSequenceClassification
|
||||
:members: call
|
||||
|
||||
|
||||
TFMPNetForMultipleChoice
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.TFMPNetForMultipleChoice
|
||||
:members: call
|
||||
|
||||
|
||||
TFMPNetForTokenClassification
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.TFMPNetForTokenClassification
|
||||
:members: call
|
||||
|
||||
|
||||
TFMPNetForQuestionAnswering
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.TFMPNetForQuestionAnswering
|
||||
:members: call
|
||||
+94
-107
@@ -5,15 +5,15 @@ Overview
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The TAPAS model was proposed in `TAPAS: Weakly Supervised Table Parsing via Pre-training
|
||||
<https://www.aclweb.org/anthology/2020.acl-main.398>`__ by Jonathan Herzig, Paweł Krzysztof Nowak, Thomas Müller,
|
||||
Francesco Piccinno and Julian Martin Eisenschlos. It's a BERT-based model specifically designed (and pre-trained) for
|
||||
answering questions about tabular data. Compared to BERT, TAPAS uses relative position embeddings and has 7 token types
|
||||
that encode tabular structure. TAPAS is pre-trained on the masked language modeling (MLM) objective on a large dataset
|
||||
comprising millions of tables from English Wikipedia and corresponding texts. For question answering, TAPAS has 2 heads
|
||||
on top: a cell selection head and an aggregation head, for (optionally) performing aggregations (such as counting or
|
||||
summing) among selected cells. TAPAS has been fine-tuned on several datasets: SQA (Sequential Question Answering by
|
||||
Microsoft), WTQ (Wiki Table Questions by Stanford University) and WikiSQL (by Salesforce). It achieves state-of-the-art
|
||||
on both SQA and WTQ, while having comparable performance to SOTA on WikiSQL, with a much simpler architecture.
|
||||
<https://www.aclweb.org/anthology/2020.acl-main.398>`__ by Jonathan Herzig, Paweł Krzysztof Nowak, Thomas Müller, Francesco
|
||||
Piccinno and Julian Martin Eisenschlos. It's a BERT-based model specifically designed (and pre-trained) for answering questions
|
||||
about tabular data. Compared to BERT, TAPAS uses relative position embeddings and has 7 token types that encode tabular
|
||||
structure. TAPAS is pre-trained on the masked language modeling (MLM) objective on a large dataset comprising millions
|
||||
of tables from English Wikipedia and corresponding texts. For question answering, TAPAS has 2 heads on top: a cell
|
||||
selection head and an aggregation head, for (optionally) performing aggregations (such as counting or summing) among
|
||||
selected cells. TAPAS has been fine-tuned on several datasets: SQA (Sequential Question Answering by Microsoft), WTQ
|
||||
(Wiki Table Questions by Stanford University) and WikiSQL (by Salesforce). It achieves state-of-the-art on both SQA and
|
||||
WTQ, while having comparable performance to SOTA on WikiSQL, with a much simpler architecture.
|
||||
|
||||
The abstract from the paper is the following:
|
||||
|
||||
@@ -35,32 +35,32 @@ of millions of automatically created training examples which are learned in an i
|
||||
The authors of TAPAS call this further pre-training intermediate pre-training (since TAPAS is first pre-trained on MLM,
|
||||
and then on another dataset). They found that intermediate pre-training further improves performance on SQA, achieving
|
||||
a new state-of-the-art as well as state-of-the-art on TabFact, a large-scale dataset with 16k Wikipedia tables for
|
||||
table entailment (a binary classification task). For more details, see their follow-up paper: `Understanding tables
|
||||
with intermediate pre-training <https://arxiv.org/abs/2010.00571>`__ by Julian Martin Eisenschlos, Syrine Krichene and
|
||||
table entailment (a binary classification task). For more details, see their follow-up paper: `Understanding tables with
|
||||
intermediate pre-training <https://arxiv.org/abs/2010.00571>`__ by Julian Martin Eisenschlos, Syrine Krichene and
|
||||
Thomas Müller.
|
||||
|
||||
The original code can be found `here <https://github.com/google-research/tapas>`__. This page also includes links to
|
||||
the datasets (SQA, WTQ, WikiSQL and TabFact).
|
||||
The original code can be found `here <https://github.com/google-research/tapas>`__. This page also includes links to the
|
||||
datasets (SQA, WTQ, WikiSQL and TabFact).
|
||||
|
||||
Tips:
|
||||
|
||||
- TAPAS is a model that uses relative position embeddings by default (restarting the position embeddings at every cell
|
||||
of the table). Note that this is something that was added after the publication of the original TAPAS paper.
|
||||
According to the authors, this usually results in a slightly better performance, and allows you to encode longer
|
||||
sequences without running out of embeddings. This is reflected in the ``reset_position_index_per_cell`` parameter of
|
||||
:class:`~transformers.TapasConfig`, which is set to ``True`` by default. The latest versions of the models available
|
||||
in the `model hub <https://huggingface.co/models?search=tapas>`_ all use relative position embeddings. You can still
|
||||
use the ones with absolute position embeddings by passing in a certain version when calling the `.from_pretrained`
|
||||
method as explained in the model cards. Note that it's usually advised to pad the inputs on the right rather than the
|
||||
left.
|
||||
- TAPAS is based on BERT, so ``TAPAS-base`` for example corresponds to a ``BERT-base`` architecture. Of course,
|
||||
TAPAS-large will result in the best performance (the results reported in the paper are from TAPAS-large). Metrics of
|
||||
the various sized models are shown on the `original Github repository <https://github.com/google-research/tapas>`_.
|
||||
of the table). Note that this is something that was added after the publication of the original TAPAS paper. According
|
||||
to the authors, this usually results in a slightly better performance, and allows you to encode longer sequences without
|
||||
running out of embeddings. This is reflected in the ``reset_position_index_per_cell`` parameter of :class:`~transformers.TapasConfig`,
|
||||
which is set to ``True`` by default.
|
||||
The latest versions of the models available in the `model hub <https://huggingface.co/models?search=tapas>`_ all use relative
|
||||
position embeddings. You can still use the ones with absolute position embeddings by passing in a certain version when calling the
|
||||
`.from_pretrained` method as explained in the model cards.
|
||||
Note that it's usually advised to pad the inputs on the right rather than the left.
|
||||
- TAPAS is based on BERT, so ``TAPAS-base`` for example corresponds to a ``BERT-base`` architecture. Of course, TAPAS-large
|
||||
will result in the best performance (the results reported in the paper are from TAPAS-large). Metrics of the various
|
||||
sized models are shown on the `original Github repository <https://github.com/google-research/tapas>`_.
|
||||
- TAPAS has checkpoints fine-tuned on SQA, which are capable of answering questions related to a table in a
|
||||
conversational set-up. This means that you can ask follow-up questions such as "what is his age?" related to the
|
||||
previous question. Note that the forward pass of TAPAS is a bit different in case of a conversational set-up: in that
|
||||
case, you have to feed every training example one by one to the model, such that the `prev_labels` token type ids can
|
||||
be overwritten by the predicted `label_ids` of the model to the previous question. See "Usage" section for more info.
|
||||
case, you have to feed every training example one by one to the model, such that the `prev_label_ids` token type ids
|
||||
can be overwritten by the predicted `label_ids` of the model to the previous question. See "Usage" section for more info.
|
||||
- TAPAS is similar to BERT and therefore relies on the masked language modeling (MLM) objective. It is therefore
|
||||
efficient at predicting masked tokens and at NLU in general, but is not optimal for text generation. Models trained
|
||||
with a causal language modeling (CLM) objective are better in that regard.
|
||||
@@ -69,26 +69,25 @@ Tips:
|
||||
Usage: fine-tuning
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Here we explain how you can fine-tune :class:`~transformers.TapasForQuestionAnswering` on your own dataset.
|
||||
Here we explain how you can fine-tune :class:`~transformers.TapasForQuestionAnswering` on your own dataset.
|
||||
|
||||
=========================================================================== STEP 1: Choose one of the 3 ways in which
|
||||
you can use TAPAS - or experiment
|
||||
=======================================================================================================================
|
||||
===========================================================================
|
||||
STEP 1: Choose one of the 3 ways in which you can use TAPAS - or experiment
|
||||
===========================================================================
|
||||
|
||||
Basically, there are 3 different ways in which one can fine-tune :class:`~transformers.TapasForQuestionAnswering`,
|
||||
corresponding to the different datasets on which Tapas was fine-tuned:
|
||||
Basically, there are 3 different ways in which one can fine-tune :class:`~transformers.TapasForQuestionAnswering`, corresponding to
|
||||
the different datasets on which Tapas was fine-tuned:
|
||||
|
||||
1. SQA: if you're interested in asking follow-up questions related to a table, in a conversational set-up. For example
|
||||
if you first ask "what's the name of the first actor?" then you can ask a follow-up question such as "how old is
|
||||
he?". Here, questions do not involve any aggregation (all questions are cell selection questions).
|
||||
2. WTQ: if you're not interested in asking questions in a conversational set-up, but rather just asking questions
|
||||
related to a table, which might involve aggregation, such as counting a number of rows, summing up cell values or
|
||||
averaging cell values. You can then for example ask "what's the total number of goals Cristiano Ronaldo made in his
|
||||
career?". This case is also called **weak supervision**, since the model itself must learn the appropriate
|
||||
aggregation operator (SUM/COUNT/AVERAGE/NONE) given only the answer to the question as supervision.
|
||||
3. WikiSQL-supervised: this dataset is based on WikiSQL with the model being given the ground truth aggregation
|
||||
operator during training. This is also called **strong supervision**. Here, learning the appropriate aggregation
|
||||
operator is much easier.
|
||||
1. SQA: if you're interested in asking follow-up questions related to a table, in a conversational set-up. For example if you
|
||||
first ask "what's the name of the first actor?" then you can ask a follow-up question such as "how old is he?". Here, questions
|
||||
do not involve any aggregation (all questions are cell selection questions).
|
||||
2. WTQ: if you're not interested in asking questions in a conversational set-up, but rather just asking questions related
|
||||
to a table, which might involve aggregation, such as counting a number of rows, summing up cell values or averaging cell values.
|
||||
You can then for example ask "what's the total number of goals Cristiano Ronaldo made in his career?". This case is also called **weak
|
||||
supervision**, since the model itself must learn the appropriate aggregation operator (SUM/COUNT/AVERAGE/NONE) given only the answer
|
||||
to the question as supervision.
|
||||
3. WikiSQL-supervised: this dataset is based on WikiSQL with the model being given the ground truth aggregation operator during training.
|
||||
This is also called **strong supervision**. Here, learning the appropriate aggregation operator is much easier.
|
||||
|
||||
To summarize:
|
||||
|
||||
@@ -102,8 +101,7 @@ To summarize:
|
||||
| Strong supervision for aggregation | WikiSQL-supervised | Questions might involve aggregation, and the model must learn this given the gold aggregation operator |
|
||||
+------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
Initializing a model with a pre-trained base and randomly initialized classification heads from the model hub is as
|
||||
easy as:
|
||||
Initializing a model with a pre-trained base and randomly initialized classification heads from the model hub is as easy as:
|
||||
|
||||
.. code-block::
|
||||
|
||||
@@ -113,11 +111,10 @@ easy as:
|
||||
>>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base-uncased')
|
||||
|
||||
|
||||
Of course, you don't necessarily have to follow one of these three ways in which TAPAS was fine-tuned. You can also
|
||||
experiment by defining any hyperparameters you want when initializing :class:`~transformers.TapasConfig`, and then
|
||||
create a :class:`~transformers.TapasForQuestionAnswering` based on that configuration. For example, if you have a
|
||||
dataset that has both conversational questions and questions that might involve aggregation, then you can do it this
|
||||
way. Here's an example:
|
||||
Of course, you don't necessarily have to follow one of these three ways in which TAPAS was fine-tuned. You can also experiment by defining any hyperparameters
|
||||
you want when initializing :class:`~transformers.TapasConfig`, and then create a :class:`~transformers.TapasForQuestionAnswering` based on that
|
||||
configuration. For example, if you have a dataset that has both conversational questions and questions that might involve aggregation, then you can do it
|
||||
this way. Here's an example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
@@ -128,50 +125,43 @@ way. Here's an example:
|
||||
>>> # initializing the pre-trained base sized model with our custom classification heads
|
||||
>>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base-uncased', config=config)
|
||||
|
||||
What you can also do is start from an already fine-tuned checkpoint. A note here is that the already fine-tuned
|
||||
checkpoint on WTQ has some issues due to the L2-loss which is somewhat brittle. See `here
|
||||
<https://github.com/google-research/tapas/issues/91#issuecomment-735719340>`__ for more info.
|
||||
What you can also do is start from an already fine-tuned checkpoint. A note here is that the already fine-tuned checkpoint on WTQ has some issues
|
||||
due to the L2-loss which is somewhat brittle. See `here <https://github.com/google-research/tapas/issues/91#issuecomment-735719340>`__ for more info.
|
||||
|
||||
For a list of all pre-trained and fine-tuned TAPAS checkpoints available in the HuggingFace model hub, see `here
|
||||
<https://huggingface.co/models?search=tapas>`__.
|
||||
For a list of all pre-trained and fine-tuned TAPAS checkpoints available in the HuggingFace model hub, see `here <https://huggingface.co/models?search=tapas>`__.
|
||||
|
||||
=========================================== STEP 2: Prepare your data in the SQA format
|
||||
=======================================================================================================================
|
||||
===========================================
|
||||
STEP 2: Prepare your data in the SQA format
|
||||
===========================================
|
||||
|
||||
Second, no matter what you picked above, you should prepare your dataset in the `SQA format
|
||||
<https://www.microsoft.com/en-us/download/details.aspx?id=54253>`__. This format is a TSV/CSV file with the following
|
||||
columns:
|
||||
Second, no matter what you picked above, you should prepare your dataset in the `SQA format <https://www.microsoft.com/en-us/download/details.aspx?id=54253>`__.
|
||||
This format is a TSV/CSV file with the following columns:
|
||||
|
||||
- ``id``: optional, id of the table-question pair, for bookkeeping purposes.
|
||||
- ``annotator``: optional, id of the person who annotated the table-question pair, for bookkeeping purposes.
|
||||
- ``position``: integer indicating if the question is the first, second, third,... related to the table. Only required
|
||||
in case of conversational setup (SQA). You don't need this column in case you're going for WTQ/WikiSQL-supervised.
|
||||
- ``id``: optional, id of the table-question pair, for bookkeeping purposes.
|
||||
- ``annotator``: optional, id of the person who annotated the table-question pair, for bookkeeping purposes.
|
||||
- ``position``: integer indicating if the question is the first, second, third,... related to the table. Only required in case of conversational setup (SQA).
|
||||
You don't need this column in case you're going for WTQ/WikiSQL-supervised.
|
||||
- ``question``: string
|
||||
- ``table_file``: string, name of a csv file containing the tabular data
|
||||
- ``answer_coordinates``: list of one or more tuples (each tuple being a cell coordinate, i.e. row, column pair that is
|
||||
part of the answer)
|
||||
- ``answer_coordinates``: list of one or more tuples (each tuple being a cell coordinate, i.e. row, column pair that is part of the answer)
|
||||
- ``answer_text``: list of one or more strings (each string being a cell value that is part of the answer)
|
||||
- ``aggregation_label``: index of the aggregation operator. Only required in case of strong supervision for aggregation
|
||||
(the WikiSQL-supervised case)
|
||||
- ``float_answer``: the float answer to the question, if there is one (np.nan if there isn't). Only required in case of
|
||||
weak supervision for aggregation (such as WTQ and WikiSQL)
|
||||
- ``aggregation_label``: index of the aggregation operator. Only required in case of strong supervision for aggregation (the WikiSQL-supervised case)
|
||||
- ``float_answer``: the float answer to the question, if there is one (np.nan if there isn't). Only required in case of weak supervision for aggregation (such as WTQ and WikiSQL)
|
||||
|
||||
The tables themselves should be present in a folder, each table being a separate csv file. Note that the authors of the
|
||||
TAPAS algorithm used conversion scripts with some automated logic to convert the other datasets (WTQ, WikiSQL) into the
|
||||
SQA format. The author explains this `here
|
||||
<https://github.com/google-research/tapas/issues/50#issuecomment-705465960>`__. Interestingly, these conversion scripts
|
||||
are not perfect (the ``answer_coordinates`` and ``float_answer`` fields are populated based on the ``answer_text``),
|
||||
The tables themselves should be present in a folder, each table being a separate csv file. Note that the authors of the TAPAS algorithm used conversion
|
||||
scripts with some automated logic to convert the other datasets (WTQ, WikiSQL) into the SQA format. The author explains this `here <https://github.com/google-research/tapas/issues/50#issuecomment-705465960>`__.
|
||||
Interestingly, these conversion scripts are not perfect (the ``answer_coordinates`` and ``float_answer`` fields are populated based on the ``answer_text``),
|
||||
meaning that WTQ and WikiSQL results could actually be improved.
|
||||
|
||||
|
||||
========================================================================================== STEP 3: Convert your data
|
||||
into PyTorch tensors using :class:`~transformers.TapasTokenizer`
|
||||
=======================================================================================================================
|
||||
==========================================================================================
|
||||
STEP 3: Convert your data into PyTorch tensors using :class:`~transformers.TapasTokenizer`
|
||||
==========================================================================================
|
||||
|
||||
Third, given that you've prepared your data in this TSV/CSV format (and corresponding CSV files containing the tabular
|
||||
data), you can then use :class:`~transformers.TapasTokenizer` to convert table-question pairs into :obj:`input_ids`,
|
||||
:obj:`attention_mask`, :obj:`token_type_ids` and so on. Again, based on which of the three cases you picked above,
|
||||
:class:`~transformers.TapasForQuestionAnswering` requires different inputs to be fine-tuned:
|
||||
Third, given that you've prepared your data in this TSV/CSV format (and corresponding CSV files containing the tabular data), you can then
|
||||
use :class:`~transformers.TapasTokenizer` to convert table-question pairs into :obj:`input_ids`, :obj:`attention_mask`, :obj:`token_type_ids`
|
||||
and so on. Again, based on which of the three cases you picked above, :class:`~transformers.TapasForQuestionAnswering` requires different inputs
|
||||
to be fine-tuned:
|
||||
|
||||
+------------------------------------+----------------------------------------------------------------------------------------------+
|
||||
| **Task** | **Required inputs** |
|
||||
@@ -184,9 +174,9 @@ data), you can then use :class:`~transformers.TapasTokenizer` to convert table-q
|
||||
| Strong supervision for aggregation | ``input ids``, ``attention mask``, ``token type ids``, ``label ids``, ``aggregation_labels`` |
|
||||
+------------------------------------+----------------------------------------------------------------------------------------------+
|
||||
|
||||
:class:`~transformers.TapasTokenizer` creates the ``label_ids``, ``numeric_values`` and ``numeric_values_scale`` based
|
||||
on the ``answer_coordinates`` and ``answer_text`` columns of the TSV file. The ``float_answer`` and
|
||||
``aggregation_labels`` are already in the TSV file of step 2. Here's an example:
|
||||
:class:`~transformers.TapasTokenizer` creates the ``label_ids``, ``numeric_values`` and ``numeric_values_scale`` based on the
|
||||
``answer_coordinates`` and ``answer_text`` columns of the TSV file. The ``float_answer`` and ``aggregation_labels`` are already in the TSV file of step 2.
|
||||
Here's an example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
@@ -206,9 +196,8 @@ on the ``answer_coordinates`` and ``answer_text`` columns of the TSV file. The `
|
||||
{'input_ids': tensor([[ ... ]]), 'attention_mask': tensor([[...]]), 'token_type_ids': tensor([[[...]]]),
|
||||
'numeric_values': tensor([[ ... ]]), 'numeric_values_scale: tensor([[ ... ]]), label_ids: tensor([[ ... ]])}
|
||||
|
||||
Note that :class:`~transformers.TapasTokenizer` expects the data of the table to be text-only. You can use
|
||||
``.astype(str)`` on a dataframe to turn it into text-only data. Of course, this only shows how to encode a single
|
||||
training example. It is advised to create a PyTorch dataset and a corresponding dataloader:
|
||||
Note that :class:`~transformers.TapasTokenizer` expects the data of the table to be text-only. You can use ``.astype(str)`` on a dataframe to turn it into
|
||||
text-only data. Of course, this only shows how to encode a single training example. It is advised to create a PyTorch dataset and a corresponding dataloader:
|
||||
|
||||
.. code-block::
|
||||
|
||||
@@ -244,14 +233,14 @@ training example. It is advised to create a PyTorch dataset and a corresponding
|
||||
>>> train_dataset = TableDataset(data, tokenizer)
|
||||
>>> train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=32)
|
||||
|
||||
Note that here, we encode each table-question pair independently. This is fine as long as your dataset is **not
|
||||
conversational**. In case your dataset involves conversational questions (such as in SQA), then you should first group
|
||||
together the ``queries``, ``answer_coordinates`` and ``answer_text`` per table (in the order of their ``position``
|
||||
index) and batch encode each table with its questions. This will make sure that the ``prev_labels`` token types (see
|
||||
docs of :class:`~transformers.TapasTokenizer`) are set correctly.
|
||||
Note that here, we encode each table-question pair independently. This is fine as long as your dataset is **not conversational**. In case your
|
||||
dataset involves conversational questions (such as in SQA), then you should first group together the ``queries``, ``answer_coordinates`` and
|
||||
``answer_text`` per table (in the order of their ``position`` index) and batch encode each table with its questions. This will make sure that
|
||||
the ``prev_label_ids`` token types (see docs of :class:`~transformers.TapasTokenizer`) are set correctly.
|
||||
|
||||
=================================================== STEP 4: Train (fine-tune) TapasForQuestionAnswering
|
||||
=======================================================================================================================
|
||||
===================================================
|
||||
STEP 4: Train (fine-tune) TapasForQuestionAnswering
|
||||
===================================================
|
||||
|
||||
You can then fine-tune :class:`~transformers.TapasForQuestionAnswering` using native PyTorch as follows:
|
||||
|
||||
@@ -280,15 +269,14 @@ You can then fine-tune :class:`~transformers.TapasForQuestionAnswering` using na
|
||||
Usage: inference
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Here we explain how you can use :class:`~transformers.TapasForQuestionAnswering` for inference (i.e. making predictions
|
||||
on new data). For inference, only ``input_ids``, ``attention_mask`` and ``token_type_ids`` (which you can obtain using
|
||||
:class:`~transformers.TapasTokenizer`) have to be provided to the model to obtain the logits. Next, you can use the
|
||||
handy ``convert_logits_to_predictions`` method of :class:`~transformers.TapasTokenizer` to convert these into predicted
|
||||
coordinates and optional aggregation indices.
|
||||
Here we explain how you can use :class:`~transformers.TapasForQuestionAnswering` for inference (i.e. making predictions on new data).
|
||||
For inference, only ``input_ids``, ``attention_mask`` and ``token_type_ids`` (which you can obtain using
|
||||
:class:`~transformers.TapasTokenizer`) have to be provided to the model to obtain the logits. Next, you can use the handy
|
||||
``convert_logits_to_predictions`` method of :class:`~transformers.TapasTokenizer` to convert these into predicted coordinates
|
||||
and optional aggregation indices.
|
||||
|
||||
However, note that inference is **different** depending on whether or not the setup is conversational. In a
|
||||
non-conversational set-up, inference can be done in parallel on all table-question pairs of a batch. Here's an example
|
||||
of that:
|
||||
However, note that inference is **different** depending on whether or not the setup is conversational. In a non-conversational set-up, inference
|
||||
can be done in parallel on all table-question pairs of a batch. Here's an example of that:
|
||||
|
||||
.. code-block::
|
||||
|
||||
@@ -341,9 +329,8 @@ of that:
|
||||
What is the average number of movies?
|
||||
Predicted answer: AVERAGE > 87, 53, 69
|
||||
|
||||
In case of a conversational set-up, then each table-question pair must be provided **sequentially** to the model, such
|
||||
that the ``prev_labels`` token types can be overwritten by the predicted ``label_ids`` of the previous table-question
|
||||
pair.
|
||||
In case of a conversational set-up, then each table-question pair must be provided **sequentially** to the model, such that
|
||||
the ``prev_label_ids`` token types can be overwritten by the predicted ``label_ids`` of the previous table-question pair.
|
||||
|
||||
|
||||
Tapas specific outputs
|
||||
@@ -392,4 +379,4 @@ TapasForQuestionAnswering
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.TapasForQuestionAnswering
|
||||
:members:
|
||||
:members:
|
||||
+1
-1
@@ -55,7 +55,7 @@ git checkout tags/v3.4.0
|
||||
| [**`text-classification`**](https://github.com/huggingface/transformers/tree/master/examples/text-classification) | GLUE, XNLI | ✅ | ✅ | ✅ | [](https://github.com/huggingface/notebooks/blob/master/examples/text_classification.ipynb)
|
||||
| [**`token-classification`**](https://github.com/huggingface/transformers/tree/master/examples/token-classification) | CoNLL NER | ✅ | ✅ | ✅ | -
|
||||
| [**`multiple-choice`**](https://github.com/huggingface/transformers/tree/master/examples/multiple-choice) | SWAG, RACE, ARC | ✅ | ✅ | - | [](https://colab.research.google.com/github/ViktorAlm/notebooks/blob/master/MPC_GPU_Demo_for_TF_and_PT.ipynb)
|
||||
| [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering) | SQuAD | ✅ | ✅ | ✅ | -
|
||||
| [**`question-answering`**](https://github.com/huggingface/transformers/tree/master/examples/question-answering) | SQuAD | ✅ | ✅ | - | -
|
||||
| [**`text-generation`**](https://github.com/huggingface/transformers/tree/master/examples/text-generation) | - | n/a | n/a | - | [](https://colab.research.google.com/github/huggingface/blog/blob/master/notebooks/02_how_to_generate.ipynb)
|
||||
| [**`distillation`**](https://github.com/huggingface/transformers/tree/master/examples/distillation) | All | - | - | - | -
|
||||
| [**`summarization`**](https://github.com/huggingface/transformers/tree/master/examples/seq2seq) | CNN/Daily Mail | ✅ | - | - | -
|
||||
|
||||
@@ -1,636 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Team All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Fine-tuning the library models for masked language modeling (BERT, ALBERT, RoBERTa...) with whole word masking on a
|
||||
text file or a dataset.
|
||||
|
||||
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
|
||||
https://huggingface.co/models?filter=masked-lm
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
# You can also adapt this script on your own masked language modeling task. Pointers for this are left as comments.
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
from datasets import load_dataset
|
||||
from tqdm import tqdm
|
||||
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
from flax import jax_utils
|
||||
from flax.optim import Adam
|
||||
from flax.training import common_utils
|
||||
from flax.training.common_utils import get_metrics
|
||||
from jax.nn import log_softmax
|
||||
from transformers import (
|
||||
CONFIG_MAPPING,
|
||||
MODEL_FOR_MASKED_LM_MAPPING,
|
||||
AutoConfig,
|
||||
AutoTokenizer,
|
||||
FlaxBertForMaskedLM,
|
||||
HfArgumentParser,
|
||||
PreTrainedTokenizerBase,
|
||||
TensorType,
|
||||
TrainingArguments,
|
||||
is_tensorboard_available,
|
||||
set_seed,
|
||||
)
|
||||
|
||||
|
||||
# Cache the result
|
||||
has_tensorboard = is_tensorboard_available()
|
||||
if has_tensorboard:
|
||||
try:
|
||||
from flax.metrics.tensorboard import SummaryWriter
|
||||
except ImportError as ie:
|
||||
has_tensorboard = False
|
||||
print(f"Unable to display metrics through TensorBoard because some package are not installed: {ie}")
|
||||
|
||||
else:
|
||||
print(
|
||||
"Unable to display metrics through TensorBoard because the package is not installed: "
|
||||
"Please run pip install tensorboard to enable."
|
||||
)
|
||||
|
||||
|
||||
MODEL_CONFIG_CLASSES = list(MODEL_FOR_MASKED_LM_MAPPING.keys())
|
||||
MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelArguments:
|
||||
"""
|
||||
Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
|
||||
"""
|
||||
|
||||
model_name_or_path: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "The model checkpoint for weights initialization."
|
||||
"Don't set if you want to train a model from scratch."
|
||||
},
|
||||
)
|
||||
model_type: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
|
||||
)
|
||||
config_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
|
||||
)
|
||||
tokenizer_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
|
||||
)
|
||||
cache_dir: Optional[str] = field(
|
||||
default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
|
||||
)
|
||||
use_fast_tokenizer: bool = field(
|
||||
default=True,
|
||||
metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataTrainingArguments:
|
||||
"""
|
||||
Arguments pertaining to what data we are going to input our model for training and eval.
|
||||
"""
|
||||
|
||||
dataset_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
|
||||
)
|
||||
dataset_config_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
|
||||
)
|
||||
train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
|
||||
validation_file: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
|
||||
)
|
||||
train_ref_file: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "An optional input train ref data file for whole word masking in Chinese."},
|
||||
)
|
||||
validation_ref_file: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "An optional input validation ref data file for whole word masking in Chinese."},
|
||||
)
|
||||
overwrite_cache: bool = field(
|
||||
default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
|
||||
)
|
||||
max_seq_length: Optional[int] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "The maximum total input sequence length after tokenization. Sequences longer "
|
||||
"than this will be truncated. Default to the max input length of the model."
|
||||
},
|
||||
)
|
||||
preprocessing_num_workers: Optional[int] = field(
|
||||
default=None,
|
||||
metadata={"help": "The number of processes to use for the preprocessing."},
|
||||
)
|
||||
mlm_probability: float = field(
|
||||
default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
|
||||
)
|
||||
pad_to_max_length: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "Whether to pad all samples to `max_seq_length`. "
|
||||
"If False, will pad the samples dynamically when batching to the maximum length in the batch."
|
||||
},
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
if self.dataset_name is None and self.train_file is None and self.validation_file is None:
|
||||
raise ValueError("Need either a dataset name or a training/validation file.")
|
||||
else:
|
||||
if self.train_file is not None:
|
||||
extension = self.train_file.split(".")[-1]
|
||||
assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
|
||||
if self.validation_file is not None:
|
||||
extension = self.validation_file.split(".")[-1]
|
||||
assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."
|
||||
|
||||
|
||||
# Adapted from transformers/data/data_collator.py
|
||||
# Letting here for now, let's discuss where it should live
|
||||
@dataclass
|
||||
class FlaxDataCollatorForLanguageModeling:
|
||||
"""
|
||||
Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
|
||||
are not all of the same length.
|
||||
|
||||
Args:
|
||||
tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
|
||||
The tokenizer used for encoding the data.
|
||||
mlm (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to use masked language modeling. If set to :obj:`False`, the labels are the same as the
|
||||
inputs with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for
|
||||
non-masked tokens and the value to predict for the masked token.
|
||||
mlm_probability (:obj:`float`, `optional`, defaults to 0.15):
|
||||
The probability with which to (randomly) mask tokens in the input, when :obj:`mlm` is set to :obj:`True`.
|
||||
|
||||
.. note::
|
||||
|
||||
For best performance, this data collator should be used with a dataset having items that are dictionaries or
|
||||
BatchEncoding, with the :obj:`"special_tokens_mask"` key, as returned by a
|
||||
:class:`~transformers.PreTrainedTokenizer` or a :class:`~transformers.PreTrainedTokenizerFast` with the
|
||||
argument :obj:`return_special_tokens_mask=True`.
|
||||
"""
|
||||
|
||||
tokenizer: PreTrainedTokenizerBase
|
||||
mlm: bool = True
|
||||
mlm_probability: float = 0.15
|
||||
|
||||
def __post_init__(self):
|
||||
if self.mlm and self.tokenizer.mask_token is None:
|
||||
raise ValueError(
|
||||
"This tokenizer does not have a mask token which is necessary for masked language modeling. "
|
||||
"You should pass `mlm=False` to train on causal language modeling instead."
|
||||
)
|
||||
|
||||
def __call__(self, examples: List[Dict[str, np.ndarray]], pad_to_multiple_of: int) -> Dict[str, np.ndarray]:
|
||||
# Handle dict or lists with proper padding and conversion to tensor.
|
||||
batch = self.tokenizer.pad(examples, pad_to_multiple_of=pad_to_multiple_of, return_tensors=TensorType.NUMPY)
|
||||
|
||||
# If special token mask has been preprocessed, pop it from the dict.
|
||||
special_tokens_mask = batch.pop("special_tokens_mask", None)
|
||||
if self.mlm:
|
||||
batch["input_ids"], batch["labels"] = self.mask_tokens(
|
||||
batch["input_ids"], special_tokens_mask=special_tokens_mask
|
||||
)
|
||||
else:
|
||||
labels = batch["input_ids"].copy()
|
||||
if self.tokenizer.pad_token_id is not None:
|
||||
labels[labels == self.tokenizer.pad_token_id] = -100
|
||||
batch["labels"] = labels
|
||||
return batch
|
||||
|
||||
def mask_tokens(
|
||||
self, inputs: np.ndarray, special_tokens_mask: Optional[np.ndarray]
|
||||
) -> Tuple[jnp.ndarray, jnp.ndarray]:
|
||||
"""
|
||||
Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
|
||||
"""
|
||||
labels = inputs.copy()
|
||||
# We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`)
|
||||
probability_matrix = np.full(labels.shape, self.mlm_probability)
|
||||
special_tokens_mask = special_tokens_mask.astype("bool")
|
||||
|
||||
probability_matrix[special_tokens_mask] = 0.0
|
||||
masked_indices = np.random.binomial(1, probability_matrix).astype("bool")
|
||||
labels[~masked_indices] = -100 # We only compute loss on masked tokens
|
||||
|
||||
# 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
|
||||
indices_replaced = np.random.binomial(1, np.full(labels.shape, 0.8)).astype("bool") & masked_indices
|
||||
inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)
|
||||
|
||||
# 10% of the time, we replace masked input tokens with random word
|
||||
indices_random = np.random.binomial(1, np.full(labels.shape, 0.5)).astype("bool")
|
||||
indices_random &= masked_indices & ~indices_replaced
|
||||
|
||||
random_words = np.random.randint(self.tokenizer.vocab_size, size=labels.shape, dtype="i4")
|
||||
inputs[indices_random] = random_words[indices_random]
|
||||
|
||||
# The rest of the time (10% of the time) we keep the masked input tokens unchanged
|
||||
return inputs, labels
|
||||
|
||||
|
||||
def create_learning_rate_scheduler(
|
||||
factors="constant * linear_warmup * rsqrt_decay",
|
||||
base_learning_rate=0.5,
|
||||
warmup_steps=1000,
|
||||
decay_factor=0.5,
|
||||
steps_per_decay=20000,
|
||||
steps_per_cycle=100000,
|
||||
):
|
||||
"""Creates learning rate schedule.
|
||||
Interprets factors in the factors string which can consist of:
|
||||
* constant: interpreted as the constant value,
|
||||
* linear_warmup: interpreted as linear warmup until warmup_steps,
|
||||
* rsqrt_decay: divide by square root of max(step, warmup_steps)
|
||||
* rsqrt_normalized_decay: divide by square root of max(step/warmup_steps, 1)
|
||||
* decay_every: Every k steps decay the learning rate by decay_factor.
|
||||
* cosine_decay: Cyclic cosine decay, uses steps_per_cycle parameter.
|
||||
Args:
|
||||
factors: string, factors separated by "*" that defines the schedule.
|
||||
base_learning_rate: float, the starting constant for the lr schedule.
|
||||
warmup_steps: int, how many steps to warm up for in the warmup schedule.
|
||||
decay_factor: float, the amount to decay the learning rate by.
|
||||
steps_per_decay: int, how often to decay the learning rate.
|
||||
steps_per_cycle: int, steps per cycle when using cosine decay.
|
||||
Returns:
|
||||
a function learning_rate(step): float -> {"learning_rate": float}, the
|
||||
step-dependent lr.
|
||||
"""
|
||||
factors = [n.strip() for n in factors.split("*")]
|
||||
|
||||
def step_fn(step):
|
||||
"""Step to learning rate function."""
|
||||
ret = 1.0
|
||||
for name in factors:
|
||||
if name == "constant":
|
||||
ret *= base_learning_rate
|
||||
elif name == "linear_warmup":
|
||||
ret *= jnp.minimum(1.0, step / warmup_steps)
|
||||
elif name == "rsqrt_decay":
|
||||
ret /= jnp.sqrt(jnp.maximum(step, warmup_steps))
|
||||
elif name == "rsqrt_normalized_decay":
|
||||
ret *= jnp.sqrt(warmup_steps)
|
||||
ret /= jnp.sqrt(jnp.maximum(step, warmup_steps))
|
||||
elif name == "decay_every":
|
||||
ret *= decay_factor ** (step // steps_per_decay)
|
||||
elif name == "cosine_decay":
|
||||
progress = jnp.maximum(0.0, (step - warmup_steps) / float(steps_per_cycle))
|
||||
ret *= jnp.maximum(0.0, 0.5 * (1.0 + jnp.cos(jnp.pi * (progress % 1.0))))
|
||||
else:
|
||||
raise ValueError("Unknown factor %s." % name)
|
||||
return jnp.asarray(ret, dtype=jnp.float32)
|
||||
|
||||
return step_fn
|
||||
|
||||
|
||||
def compute_metrics(logits, labels, weights, label_smoothing=0.0):
|
||||
"""Compute summary metrics."""
|
||||
loss, normalizer = cross_entropy(logits, labels, weights, label_smoothing)
|
||||
acc, _ = accuracy(logits, labels, weights)
|
||||
metrics = {"loss": loss, "accuracy": acc, "normalizer": normalizer}
|
||||
metrics = jax.lax.psum(metrics, axis_name="batch")
|
||||
return metrics
|
||||
|
||||
|
||||
def accuracy(logits, targets, weights=None):
|
||||
"""Compute weighted accuracy for log probs and targets.
|
||||
Args:
|
||||
logits: [batch, length, num_classes] float array.
|
||||
targets: categorical targets [batch, length] int array.
|
||||
weights: None or array of shape [batch, length]
|
||||
Returns:
|
||||
Tuple of scalar loss and batch normalizing factor.
|
||||
"""
|
||||
if logits.ndim != targets.ndim + 1:
|
||||
raise ValueError(
|
||||
"Incorrect shapes. Got shape %s logits and %s targets" % (str(logits.shape), str(targets.shape))
|
||||
)
|
||||
|
||||
loss = jnp.equal(jnp.argmax(logits, axis=-1), targets)
|
||||
loss *= weights
|
||||
|
||||
return loss.sum(), weights.sum()
|
||||
|
||||
|
||||
def cross_entropy(logits, targets, weights=None, label_smoothing=0.0):
|
||||
"""Compute cross entropy and entropy for log probs and targets.
|
||||
Args:
|
||||
logits: [batch, length, num_classes] float array.
|
||||
targets: categorical targets [batch, length] int array.
|
||||
weights: None or array of shape [batch, length]
|
||||
label_smoothing: label smoothing constant, used to determine the on and off values.
|
||||
Returns:
|
||||
Tuple of scalar loss and batch normalizing factor.
|
||||
"""
|
||||
if logits.ndim != targets.ndim + 1:
|
||||
raise ValueError(
|
||||
"Incorrect shapes. Got shape %s logits and %s targets" % (str(logits.shape), str(targets.shape))
|
||||
)
|
||||
|
||||
vocab_size = logits.shape[-1]
|
||||
confidence = 1.0 - label_smoothing
|
||||
low_confidence = (1.0 - confidence) / (vocab_size - 1)
|
||||
normalizing_constant = -(
|
||||
confidence * jnp.log(confidence) + (vocab_size - 1) * low_confidence * jnp.log(low_confidence + 1e-20)
|
||||
)
|
||||
soft_targets = common_utils.onehot(targets, vocab_size, on_value=confidence, off_value=low_confidence)
|
||||
|
||||
loss = -jnp.sum(soft_targets * log_softmax(logits), axis=-1)
|
||||
loss = loss - normalizing_constant
|
||||
|
||||
if weights is not None:
|
||||
loss = loss * weights
|
||||
normalizing_factor = weights.sum()
|
||||
else:
|
||||
normalizing_factor = np.prod(targets.shape)
|
||||
|
||||
return loss.sum(), normalizing_factor
|
||||
|
||||
|
||||
def training_step(optimizer, batch, dropout_rng):
|
||||
dropout_rng, new_dropout_rng = jax.random.split(dropout_rng)
|
||||
|
||||
def loss_fn(params):
|
||||
targets = batch.pop("labels")
|
||||
|
||||
# Hide away tokens which doesn't participate in the optimization
|
||||
token_mask = jnp.where(targets > 0, 1.0, 0.0)
|
||||
|
||||
pooled, logits = model(**batch, params=params, dropout_rng=dropout_rng, train=True)
|
||||
loss, weight_sum = cross_entropy(logits, targets, token_mask)
|
||||
return loss / weight_sum
|
||||
|
||||
step = optimizer.state.step
|
||||
lr = lr_scheduler_fn(step)
|
||||
grad_fn = jax.value_and_grad(loss_fn)
|
||||
loss, grad = grad_fn(optimizer.target)
|
||||
grad = jax.lax.pmean(grad, "batch")
|
||||
optimizer = optimizer.apply_gradient(grad, learning_rate=lr)
|
||||
|
||||
return loss, optimizer, new_dropout_rng
|
||||
|
||||
|
||||
def eval_step(params, batch):
|
||||
"""
|
||||
Calculate evaluation metrics on a batch.
|
||||
"""
|
||||
targets = batch.pop("labels")
|
||||
|
||||
# Hide away tokens which doesn't participate in the optimization
|
||||
token_mask = jnp.where(targets > 0, 1.0, 0.0)
|
||||
_, logits = model(**batch, params=params, train=False)
|
||||
|
||||
return compute_metrics(logits, targets, token_mask)
|
||||
|
||||
|
||||
def generate_batch_splits(samples_idx: jnp.ndarray, batch_size: int) -> jnp.ndarray:
|
||||
nb_samples = len(samples_idx)
|
||||
samples_to_remove = nb_samples % batch_size
|
||||
|
||||
if samples_to_remove != 0:
|
||||
samples_idx = samples_idx[:-samples_to_remove]
|
||||
sections_split = nb_samples // batch_size
|
||||
batch_idx = jnp.split(samples_idx, sections_split)
|
||||
return batch_idx
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# See all possible arguments in src/transformers/training_args.py
|
||||
# or by passing the --help flag to this script.
|
||||
# We now keep distinct sets of args, for a cleaner separation of concerns.
|
||||
|
||||
parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
|
||||
if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
|
||||
# If we pass only one argument to the script and it's the path to a json file,
|
||||
# let's parse it to get our arguments.
|
||||
model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
|
||||
else:
|
||||
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
|
||||
|
||||
if (
|
||||
os.path.exists(training_args.output_dir)
|
||||
and os.listdir(training_args.output_dir)
|
||||
and training_args.do_train
|
||||
and not training_args.overwrite_output_dir
|
||||
):
|
||||
raise ValueError(
|
||||
f"Output directory ({training_args.output_dir}) already exists and is not empty."
|
||||
"Use --overwrite_output_dir to overcome."
|
||||
)
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
|
||||
level="NOTSET",
|
||||
datefmt="[%X]",
|
||||
)
|
||||
|
||||
# Log on each process the small summary:
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.warning(
|
||||
f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
|
||||
+ f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
|
||||
)
|
||||
|
||||
# Set the verbosity to info of the Transformers logger (on main process only):
|
||||
logger.info("Training/evaluation parameters %s", training_args)
|
||||
|
||||
# Set seed before initializing model.
|
||||
set_seed(training_args.seed)
|
||||
|
||||
# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
|
||||
# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
|
||||
# (the dataset will be downloaded automatically from the datasets Hub).
|
||||
#
|
||||
# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
|
||||
# 'text' is found. You can easily tweak this behavior (see below).
|
||||
#
|
||||
# In distributed training, the load_dataset function guarantees that only one local process can concurrently
|
||||
# download the dataset.
|
||||
if data_args.dataset_name is not None:
|
||||
# Downloading and loading a dataset from the hub.
|
||||
datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
|
||||
else:
|
||||
data_files = {}
|
||||
if data_args.train_file is not None:
|
||||
data_files["train"] = data_args.train_file
|
||||
if data_args.validation_file is not None:
|
||||
data_files["validation"] = data_args.validation_file
|
||||
extension = data_args.train_file.split(".")[-1]
|
||||
if extension == "txt":
|
||||
extension = "text"
|
||||
datasets = load_dataset(extension, data_files=data_files)
|
||||
# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
|
||||
# https://huggingface.co/docs/datasets/loading_datasets.html.
|
||||
|
||||
# Load pretrained model and tokenizer
|
||||
|
||||
# Distributed training:
|
||||
# The .from_pretrained methods guarantee that only one local process can concurrently
|
||||
# download model & vocab.
|
||||
if model_args.config_name:
|
||||
config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir)
|
||||
elif model_args.model_name_or_path:
|
||||
config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
|
||||
else:
|
||||
config = CONFIG_MAPPING[model_args.model_type]()
|
||||
logger.warning("You are instantiating a new config instance from scratch.")
|
||||
|
||||
if model_args.tokenizer_name:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_args.tokenizer_name, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer
|
||||
)
|
||||
elif model_args.model_name_or_path:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"You are instantiating a new tokenizer from scratch. This is not supported by this script."
|
||||
"You can do it from another script, save it, and load it from here, using --tokenizer_name."
|
||||
)
|
||||
|
||||
# Preprocessing the datasets.
|
||||
# First we tokenize all the texts.
|
||||
if training_args.do_train:
|
||||
column_names = datasets["train"].column_names
|
||||
else:
|
||||
column_names = datasets["validation"].column_names
|
||||
text_column_name = "text" if "text" in column_names else column_names[0]
|
||||
|
||||
padding = "max_length" if data_args.pad_to_max_length else False
|
||||
|
||||
def tokenize_function(examples):
|
||||
# Remove empty lines
|
||||
examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
|
||||
return tokenizer(
|
||||
examples["text"],
|
||||
return_special_tokens_mask=True,
|
||||
padding=padding,
|
||||
truncation=True,
|
||||
max_length=data_args.max_seq_length,
|
||||
)
|
||||
|
||||
tokenized_datasets = datasets.map(
|
||||
tokenize_function,
|
||||
batched=True,
|
||||
num_proc=data_args.preprocessing_num_workers,
|
||||
remove_columns=[text_column_name],
|
||||
load_from_cache_file=not data_args.overwrite_cache,
|
||||
)
|
||||
|
||||
# Enable tensorboard only on the master node
|
||||
if has_tensorboard and jax.host_id() == 0:
|
||||
summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir).joinpath("logs").as_posix())
|
||||
|
||||
# Data collator
|
||||
# This one will take care of randomly masking the tokens.
|
||||
data_collator = FlaxDataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=data_args.mlm_probability)
|
||||
|
||||
# Initialize our training
|
||||
rng = jax.random.PRNGKey(training_args.seed)
|
||||
dropout_rngs = jax.random.split(rng, jax.local_device_count())
|
||||
|
||||
model = FlaxBertForMaskedLM.from_pretrained("bert-base-cased", dtype=jnp.float32, dropout_rate=0.1)
|
||||
model.init(jax.random.PRNGKey(training_args.seed), (training_args.train_batch_size, model.config.max_length))
|
||||
|
||||
# Setup optimizer
|
||||
optimizer = Adam(
|
||||
learning_rate=training_args.learning_rate,
|
||||
weight_decay=training_args.weight_decay,
|
||||
beta1=training_args.adam_beta1,
|
||||
beta2=training_args.adam_beta2,
|
||||
).create(model.params)
|
||||
|
||||
# Create learning rate scheduler
|
||||
lr_scheduler_fn = create_learning_rate_scheduler(
|
||||
base_learning_rate=training_args.learning_rate, warmup_steps=training_args.warmup_steps
|
||||
)
|
||||
|
||||
# Create parallel version of the training and evaluation steps
|
||||
p_training_step = jax.pmap(training_step, "batch", donate_argnums=(0,))
|
||||
p_eval_step = jax.pmap(eval_step, "batch", donate_argnums=(0,))
|
||||
|
||||
# Replicate the optimizer on each device
|
||||
optimizer = jax_utils.replicate(optimizer)
|
||||
|
||||
# Store some constant
|
||||
nb_epochs = int(training_args.num_train_epochs)
|
||||
batch_size = int(training_args.train_batch_size)
|
||||
eval_batch_size = int(training_args.eval_batch_size)
|
||||
|
||||
epochs = tqdm(range(nb_epochs), desc=f"Epoch ... (1/{nb_epochs})", position=0)
|
||||
for epoch in epochs:
|
||||
|
||||
# ======================== Training ================================
|
||||
# Create sampling rng
|
||||
rng, training_rng, eval_rng = jax.random.split(rng, 3)
|
||||
|
||||
# Generate an epoch by shuffling sampling indices from the train dataset
|
||||
nb_training_samples = len(tokenized_datasets["train"])
|
||||
training_samples_idx = jax.random.permutation(training_rng, jnp.arange(nb_training_samples))
|
||||
training_batch_idx = generate_batch_splits(training_samples_idx, batch_size)
|
||||
|
||||
# Gather the indexes for creating the batch and do a training step
|
||||
for batch_idx in tqdm(training_batch_idx, desc="Training...", position=1):
|
||||
samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx]
|
||||
model_inputs = data_collator(samples, pad_to_multiple_of=16)
|
||||
|
||||
# Model forward
|
||||
model_inputs = common_utils.shard(model_inputs.data)
|
||||
loss, optimizer, dropout_rngs = p_training_step(optimizer, model_inputs, dropout_rngs)
|
||||
|
||||
epochs.write(f"Loss: {loss}")
|
||||
|
||||
# ======================== Evaluating ==============================
|
||||
nb_eval_samples = len(tokenized_datasets["test"])
|
||||
eval_samples_idx = jnp.arange(nb_eval_samples)
|
||||
eval_batch_idx = generate_batch_splits(eval_samples_idx, eval_batch_size)
|
||||
|
||||
eval_metrics = []
|
||||
for i, batch_idx in enumerate(tqdm(eval_batch_idx, desc="Evaluating ...", position=2)):
|
||||
samples = [tokenized_datasets["test"][int(idx)] for idx in batch_idx]
|
||||
model_inputs = data_collator(samples, pad_to_multiple_of=16)
|
||||
|
||||
# Model forward
|
||||
model_inputs = common_utils.shard(model_inputs.data)
|
||||
metrics = p_eval_step(optimizer.target, model_inputs)
|
||||
eval_metrics.append(metrics)
|
||||
|
||||
eval_metrics_np = get_metrics(eval_metrics)
|
||||
eval_metrics_np = jax.tree_map(jnp.sum, eval_metrics_np)
|
||||
eval_normalizer = eval_metrics_np.pop("normalizer")
|
||||
eval_summary = jax.tree_map(lambda x: x / eval_normalizer, eval_metrics_np)
|
||||
|
||||
# Update progress bar
|
||||
epochs.desc = (
|
||||
f"Epoch... ({epoch + 1}/{nb_epochs} | Loss: {eval_summary['loss']}, Acc: {eval_summary['accuracy']})"
|
||||
)
|
||||
|
||||
# Save metrics
|
||||
if has_tensorboard and jax.host_id() == 0:
|
||||
for name, value in eval_summary.items():
|
||||
summary_writer.scalar(name, value, epoch)
|
||||
@@ -7,17 +7,33 @@ Based on the script [`run_squad.py`](https://github.com/huggingface/transformers
|
||||
#### Fine-tuning BERT on SQuAD1.0
|
||||
|
||||
This example code fine-tunes BERT on the SQuAD1.0 dataset. It runs in 24 min (with BERT-base) or 68 min (with BERT-large)
|
||||
on a single tesla V100 16GB.
|
||||
on a single tesla V100 16GB. The data for SQuAD can be downloaded with the following links and should be saved in a
|
||||
$SQUAD_DIR directory.
|
||||
|
||||
* [train-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json)
|
||||
* [dev-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json)
|
||||
* [evaluate-v1.1.py](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py)
|
||||
|
||||
And for SQuAD2.0, you need to download:
|
||||
|
||||
- [train-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json)
|
||||
- [dev-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json)
|
||||
- [evaluate-v2.0.py](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/)
|
||||
|
||||
```bash
|
||||
python run_qa.py \
|
||||
export SQUAD_DIR=/path/to/SQUAD
|
||||
|
||||
python run_squad.py \
|
||||
--model_type bert \
|
||||
--model_name_or_path bert-base-uncased \
|
||||
--dataset_name squad \
|
||||
--do_train \
|
||||
--do_eval \
|
||||
--per_device_train_batch_size 12 \
|
||||
--do_lower_case \
|
||||
--train_file $SQUAD_DIR/train-v1.1.json \
|
||||
--predict_file $SQUAD_DIR/dev-v1.1.json \
|
||||
--per_gpu_train_batch_size 12 \
|
||||
--learning_rate 3e-5 \
|
||||
--num_train_epochs 2 \
|
||||
--num_train_epochs 2.0 \
|
||||
--max_seq_length 384 \
|
||||
--doc_stride 128 \
|
||||
--output_dir /tmp/debug_squad/
|
||||
@@ -37,17 +53,20 @@ Here is an example using distributed training on 8 V100 GPUs and Bert Whole Word
|
||||
|
||||
```bash
|
||||
python -m torch.distributed.launch --nproc_per_node=8 ./examples/question-answering/run_squad.py \
|
||||
--model_type bert \
|
||||
--model_name_or_path bert-large-uncased-whole-word-masking \
|
||||
--dataset_name squad \
|
||||
--do_train \
|
||||
--do_eval \
|
||||
--do_lower_case \
|
||||
--train_file $SQUAD_DIR/train-v1.1.json \
|
||||
--predict_file $SQUAD_DIR/dev-v1.1.json \
|
||||
--learning_rate 3e-5 \
|
||||
--num_train_epochs 2 \
|
||||
--max_seq_length 384 \
|
||||
--doc_stride 128 \
|
||||
--output_dir ./examples/models/wwm_uncased_finetuned_squad/ \
|
||||
--per_device_eval_batch_size=3 \
|
||||
--per_device_train_batch_size=3 \
|
||||
--per_gpu_eval_batch_size=3 \
|
||||
--per_gpu_train_batch_size=3 \
|
||||
```
|
||||
|
||||
Training with the previously defined hyper-parameters yields the following results:
|
||||
@@ -60,25 +79,29 @@ exact_match = 86.91
|
||||
This fine-tuned model is available as a checkpoint under the reference
|
||||
[`bert-large-uncased-whole-word-masking-finetuned-squad`](https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad).
|
||||
|
||||
#### Fine-tuning XLNet with beam search on SQuAD
|
||||
#### Fine-tuning XLNet on SQuAD
|
||||
|
||||
This example code fine-tunes XLNet on both SQuAD1.0 and SQuAD2.0 dataset.
|
||||
This example code fine-tunes XLNet on both SQuAD1.0 and SQuAD2.0 dataset. See above to download the data for SQuAD .
|
||||
|
||||
##### Command for SQuAD1.0:
|
||||
|
||||
```bash
|
||||
python run_qa_beam_search.py \
|
||||
export SQUAD_DIR=/path/to/SQUAD
|
||||
|
||||
python run_squad.py \
|
||||
--model_type xlnet \
|
||||
--model_name_or_path xlnet-large-cased \
|
||||
--dataset_name squad \
|
||||
--do_train \
|
||||
--do_eval \
|
||||
--train_file $SQUAD_DIR/train-v1.1.json \
|
||||
--predict_file $SQUAD_DIR/dev-v1.1.json \
|
||||
--learning_rate 3e-5 \
|
||||
--num_train_epochs 2 \
|
||||
--max_seq_length 384 \
|
||||
--doc_stride 128 \
|
||||
--output_dir ./wwm_cased_finetuned_squad/ \
|
||||
--per_device_eval_batch_size=4 \
|
||||
--per_device_train_batch_size=4 \
|
||||
--per_gpu_eval_batch_size=4 \
|
||||
--per_gpu_train_batch_size=4 \
|
||||
--save_steps 5000
|
||||
```
|
||||
|
||||
@@ -87,19 +110,21 @@ python run_qa_beam_search.py \
|
||||
```bash
|
||||
export SQUAD_DIR=/path/to/SQUAD
|
||||
|
||||
python run_qa_beam_search.py \
|
||||
python run_squad.py \
|
||||
--model_type xlnet \
|
||||
--model_name_or_path xlnet-large-cased \
|
||||
--dataset_name squad_v2 \
|
||||
--do_train \
|
||||
--do_eval \
|
||||
--version_2_with_negative \
|
||||
--train_file $SQUAD_DIR/train-v2.0.json \
|
||||
--predict_file $SQUAD_DIR/dev-v2.0.json \
|
||||
--learning_rate 3e-5 \
|
||||
--num_train_epochs 4 \
|
||||
--max_seq_length 384 \
|
||||
--doc_stride 128 \
|
||||
--output_dir ./wwm_cased_finetuned_squad/ \
|
||||
--per_device_eval_batch_size=2 \
|
||||
--per_device_train_batch_size=2 \
|
||||
--per_gpu_eval_batch_size=2 \
|
||||
--per_gpu_train_batch_size=2 \
|
||||
--save_steps 5000
|
||||
```
|
||||
|
||||
@@ -137,7 +162,7 @@ Larger batch size may improve the performance while costing more memory.
|
||||
#### Fine-tuning BERT on SQuAD1.0 with relative position embeddings
|
||||
|
||||
The following examples show how to fine-tune BERT models with different relative position embeddings. The BERT model
|
||||
`bert-base-uncased` was pretrained with default absolute position embeddings. We provide the following pretrained
|
||||
`bert-base-uncased` was pre-trained with default absolute position embeddings. We provide the following pre-trained
|
||||
models which were pre-trained on the same training data (BooksCorpus and English Wikipedia) as in the BERT model
|
||||
training, but with different relative position embeddings.
|
||||
|
||||
@@ -153,19 +178,24 @@ in Huang et al. [Improve Transformer Models with Better Relative Position Embedd
|
||||
##### Base models fine-tuning
|
||||
|
||||
```bash
|
||||
export SQUAD_DIR=/path/to/SQUAD
|
||||
output_dir=relative_squad
|
||||
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
||||
python -m torch.distributed.launch --nproc_per_node=8 ./examples/question-answering/run_squad.py \
|
||||
--model_type bert \
|
||||
--model_name_or_path zhiheng-huang/bert-base-uncased-embedding-relative-key-query \
|
||||
--dataset_name squad \
|
||||
--do_train \
|
||||
--do_eval \
|
||||
--do_lower_case \
|
||||
--train_file $SQUAD_DIR/train-v1.1.json \
|
||||
--predict_file $SQUAD_DIR/dev-v1.1.json \
|
||||
--learning_rate 3e-5 \
|
||||
--num_train_epochs 2 \
|
||||
--max_seq_length 512 \
|
||||
--doc_stride 128 \
|
||||
--output_dir relative_squad \
|
||||
--per_device_eval_batch_size=60 \
|
||||
--per_device_train_batch_size=6
|
||||
--output_dir ${output_dir} \
|
||||
--per_gpu_eval_batch_size=60 \
|
||||
--per_gpu_train_batch_size=6
|
||||
```
|
||||
Training with the above command leads to the following results. It boosts the BERT default from f1 score of 88.52 to 90.54.
|
||||
|
||||
@@ -181,17 +211,22 @@ gpu training leads to the f1 score of 90.71.
|
||||
##### Large models fine-tuning
|
||||
|
||||
```bash
|
||||
export SQUAD_DIR=/path/to/SQUAD
|
||||
output_dir=relative_squad
|
||||
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
||||
python -m torch.distributed.launch --nproc_per_node=8 ./examples/question-answering/run_squad.py \
|
||||
--model_type bert \
|
||||
--model_name_or_path zhiheng-huang/bert-large-uncased-whole-word-masking-embedding-relative-key-query \
|
||||
--dataset_name squad \
|
||||
--do_train \
|
||||
--do_eval \
|
||||
--do_lower_case \
|
||||
--train_file $SQUAD_DIR/train-v1.1.json \
|
||||
--predict_file $SQUAD_DIR/dev-v1.1.json \
|
||||
--learning_rate 3e-5 \
|
||||
--num_train_epochs 2 \
|
||||
--max_seq_length 512 \
|
||||
--doc_stride 128 \
|
||||
--output_dir relative_squad \
|
||||
--output_dir ${output_dir} \
|
||||
--per_gpu_eval_batch_size=6 \
|
||||
--per_gpu_train_batch_size=2 \
|
||||
--gradient_accumulation_steps 3
|
||||
@@ -216,4 +251,5 @@ python run_tf_squad.py \
|
||||
--doc_stride 128
|
||||
```
|
||||
|
||||
|
||||
For the moment evaluation is not available in the Tensorflow Trainer only the training.
|
||||
@@ -1,469 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Team All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Fine-tuning the library models for question answering.
|
||||
"""
|
||||
# You can also adapt this script on your own question answering task. Pointers for this are left as comments.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
from datasets import load_dataset, load_metric
|
||||
|
||||
import transformers
|
||||
from trainer_qa import QuestionAnsweringTrainer
|
||||
from transformers import (
|
||||
AutoConfig,
|
||||
AutoModelForQuestionAnswering,
|
||||
AutoTokenizer,
|
||||
DataCollatorWithPadding,
|
||||
EvalPrediction,
|
||||
HfArgumentParser,
|
||||
PreTrainedTokenizerFast,
|
||||
TrainingArguments,
|
||||
default_data_collator,
|
||||
set_seed,
|
||||
)
|
||||
from transformers.trainer_utils import is_main_process
|
||||
from utils_qa import postprocess_qa_predictions
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelArguments:
|
||||
"""
|
||||
Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
|
||||
"""
|
||||
|
||||
model_name_or_path: str = field(
|
||||
metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
|
||||
)
|
||||
config_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
|
||||
)
|
||||
tokenizer_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
|
||||
)
|
||||
cache_dir: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "Path to directory to store the pretrained models downloaded from huggingface.co"},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataTrainingArguments:
|
||||
"""
|
||||
Arguments pertaining to what data we are going to input our model for training and eval.
|
||||
"""
|
||||
|
||||
dataset_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
|
||||
)
|
||||
dataset_config_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
|
||||
)
|
||||
train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
|
||||
validation_file: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
|
||||
)
|
||||
overwrite_cache: bool = field(
|
||||
default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
|
||||
)
|
||||
preprocessing_num_workers: Optional[int] = field(
|
||||
default=None,
|
||||
metadata={"help": "The number of processes to use for the preprocessing."},
|
||||
)
|
||||
max_seq_length: int = field(
|
||||
default=384,
|
||||
metadata={
|
||||
"help": "The maximum total input sequence length after tokenization. Sequences longer "
|
||||
"than this will be truncated, sequences shorter will be padded."
|
||||
},
|
||||
)
|
||||
pad_to_max_length: bool = field(
|
||||
default=True,
|
||||
metadata={
|
||||
"help": "Whether to pad all samples to `max_seq_length`. "
|
||||
"If False, will pad the samples dynamically when batching to the maximum length in the batch (which can "
|
||||
"be faster on GPU but will be slower on TPU)."
|
||||
},
|
||||
)
|
||||
version_2_with_negative: bool = field(
|
||||
default=False, metadata={"help": "If true, some of the examples do not have an answer."}
|
||||
)
|
||||
null_score_diff_threshold: float = field(
|
||||
default=0.0,
|
||||
metadata={
|
||||
"help": "The threshold used to select the null answer: if the best answer has a score that is less than "
|
||||
"the score of the null answer minus this threshold, the null answer is selected for this example. "
|
||||
"Only useful when `version_2_with_negative=True`."
|
||||
},
|
||||
)
|
||||
doc_stride: int = field(
|
||||
default=128,
|
||||
metadata={"help": "When splitting up a long document into chunks, how much stride to take between chunks."},
|
||||
)
|
||||
n_best_size: int = field(
|
||||
default=20,
|
||||
metadata={"help": "The total number of n-best predictions to generate when looking for an answer."},
|
||||
)
|
||||
max_answer_length: int = field(
|
||||
default=30,
|
||||
metadata={
|
||||
"help": "The maximum length of an answer that can be generated. This is needed because the start "
|
||||
"and end predictions are not conditioned on one another."
|
||||
},
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
if self.dataset_name is None and self.train_file is None and self.validation_file is None:
|
||||
raise ValueError("Need either a dataset name or a training/validation file.")
|
||||
else:
|
||||
if self.train_file is not None:
|
||||
extension = self.train_file.split(".")[-1]
|
||||
assert extension in ["csv", "json"], "`train_file` should be a csv or a json file."
|
||||
if self.validation_file is not None:
|
||||
extension = self.validation_file.split(".")[-1]
|
||||
assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file."
|
||||
|
||||
|
||||
def main():
|
||||
# See all possible arguments in src/transformers/training_args.py
|
||||
# or by passing the --help flag to this script.
|
||||
# We now keep distinct sets of args, for a cleaner separation of concerns.
|
||||
|
||||
parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
|
||||
if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
|
||||
# If we pass only one argument to the script and it's the path to a json file,
|
||||
# let's parse it to get our arguments.
|
||||
model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
|
||||
else:
|
||||
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
|
||||
|
||||
if (
|
||||
os.path.exists(training_args.output_dir)
|
||||
and os.listdir(training_args.output_dir)
|
||||
and training_args.do_train
|
||||
and not training_args.overwrite_output_dir
|
||||
):
|
||||
raise ValueError(
|
||||
f"Output directory ({training_args.output_dir}) already exists and is not empty."
|
||||
"Use --overwrite_output_dir to overcome."
|
||||
)
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
|
||||
datefmt="%m/%d/%Y %H:%M:%S",
|
||||
)
|
||||
logger.setLevel(logging.INFO if is_main_process(training_args.local_rank) else logging.WARN)
|
||||
|
||||
# Log on each process the small summary:
|
||||
logger.warning(
|
||||
f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
|
||||
+ f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
|
||||
)
|
||||
# Set the verbosity to info of the Transformers logger (on main process only):
|
||||
if is_main_process(training_args.local_rank):
|
||||
transformers.utils.logging.set_verbosity_info()
|
||||
logger.info("Training/evaluation parameters %s", training_args)
|
||||
|
||||
# Set seed before initializing model.
|
||||
set_seed(training_args.seed)
|
||||
|
||||
# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
|
||||
# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
|
||||
# (the dataset will be downloaded automatically from the datasets Hub).
|
||||
#
|
||||
# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
|
||||
# 'text' is found. You can easily tweak this behavior (see below).
|
||||
#
|
||||
# In distributed training, the load_dataset function guarantee that only one local process can concurrently
|
||||
# download the dataset.
|
||||
if data_args.dataset_name is not None:
|
||||
# Downloading and loading a dataset from the hub.
|
||||
datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
|
||||
else:
|
||||
data_files = {}
|
||||
if data_args.train_file is not None:
|
||||
data_files["train"] = data_args.train_file
|
||||
if data_args.validation_file is not None:
|
||||
data_files["validation"] = data_args.validation_file
|
||||
extension = data_args.train_file.split(".")[-1]
|
||||
datasets = load_dataset(extension, data_files=data_files, field="data")
|
||||
# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
|
||||
# https://huggingface.co/docs/datasets/loading_datasets.html.
|
||||
|
||||
# Load pretrained model and tokenizer
|
||||
#
|
||||
# Distributed training:
|
||||
# The .from_pretrained methods guarantee that only one local process can concurrently
|
||||
# download model & vocab.
|
||||
config = AutoConfig.from_pretrained(
|
||||
model_args.config_name if model_args.config_name else model_args.model_name_or_path,
|
||||
cache_dir=model_args.cache_dir,
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
|
||||
cache_dir=model_args.cache_dir,
|
||||
use_fast=True,
|
||||
)
|
||||
model = AutoModelForQuestionAnswering.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
from_tf=bool(".ckpt" in model_args.model_name_or_path),
|
||||
config=config,
|
||||
cache_dir=model_args.cache_dir,
|
||||
)
|
||||
|
||||
# Tokenizer check: this script requires a fast tokenizer.
|
||||
if not isinstance(tokenizer, PreTrainedTokenizerFast):
|
||||
raise ValueError(
|
||||
"This example script only works for models that have a fast tokenizer. Checkout the big table of models "
|
||||
"at https://huggingface.co/transformers/index.html#bigtable to find the model types that meet this "
|
||||
"requirement"
|
||||
)
|
||||
|
||||
# Preprocessing the datasets.
|
||||
# Preprocessing is slighlty different for training and evaluation.
|
||||
if training_args.do_train:
|
||||
column_names = datasets["train"].column_names
|
||||
else:
|
||||
column_names = datasets["validation"].column_names
|
||||
question_column_name = "question" if "question" in column_names else column_names[0]
|
||||
context_column_name = "context" if "context" in column_names else column_names[1]
|
||||
answer_column_name = "answers" if "answers" in column_names else column_names[2]
|
||||
|
||||
# Padding side determines if we do (question|context) or (context|question).
|
||||
pad_on_right = tokenizer.padding_side == "right"
|
||||
|
||||
# Training preprocessing
|
||||
def prepare_train_features(examples):
|
||||
# Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results
|
||||
# in one example possible giving several features when a context is long, each of those features having a
|
||||
# context that overlaps a bit the context of the previous feature.
|
||||
tokenized_examples = tokenizer(
|
||||
examples[question_column_name if pad_on_right else context_column_name],
|
||||
examples[context_column_name if pad_on_right else question_column_name],
|
||||
truncation="only_second" if pad_on_right else "only_first",
|
||||
max_length=data_args.max_seq_length,
|
||||
stride=data_args.doc_stride,
|
||||
return_overflowing_tokens=True,
|
||||
return_offsets_mapping=True,
|
||||
padding="max_length" if data_args.pad_to_max_length else False,
|
||||
)
|
||||
|
||||
# Since one example might give us several features if it has a long context, we need a map from a feature to
|
||||
# its corresponding example. This key gives us just that.
|
||||
sample_mapping = tokenized_examples.pop("overflow_to_sample_mapping")
|
||||
# The offset mappings will give us a map from token to character position in the original context. This will
|
||||
# help us compute the start_positions and end_positions.
|
||||
offset_mapping = tokenized_examples.pop("offset_mapping")
|
||||
|
||||
# Let's label those examples!
|
||||
tokenized_examples["start_positions"] = []
|
||||
tokenized_examples["end_positions"] = []
|
||||
|
||||
for i, offsets in enumerate(offset_mapping):
|
||||
# We will label impossible answers with the index of the CLS token.
|
||||
input_ids = tokenized_examples["input_ids"][i]
|
||||
cls_index = input_ids.index(tokenizer.cls_token_id)
|
||||
|
||||
# Grab the sequence corresponding to that example (to know what is the context and what is the question).
|
||||
sequence_ids = tokenized_examples.sequence_ids(i)
|
||||
|
||||
# One example can give several spans, this is the index of the example containing this span of text.
|
||||
sample_index = sample_mapping[i]
|
||||
answers = examples[answer_column_name][sample_index]
|
||||
# If no answers are given, set the cls_index as answer.
|
||||
if len(answers["answer_start"]) == 0:
|
||||
tokenized_examples["start_positions"].append(cls_index)
|
||||
tokenized_examples["end_positions"].append(cls_index)
|
||||
else:
|
||||
# Start/end character index of the answer in the text.
|
||||
start_char = answers["answer_start"][0]
|
||||
end_char = start_char + len(answers["text"][0])
|
||||
|
||||
# Start token index of the current span in the text.
|
||||
token_start_index = 0
|
||||
while sequence_ids[token_start_index] != (1 if pad_on_right else 0):
|
||||
token_start_index += 1
|
||||
|
||||
# End token index of the current span in the text.
|
||||
token_end_index = len(input_ids) - 1
|
||||
while sequence_ids[token_end_index] != (1 if pad_on_right else 0):
|
||||
token_end_index -= 1
|
||||
|
||||
# Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).
|
||||
if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):
|
||||
tokenized_examples["start_positions"].append(cls_index)
|
||||
tokenized_examples["end_positions"].append(cls_index)
|
||||
else:
|
||||
# Otherwise move the token_start_index and token_end_index to the two ends of the answer.
|
||||
# Note: we could go after the last offset if the answer is the last word (edge case).
|
||||
while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:
|
||||
token_start_index += 1
|
||||
tokenized_examples["start_positions"].append(token_start_index - 1)
|
||||
while offsets[token_end_index][1] >= end_char:
|
||||
token_end_index -= 1
|
||||
tokenized_examples["end_positions"].append(token_end_index + 1)
|
||||
|
||||
return tokenized_examples
|
||||
|
||||
if training_args.do_train:
|
||||
train_dataset = datasets["train"].map(
|
||||
prepare_train_features,
|
||||
batched=True,
|
||||
num_proc=data_args.preprocessing_num_workers,
|
||||
remove_columns=column_names,
|
||||
load_from_cache_file=not data_args.overwrite_cache,
|
||||
)
|
||||
|
||||
# Validation preprocessing
|
||||
def prepare_validation_features(examples):
|
||||
# Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results
|
||||
# in one example possible giving several features when a context is long, each of those features having a
|
||||
# context that overlaps a bit the context of the previous feature.
|
||||
tokenized_examples = tokenizer(
|
||||
examples[question_column_name if pad_on_right else context_column_name],
|
||||
examples[context_column_name if pad_on_right else question_column_name],
|
||||
truncation="only_second" if pad_on_right else "only_first",
|
||||
max_length=data_args.max_seq_length,
|
||||
stride=data_args.doc_stride,
|
||||
return_overflowing_tokens=True,
|
||||
return_offsets_mapping=True,
|
||||
padding="max_length" if data_args.pad_to_max_length else False,
|
||||
)
|
||||
|
||||
# Since one example might give us several features if it has a long context, we need a map from a feature to
|
||||
# its corresponding example. This key gives us just that.
|
||||
sample_mapping = tokenized_examples.pop("overflow_to_sample_mapping")
|
||||
|
||||
# For evaluation, we will need to convert our predictions to substrings of the context, so we keep the
|
||||
# corresponding example_id and we will store the offset mappings.
|
||||
tokenized_examples["example_id"] = []
|
||||
|
||||
for i in range(len(tokenized_examples["input_ids"])):
|
||||
# Grab the sequence corresponding to that example (to know what is the context and what is the question).
|
||||
sequence_ids = tokenized_examples.sequence_ids(i)
|
||||
context_index = 1 if pad_on_right else 0
|
||||
|
||||
# One example can give several spans, this is the index of the example containing this span of text.
|
||||
sample_index = sample_mapping[i]
|
||||
tokenized_examples["example_id"].append(examples["id"][sample_index])
|
||||
|
||||
# Set to None the offset_mapping that are not part of the context so it's easy to determine if a token
|
||||
# position is part of the context or not.
|
||||
tokenized_examples["offset_mapping"][i] = [
|
||||
(o if sequence_ids[k] == context_index else None)
|
||||
for k, o in enumerate(tokenized_examples["offset_mapping"][i])
|
||||
]
|
||||
|
||||
return tokenized_examples
|
||||
|
||||
if training_args.do_eval:
|
||||
validation_dataset = datasets["validation"].map(
|
||||
prepare_validation_features,
|
||||
batched=True,
|
||||
num_proc=data_args.preprocessing_num_workers,
|
||||
remove_columns=column_names,
|
||||
load_from_cache_file=not data_args.overwrite_cache,
|
||||
)
|
||||
|
||||
# Data collator
|
||||
# We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data
|
||||
# collator.
|
||||
data_collator = default_data_collator if data_args.pad_to_max_length else DataCollatorWithPadding(tokenizer)
|
||||
|
||||
# Post-processing:
|
||||
def post_processing_function(examples, features, predictions):
|
||||
# Post-processing: we match the start logits and end logits to answers in the original context.
|
||||
predictions = postprocess_qa_predictions(
|
||||
examples=examples,
|
||||
features=features,
|
||||
predictions=predictions,
|
||||
version_2_with_negative=data_args.version_2_with_negative,
|
||||
n_best_size=data_args.n_best_size,
|
||||
max_answer_length=data_args.max_answer_length,
|
||||
null_score_diff_threshold=data_args.null_score_diff_threshold,
|
||||
output_dir=training_args.output_dir,
|
||||
is_world_process_zero=trainer.is_world_process_zero(),
|
||||
)
|
||||
# Format the result to the format the metric expects.
|
||||
if data_args.version_2_with_negative:
|
||||
formatted_predictions = [
|
||||
{"id": k, "prediction_text": v, "no_answer_probability": 0.0} for k, v in predictions.items()
|
||||
]
|
||||
else:
|
||||
formatted_predictions = [{"id": k, "prediction_text": v} for k, v in predictions.items()]
|
||||
references = [{"id": ex["id"], "answers": ex[answer_column_name]} for ex in datasets["validation"]]
|
||||
return EvalPrediction(predictions=formatted_predictions, label_ids=references)
|
||||
|
||||
# TODO: Once the fix lands in a Datasets release, remove the _local here and the squad_v2_local folder.
|
||||
current_dir = os.path.sep.join(os.path.join(__file__).split(os.path.sep)[:-1])
|
||||
metric = load_metric(os.path.join(current_dir, "squad_v2_local") if data_args.version_2_with_negative else "squad")
|
||||
|
||||
def compute_metrics(p: EvalPrediction):
|
||||
return metric.compute(predictions=p.predictions, references=p.label_ids)
|
||||
|
||||
# Initialize our Trainer
|
||||
trainer = QuestionAnsweringTrainer(
|
||||
model=model,
|
||||
args=training_args,
|
||||
train_dataset=train_dataset if training_args.do_train else None,
|
||||
eval_dataset=validation_dataset if training_args.do_eval else None,
|
||||
eval_examples=datasets["validation"] if training_args.do_eval else None,
|
||||
tokenizer=tokenizer,
|
||||
data_collator=data_collator,
|
||||
post_process_function=post_processing_function,
|
||||
compute_metrics=compute_metrics,
|
||||
)
|
||||
|
||||
# Training
|
||||
if training_args.do_train:
|
||||
trainer.train(
|
||||
model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
|
||||
)
|
||||
trainer.save_model() # Saves the tokenizer too for easy upload
|
||||
|
||||
# Evaluation
|
||||
results = {}
|
||||
if training_args.do_eval:
|
||||
logger.info("*** Evaluate ***")
|
||||
results = trainer.evaluate()
|
||||
|
||||
output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")
|
||||
if trainer.is_world_process_zero():
|
||||
with open(output_eval_file, "w") as writer:
|
||||
logger.info("***** Eval results *****")
|
||||
for key, value in results.items():
|
||||
logger.info(f" {key} = {value}")
|
||||
writer.write(f"{key} = {value}\n")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _mp_fn(index):
|
||||
# For xla_spawn (TPUs)
|
||||
main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,512 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Team All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Fine-tuning XLNet for question answering with beam search.
|
||||
"""
|
||||
# You can also adapt this script on your own question answering task. Pointers for this are left as comments.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
from datasets import load_dataset, load_metric
|
||||
|
||||
import transformers
|
||||
from trainer_qa import QuestionAnsweringTrainer
|
||||
from transformers import (
|
||||
DataCollatorWithPadding,
|
||||
EvalPrediction,
|
||||
HfArgumentParser,
|
||||
TrainingArguments,
|
||||
XLNetConfig,
|
||||
XLNetForQuestionAnswering,
|
||||
XLNetTokenizerFast,
|
||||
default_data_collator,
|
||||
set_seed,
|
||||
)
|
||||
from transformers.trainer_utils import is_main_process
|
||||
from utils_qa import postprocess_qa_predictions_with_beam_search
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelArguments:
|
||||
"""
|
||||
Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
|
||||
"""
|
||||
|
||||
model_name_or_path: str = field(
|
||||
metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
|
||||
)
|
||||
config_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
|
||||
)
|
||||
tokenizer_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
|
||||
)
|
||||
cache_dir: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
|
||||
)
|
||||
use_fast_tokenizer: bool = field(
|
||||
default=True,
|
||||
metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataTrainingArguments:
|
||||
"""
|
||||
Arguments pertaining to what data we are going to input our model for training and eval.
|
||||
"""
|
||||
|
||||
dataset_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
|
||||
)
|
||||
dataset_config_name: Optional[str] = field(
|
||||
default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
|
||||
)
|
||||
train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
|
||||
validation_file: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
|
||||
)
|
||||
overwrite_cache: bool = field(
|
||||
default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
|
||||
)
|
||||
preprocessing_num_workers: Optional[int] = field(
|
||||
default=None,
|
||||
metadata={"help": "The number of processes to use for the preprocessing."},
|
||||
)
|
||||
max_seq_length: int = field(
|
||||
default=384,
|
||||
metadata={
|
||||
"help": "The maximum total input sequence length after tokenization. Sequences longer "
|
||||
"than this will be truncated, sequences shorter will be padded."
|
||||
},
|
||||
)
|
||||
pad_to_max_length: bool = field(
|
||||
default=True,
|
||||
metadata={
|
||||
"help": "Whether to pad all samples to `max_seq_length`. "
|
||||
"If False, will pad the samples dynamically when batching to the maximum length in the batch (which can "
|
||||
"be faster on GPU but will be slower on TPU)."
|
||||
},
|
||||
)
|
||||
version_2_with_negative: bool = field(
|
||||
default=False, metadata={"help": "If true, some of the examples do not have an answer."}
|
||||
)
|
||||
null_score_diff_threshold: float = field(
|
||||
default=0.0,
|
||||
metadata={
|
||||
"help": "The threshold used to select the null answer: if the best answer has a score that is less than "
|
||||
"the score of the null answer minus this threshold, the null answer is selected for this example. "
|
||||
"Only useful when `version_2_with_negative=True`."
|
||||
},
|
||||
)
|
||||
doc_stride: int = field(
|
||||
default=128,
|
||||
metadata={"help": "When splitting up a long document into chunks, how much stride to take between chunks."},
|
||||
)
|
||||
n_best_size: int = field(
|
||||
default=20,
|
||||
metadata={"help": "The total number of n-best predictions to generate when looking for an answer."},
|
||||
)
|
||||
max_answer_length: int = field(
|
||||
default=30,
|
||||
metadata={
|
||||
"help": "The maximum length of an answer that can be generated. This is needed because the start "
|
||||
"and end predictions are not conditioned on one another."
|
||||
},
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
if self.dataset_name is None and self.train_file is None and self.validation_file is None:
|
||||
raise ValueError("Need either a dataset name or a training/validation file.")
|
||||
else:
|
||||
if self.train_file is not None:
|
||||
extension = self.train_file.split(".")[-1]
|
||||
assert extension in ["csv", "json"], "`train_file` should be a csv or a json file."
|
||||
if self.validation_file is not None:
|
||||
extension = self.validation_file.split(".")[-1]
|
||||
assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file."
|
||||
|
||||
|
||||
def main():
|
||||
# See all possible arguments in src/transformers/training_args.py
|
||||
# or by passing the --help flag to this script.
|
||||
# We now keep distinct sets of args, for a cleaner separation of concerns.
|
||||
|
||||
parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
|
||||
if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
|
||||
# If we pass only one argument to the script and it's the path to a json file,
|
||||
# let's parse it to get our arguments.
|
||||
model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
|
||||
else:
|
||||
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
|
||||
|
||||
if (
|
||||
os.path.exists(training_args.output_dir)
|
||||
and os.listdir(training_args.output_dir)
|
||||
and training_args.do_train
|
||||
and not training_args.overwrite_output_dir
|
||||
):
|
||||
raise ValueError(
|
||||
f"Output directory ({training_args.output_dir}) already exists and is not empty."
|
||||
"Use --overwrite_output_dir to overcome."
|
||||
)
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
|
||||
datefmt="%m/%d/%Y %H:%M:%S",
|
||||
)
|
||||
logger.setLevel(logging.INFO if is_main_process(training_args.local_rank) else logging.WARN)
|
||||
|
||||
# Log on each process the small summary:
|
||||
logger.warning(
|
||||
f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
|
||||
+ f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
|
||||
)
|
||||
# Set the verbosity to info of the Transformers logger (on main process only):
|
||||
if is_main_process(training_args.local_rank):
|
||||
transformers.utils.logging.set_verbosity_info()
|
||||
logger.info("Training/evaluation parameters %s", training_args)
|
||||
|
||||
# Set seed before initializing model.
|
||||
set_seed(training_args.seed)
|
||||
|
||||
# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
|
||||
# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
|
||||
# (the dataset will be downloaded automatically from the datasets Hub).
|
||||
#
|
||||
# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
|
||||
# 'text' is found. You can easily tweak this behavior (see below).
|
||||
#
|
||||
# In distributed training, the load_dataset function guarantee that only one local process can concurrently
|
||||
# download the dataset.
|
||||
if data_args.dataset_name is not None:
|
||||
# Downloading and loading a dataset from the hub.
|
||||
datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name)
|
||||
else:
|
||||
data_files = {}
|
||||
if data_args.train_file is not None:
|
||||
data_files["train"] = data_args.train_file
|
||||
if data_args.validation_file is not None:
|
||||
data_files["validation"] = data_args.validation_file
|
||||
extension = data_args.train_file.split(".")[-1]
|
||||
datasets = load_dataset(extension, data_files=data_files, field="data")
|
||||
# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
|
||||
# https://huggingface.co/docs/datasets/loading_datasets.html.
|
||||
|
||||
# Load pretrained model and tokenizer
|
||||
#
|
||||
# Distributed training:
|
||||
# The .from_pretrained methods guarantee that only one local process can concurrently
|
||||
# download model & vocab.
|
||||
config = XLNetConfig.from_pretrained(
|
||||
model_args.config_name if model_args.config_name else model_args.model_name_or_path,
|
||||
cache_dir=model_args.cache_dir,
|
||||
)
|
||||
tokenizer = XLNetTokenizerFast.from_pretrained(
|
||||
model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
|
||||
cache_dir=model_args.cache_dir,
|
||||
)
|
||||
model = XLNetForQuestionAnswering.from_pretrained(
|
||||
model_args.model_name_or_path,
|
||||
from_tf=bool(".ckpt" in model_args.model_name_or_path),
|
||||
config=config,
|
||||
cache_dir=model_args.cache_dir,
|
||||
)
|
||||
|
||||
# Preprocessing the datasets.
|
||||
# Preprocessing is slighlty different for training and evaluation.
|
||||
if training_args.do_train:
|
||||
column_names = datasets["train"].column_names
|
||||
else:
|
||||
column_names = datasets["validation"].column_names
|
||||
question_column_name = "question" if "question" in column_names else column_names[0]
|
||||
context_column_name = "context" if "context" in column_names else column_names[1]
|
||||
answer_column_name = "answers" if "answers" in column_names else column_names[2]
|
||||
|
||||
# Padding side determines if we do (question|context) or (context|question).
|
||||
pad_on_right = tokenizer.padding_side == "right"
|
||||
|
||||
# Training preprocessing
|
||||
def prepare_train_features(examples):
|
||||
# Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results
|
||||
# in one example possible giving several features when a context is long, each of those features having a
|
||||
# context that overlaps a bit the context of the previous feature.
|
||||
tokenized_examples = tokenizer(
|
||||
examples[question_column_name if pad_on_right else context_column_name],
|
||||
examples[context_column_name if pad_on_right else question_column_name],
|
||||
truncation="only_second" if pad_on_right else "only_first",
|
||||
max_length=data_args.max_seq_length,
|
||||
stride=data_args.doc_stride,
|
||||
return_overflowing_tokens=True,
|
||||
return_offsets_mapping=True,
|
||||
return_special_tokens_mask=True,
|
||||
return_token_type_ids=True,
|
||||
padding="max_length",
|
||||
)
|
||||
|
||||
# Since one example might give us several features if it has a long context, we need a map from a feature to
|
||||
# its corresponding example. This key gives us just that.
|
||||
sample_mapping = tokenized_examples.pop("overflow_to_sample_mapping")
|
||||
# The offset mappings will give us a map from token to character position in the original context. This will
|
||||
# help us compute the start_positions and end_positions.
|
||||
offset_mapping = tokenized_examples.pop("offset_mapping")
|
||||
# The special tokens will help us build the p_mask (which indicates the tokens that can't be in answers).
|
||||
special_tokens = tokenized_examples.pop("special_tokens_mask")
|
||||
|
||||
# Let's label those examples!
|
||||
tokenized_examples["start_positions"] = []
|
||||
tokenized_examples["end_positions"] = []
|
||||
tokenized_examples["is_impossible"] = []
|
||||
tokenized_examples["cls_index"] = []
|
||||
tokenized_examples["p_mask"] = []
|
||||
|
||||
for i, offsets in enumerate(offset_mapping):
|
||||
# We will label impossible answers with the index of the CLS token.
|
||||
input_ids = tokenized_examples["input_ids"][i]
|
||||
cls_index = input_ids.index(tokenizer.cls_token_id)
|
||||
tokenized_examples["cls_index"].append(cls_index)
|
||||
|
||||
# Grab the sequence corresponding to that example (to know what is the context and what is the question).
|
||||
sequence_ids = tokenized_examples["token_type_ids"][i]
|
||||
for k, s in enumerate(special_tokens[i]):
|
||||
if s:
|
||||
sequence_ids[k] = 3
|
||||
context_idx = 1 if pad_on_right else 0
|
||||
|
||||
# Build the p_mask: non special tokens and context gets 0.0, the others get 1.0.
|
||||
# The cls token gets 1.0 too (for predictions of empty answers).
|
||||
tokenized_examples["p_mask"].append(
|
||||
[
|
||||
0.0 if (not special_tokens[i][k] and s == context_idx) or k == cls_index else 1.0
|
||||
for k, s in enumerate(sequence_ids)
|
||||
]
|
||||
)
|
||||
|
||||
# One example can give several spans, this is the index of the example containing this span of text.
|
||||
sample_index = sample_mapping[i]
|
||||
answers = examples[answer_column_name][sample_index]
|
||||
# If no answers are given, set the cls_index as answer.
|
||||
if len(answers["answer_start"]) == 0:
|
||||
tokenized_examples["start_positions"].append(cls_index)
|
||||
tokenized_examples["end_positions"].append(cls_index)
|
||||
tokenized_examples["is_impossible"].append(1.0)
|
||||
else:
|
||||
# Start/end character index of the answer in the text.
|
||||
start_char = answers["answer_start"][0]
|
||||
end_char = start_char + len(answers["text"][0])
|
||||
|
||||
# Start token index of the current span in the text.
|
||||
token_start_index = 0
|
||||
while sequence_ids[token_start_index] != context_idx:
|
||||
token_start_index += 1
|
||||
|
||||
# End token index of the current span in the text.
|
||||
token_end_index = len(input_ids) - 1
|
||||
while sequence_ids[token_end_index] != context_idx:
|
||||
token_end_index -= 1
|
||||
# Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).
|
||||
if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):
|
||||
tokenized_examples["start_positions"].append(cls_index)
|
||||
tokenized_examples["end_positions"].append(cls_index)
|
||||
tokenized_examples["is_impossible"].append(1.0)
|
||||
else:
|
||||
# Otherwise move the token_start_index and token_end_index to the two ends of the answer.
|
||||
# Note: we could go after the last offset if the answer is the last word (edge case).
|
||||
while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:
|
||||
token_start_index += 1
|
||||
tokenized_examples["start_positions"].append(token_start_index - 1)
|
||||
while offsets[token_end_index][1] >= end_char:
|
||||
token_end_index -= 1
|
||||
tokenized_examples["end_positions"].append(token_end_index + 1)
|
||||
tokenized_examples["is_impossible"].append(0.0)
|
||||
|
||||
return tokenized_examples
|
||||
|
||||
if training_args.do_train:
|
||||
train_dataset = datasets["train"].map(
|
||||
prepare_train_features,
|
||||
batched=True,
|
||||
num_proc=data_args.preprocessing_num_workers,
|
||||
remove_columns=column_names,
|
||||
load_from_cache_file=not data_args.overwrite_cache,
|
||||
)
|
||||
|
||||
# Validation preprocessing
|
||||
def prepare_validation_features(examples):
|
||||
# Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results
|
||||
# in one example possible giving several features when a context is long, each of those features having a
|
||||
# context that overlaps a bit the context of the previous feature.
|
||||
tokenized_examples = tokenizer(
|
||||
examples[question_column_name if pad_on_right else context_column_name],
|
||||
examples[context_column_name if pad_on_right else question_column_name],
|
||||
truncation="only_second" if pad_on_right else "only_first",
|
||||
max_length=data_args.max_seq_length,
|
||||
stride=data_args.doc_stride,
|
||||
return_overflowing_tokens=True,
|
||||
return_offsets_mapping=True,
|
||||
return_special_tokens_mask=True,
|
||||
return_token_type_ids=True,
|
||||
padding="max_length",
|
||||
)
|
||||
|
||||
# Since one example might give us several features if it has a long context, we need a map from a feature to
|
||||
# its corresponding example. This key gives us just that.
|
||||
sample_mapping = tokenized_examples.pop("overflow_to_sample_mapping")
|
||||
|
||||
# The special tokens will help us build the p_mask (which indicates the tokens that can't be in answers).
|
||||
special_tokens = tokenized_examples.pop("special_tokens_mask")
|
||||
|
||||
# For evaluation, we will need to convert our predictions to substrings of the context, so we keep the
|
||||
# corresponding example_id and we will store the offset mappings.
|
||||
tokenized_examples["example_id"] = []
|
||||
|
||||
# We still provide the index of the CLS token and the p_mask to the model, but not the is_impossible label.
|
||||
tokenized_examples["cls_index"] = []
|
||||
tokenized_examples["p_mask"] = []
|
||||
|
||||
for i, input_ids in enumerate(tokenized_examples["input_ids"]):
|
||||
# Find the CLS token in the input ids.
|
||||
cls_index = input_ids.index(tokenizer.cls_token_id)
|
||||
tokenized_examples["cls_index"].append(cls_index)
|
||||
|
||||
# Grab the sequence corresponding to that example (to know what is the context and what is the question).
|
||||
sequence_ids = tokenized_examples["token_type_ids"][i]
|
||||
for k, s in enumerate(special_tokens[i]):
|
||||
if s:
|
||||
sequence_ids[k] = 3
|
||||
context_idx = 1 if pad_on_right else 0
|
||||
|
||||
# Build the p_mask: non special tokens and context gets 0.0, the others 1.0.
|
||||
tokenized_examples["p_mask"].append(
|
||||
[
|
||||
0.0 if (not special_tokens[i][k] and s == context_idx) or k == cls_index else 1.0
|
||||
for k, s in enumerate(sequence_ids)
|
||||
]
|
||||
)
|
||||
|
||||
# One example can give several spans, this is the index of the example containing this span of text.
|
||||
sample_index = sample_mapping[i]
|
||||
tokenized_examples["example_id"].append(examples["id"][sample_index])
|
||||
|
||||
# Set to None the offset_mapping that are not part of the context so it's easy to determine if a token
|
||||
# position is part of the context or not.
|
||||
tokenized_examples["offset_mapping"][i] = [
|
||||
(o if sequence_ids[k] == context_idx else None)
|
||||
for k, o in enumerate(tokenized_examples["offset_mapping"][i])
|
||||
]
|
||||
|
||||
return tokenized_examples
|
||||
|
||||
if training_args.do_eval:
|
||||
validation_dataset = datasets["validation"].map(
|
||||
prepare_validation_features,
|
||||
batched=True,
|
||||
num_proc=data_args.preprocessing_num_workers,
|
||||
remove_columns=column_names,
|
||||
load_from_cache_file=not data_args.overwrite_cache,
|
||||
)
|
||||
|
||||
# Data collator
|
||||
# We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data
|
||||
# collator.
|
||||
data_collator = default_data_collator if data_args.pad_to_max_length else DataCollatorWithPadding(tokenizer)
|
||||
|
||||
# Post-processing:
|
||||
def post_processing_function(examples, features, predictions):
|
||||
# Post-processing: we match the start logits and end logits to answers in the original context.
|
||||
predictions, scores_diff_json = postprocess_qa_predictions_with_beam_search(
|
||||
examples=examples,
|
||||
features=features,
|
||||
predictions=predictions,
|
||||
version_2_with_negative=data_args.version_2_with_negative,
|
||||
n_best_size=data_args.n_best_size,
|
||||
max_answer_length=data_args.max_answer_length,
|
||||
start_n_top=model.config.start_n_top,
|
||||
end_n_top=model.config.end_n_top,
|
||||
output_dir=training_args.output_dir,
|
||||
is_world_process_zero=trainer.is_world_process_zero(),
|
||||
)
|
||||
# Format the result to the format the metric expects.
|
||||
if data_args.version_2_with_negative:
|
||||
formatted_predictions = [
|
||||
{"id": k, "prediction_text": v, "no_answer_probability": scores_diff_json[k]}
|
||||
for k, v in predictions.items()
|
||||
]
|
||||
else:
|
||||
formatted_predictions = [{"id": k, "prediction_text": v} for k, v in predictions.items()]
|
||||
references = [{"id": ex["id"], "answers": ex[answer_column_name]} for ex in datasets["validation"]]
|
||||
return EvalPrediction(predictions=formatted_predictions, label_ids=references)
|
||||
|
||||
# TODO: Once the fix lands in a Datasets release, remove the _local here and the squad_v2_local folder.
|
||||
current_dir = os.path.sep.join(os.path.join(__file__).split(os.path.sep)[:-1])
|
||||
metric = load_metric(os.path.join(current_dir, "squad_v2_local") if data_args.version_2_with_negative else "squad")
|
||||
|
||||
def compute_metrics(p: EvalPrediction):
|
||||
return metric.compute(predictions=p.predictions, references=p.label_ids)
|
||||
|
||||
# Initialize our Trainer
|
||||
trainer = QuestionAnsweringTrainer(
|
||||
model=model,
|
||||
args=training_args,
|
||||
train_dataset=train_dataset if training_args.do_train else None,
|
||||
eval_dataset=validation_dataset if training_args.do_eval else None,
|
||||
eval_examples=datasets["validation"] if training_args.do_eval else None,
|
||||
tokenizer=tokenizer,
|
||||
data_collator=data_collator,
|
||||
post_process_function=post_processing_function,
|
||||
compute_metrics=compute_metrics,
|
||||
)
|
||||
|
||||
# Training
|
||||
if training_args.do_train:
|
||||
trainer.train(
|
||||
model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None
|
||||
)
|
||||
trainer.save_model() # Saves the tokenizer too for easy upload
|
||||
|
||||
# Evaluation
|
||||
results = {}
|
||||
if training_args.do_eval:
|
||||
logger.info("*** Evaluate ***")
|
||||
results = trainer.evaluate()
|
||||
|
||||
output_eval_file = os.path.join(training_args.output_dir, "eval_results.txt")
|
||||
if trainer.is_world_process_zero():
|
||||
with open(output_eval_file, "w") as writer:
|
||||
logger.info("***** Eval results *****")
|
||||
for key, value in results.items():
|
||||
logger.info(f" {key} = {value}")
|
||||
writer.write(f"{key} = {value}\n")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _mp_fn(index):
|
||||
# For xla_spawn (TPUs)
|
||||
main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,322 +0,0 @@
|
||||
"""Official evaluation script for SQuAD version 2.0.
|
||||
|
||||
In addition to basic functionality, we also compute additional statistics and
|
||||
plot precision-recall curves if an additional na_prob.json file is provided.
|
||||
This file is expected to map question ID's to the model's predicted probability
|
||||
that a question is unanswerable.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import string
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
OPTS = None
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0.")
|
||||
parser.add_argument("data_file", metavar="data.json", help="Input data JSON file.")
|
||||
parser.add_argument("pred_file", metavar="pred.json", help="Model predictions.")
|
||||
parser.add_argument(
|
||||
"--out-file", "-o", metavar="eval.json", help="Write accuracy metrics to file (default is stdout)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--na-prob-file", "-n", metavar="na_prob.json", help="Model estimates of probability of no answer."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--na-prob-thresh",
|
||||
"-t",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help='Predict "" if no-answer probability exceeds this (default = 1.0).',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out-image-dir", "-p", metavar="out_images", default=None, help="Save precision-recall curves to directory."
|
||||
)
|
||||
parser.add_argument("--verbose", "-v", action="store_true")
|
||||
if len(sys.argv) == 1:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def make_qid_to_has_ans(dataset):
|
||||
qid_to_has_ans = {}
|
||||
for article in dataset:
|
||||
for p in article["paragraphs"]:
|
||||
for qa in p["qas"]:
|
||||
qid_to_has_ans[qa["id"]] = bool(qa["answers"]["text"])
|
||||
return qid_to_has_ans
|
||||
|
||||
|
||||
def normalize_answer(s):
|
||||
"""Lower text and remove punctuation, articles and extra whitespace."""
|
||||
|
||||
def remove_articles(text):
|
||||
regex = re.compile(r"\b(a|an|the)\b", re.UNICODE)
|
||||
return re.sub(regex, " ", text)
|
||||
|
||||
def white_space_fix(text):
|
||||
return " ".join(text.split())
|
||||
|
||||
def remove_punc(text):
|
||||
exclude = set(string.punctuation)
|
||||
return "".join(ch for ch in text if ch not in exclude)
|
||||
|
||||
def lower(text):
|
||||
return text.lower()
|
||||
|
||||
return white_space_fix(remove_articles(remove_punc(lower(s))))
|
||||
|
||||
|
||||
def get_tokens(s):
|
||||
if not s:
|
||||
return []
|
||||
return normalize_answer(s).split()
|
||||
|
||||
|
||||
def compute_exact(a_gold, a_pred):
|
||||
return int(normalize_answer(a_gold) == normalize_answer(a_pred))
|
||||
|
||||
|
||||
def compute_f1(a_gold, a_pred):
|
||||
gold_toks = get_tokens(a_gold)
|
||||
pred_toks = get_tokens(a_pred)
|
||||
common = collections.Counter(gold_toks) & collections.Counter(pred_toks)
|
||||
num_same = sum(common.values())
|
||||
if len(gold_toks) == 0 or len(pred_toks) == 0:
|
||||
# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
|
||||
return int(gold_toks == pred_toks)
|
||||
if num_same == 0:
|
||||
return 0
|
||||
precision = 1.0 * num_same / len(pred_toks)
|
||||
recall = 1.0 * num_same / len(gold_toks)
|
||||
f1 = (2 * precision * recall) / (precision + recall)
|
||||
return f1
|
||||
|
||||
|
||||
def get_raw_scores(dataset, preds):
|
||||
exact_scores = {}
|
||||
f1_scores = {}
|
||||
for article in dataset:
|
||||
for p in article["paragraphs"]:
|
||||
for qa in p["qas"]:
|
||||
qid = qa["id"]
|
||||
gold_answers = [t for t in qa["answers"]["text"] if normalize_answer(t)]
|
||||
if not gold_answers:
|
||||
# For unanswerable questions, only correct answer is empty string
|
||||
gold_answers = [""]
|
||||
if qid not in preds:
|
||||
print("Missing prediction for %s" % qid)
|
||||
continue
|
||||
a_pred = preds[qid]
|
||||
# Take max over all gold answers
|
||||
exact_scores[qid] = max(compute_exact(a, a_pred) for a in gold_answers)
|
||||
f1_scores[qid] = max(compute_f1(a, a_pred) for a in gold_answers)
|
||||
return exact_scores, f1_scores
|
||||
|
||||
|
||||
def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh):
|
||||
new_scores = {}
|
||||
for qid, s in scores.items():
|
||||
pred_na = na_probs[qid] > na_prob_thresh
|
||||
if pred_na:
|
||||
new_scores[qid] = float(not qid_to_has_ans[qid])
|
||||
else:
|
||||
new_scores[qid] = s
|
||||
return new_scores
|
||||
|
||||
|
||||
def make_eval_dict(exact_scores, f1_scores, qid_list=None):
|
||||
if not qid_list:
|
||||
total = len(exact_scores)
|
||||
return collections.OrderedDict(
|
||||
[
|
||||
("exact", 100.0 * sum(exact_scores.values()) / total),
|
||||
("f1", 100.0 * sum(f1_scores.values()) / total),
|
||||
("total", total),
|
||||
]
|
||||
)
|
||||
else:
|
||||
total = len(qid_list)
|
||||
return collections.OrderedDict(
|
||||
[
|
||||
("exact", 100.0 * sum(exact_scores[k] for k in qid_list) / total),
|
||||
("f1", 100.0 * sum(f1_scores[k] for k in qid_list) / total),
|
||||
("total", total),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def merge_eval(main_eval, new_eval, prefix):
|
||||
for k in new_eval:
|
||||
main_eval["%s_%s" % (prefix, k)] = new_eval[k]
|
||||
|
||||
|
||||
def plot_pr_curve(precisions, recalls, out_image, title):
|
||||
plt.step(recalls, precisions, color="b", alpha=0.2, where="post")
|
||||
plt.fill_between(recalls, precisions, step="post", alpha=0.2, color="b")
|
||||
plt.xlabel("Recall")
|
||||
plt.ylabel("Precision")
|
||||
plt.xlim([0.0, 1.05])
|
||||
plt.ylim([0.0, 1.05])
|
||||
plt.title(title)
|
||||
plt.savefig(out_image)
|
||||
plt.clf()
|
||||
|
||||
|
||||
def make_precision_recall_eval(scores, na_probs, num_true_pos, qid_to_has_ans, out_image=None, title=None):
|
||||
qid_list = sorted(na_probs, key=lambda k: na_probs[k])
|
||||
true_pos = 0.0
|
||||
cur_p = 1.0
|
||||
cur_r = 0.0
|
||||
precisions = [1.0]
|
||||
recalls = [0.0]
|
||||
avg_prec = 0.0
|
||||
for i, qid in enumerate(qid_list):
|
||||
if qid_to_has_ans[qid]:
|
||||
true_pos += scores[qid]
|
||||
cur_p = true_pos / float(i + 1)
|
||||
cur_r = true_pos / float(num_true_pos)
|
||||
if i == len(qid_list) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
|
||||
# i.e., if we can put a threshold after this point
|
||||
avg_prec += cur_p * (cur_r - recalls[-1])
|
||||
precisions.append(cur_p)
|
||||
recalls.append(cur_r)
|
||||
if out_image:
|
||||
plot_pr_curve(precisions, recalls, out_image, title)
|
||||
return {"ap": 100.0 * avg_prec}
|
||||
|
||||
|
||||
def run_precision_recall_analysis(main_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, out_image_dir):
|
||||
if out_image_dir and not os.path.exists(out_image_dir):
|
||||
os.makedirs(out_image_dir)
|
||||
num_true_pos = sum(1 for v in qid_to_has_ans.values() if v)
|
||||
if num_true_pos == 0:
|
||||
return
|
||||
pr_exact = make_precision_recall_eval(
|
||||
exact_raw,
|
||||
na_probs,
|
||||
num_true_pos,
|
||||
qid_to_has_ans,
|
||||
out_image=os.path.join(out_image_dir, "pr_exact.png"),
|
||||
title="Precision-Recall curve for Exact Match score",
|
||||
)
|
||||
pr_f1 = make_precision_recall_eval(
|
||||
f1_raw,
|
||||
na_probs,
|
||||
num_true_pos,
|
||||
qid_to_has_ans,
|
||||
out_image=os.path.join(out_image_dir, "pr_f1.png"),
|
||||
title="Precision-Recall curve for F1 score",
|
||||
)
|
||||
oracle_scores = {k: float(v) for k, v in qid_to_has_ans.items()}
|
||||
pr_oracle = make_precision_recall_eval(
|
||||
oracle_scores,
|
||||
na_probs,
|
||||
num_true_pos,
|
||||
qid_to_has_ans,
|
||||
out_image=os.path.join(out_image_dir, "pr_oracle.png"),
|
||||
title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)",
|
||||
)
|
||||
merge_eval(main_eval, pr_exact, "pr_exact")
|
||||
merge_eval(main_eval, pr_f1, "pr_f1")
|
||||
merge_eval(main_eval, pr_oracle, "pr_oracle")
|
||||
|
||||
|
||||
def histogram_na_prob(na_probs, qid_list, image_dir, name):
|
||||
if not qid_list:
|
||||
return
|
||||
x = [na_probs[k] for k in qid_list]
|
||||
weights = np.ones_like(x) / float(len(x))
|
||||
plt.hist(x, weights=weights, bins=20, range=(0.0, 1.0))
|
||||
plt.xlabel("Model probability of no-answer")
|
||||
plt.ylabel("Proportion of dataset")
|
||||
plt.title("Histogram of no-answer probability: %s" % name)
|
||||
plt.savefig(os.path.join(image_dir, "na_prob_hist_%s.png" % name))
|
||||
plt.clf()
|
||||
|
||||
|
||||
def find_best_thresh(preds, scores, na_probs, qid_to_has_ans):
|
||||
num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k])
|
||||
cur_score = num_no_ans
|
||||
best_score = cur_score
|
||||
best_thresh = 0.0
|
||||
qid_list = sorted(na_probs, key=lambda k: na_probs[k])
|
||||
for i, qid in enumerate(qid_list):
|
||||
if qid not in scores:
|
||||
continue
|
||||
if qid_to_has_ans[qid]:
|
||||
diff = scores[qid]
|
||||
else:
|
||||
if preds[qid]:
|
||||
diff = -1
|
||||
else:
|
||||
diff = 0
|
||||
cur_score += diff
|
||||
if cur_score > best_score:
|
||||
best_score = cur_score
|
||||
best_thresh = na_probs[qid]
|
||||
return 100.0 * best_score / len(scores), best_thresh
|
||||
|
||||
|
||||
def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans):
|
||||
best_exact, exact_thresh = find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans)
|
||||
best_f1, f1_thresh = find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans)
|
||||
main_eval["best_exact"] = best_exact
|
||||
main_eval["best_exact_thresh"] = exact_thresh
|
||||
main_eval["best_f1"] = best_f1
|
||||
main_eval["best_f1_thresh"] = f1_thresh
|
||||
|
||||
|
||||
def main():
|
||||
with open(OPTS.data_file) as f:
|
||||
dataset_json = json.load(f)
|
||||
dataset = dataset_json["data"]
|
||||
with open(OPTS.pred_file) as f:
|
||||
preds = json.load(f)
|
||||
if OPTS.na_prob_file:
|
||||
with open(OPTS.na_prob_file) as f:
|
||||
na_probs = json.load(f)
|
||||
else:
|
||||
na_probs = {k: 0.0 for k in preds}
|
||||
qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False
|
||||
has_ans_qids = [k for k, v in qid_to_has_ans.items() if v]
|
||||
no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v]
|
||||
exact_raw, f1_raw = get_raw_scores(dataset, preds)
|
||||
exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh)
|
||||
f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh)
|
||||
out_eval = make_eval_dict(exact_thresh, f1_thresh)
|
||||
if has_ans_qids:
|
||||
has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids)
|
||||
merge_eval(out_eval, has_ans_eval, "HasAns")
|
||||
if no_ans_qids:
|
||||
no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids)
|
||||
merge_eval(out_eval, no_ans_eval, "NoAns")
|
||||
if OPTS.na_prob_file:
|
||||
find_all_best_thresh(out_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans)
|
||||
if OPTS.na_prob_file and OPTS.out_image_dir:
|
||||
run_precision_recall_analysis(out_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, OPTS.out_image_dir)
|
||||
histogram_na_prob(na_probs, has_ans_qids, OPTS.out_image_dir, "hasAns")
|
||||
histogram_na_prob(na_probs, no_ans_qids, OPTS.out_image_dir, "noAns")
|
||||
if OPTS.out_file:
|
||||
with open(OPTS.out_file, "w") as f:
|
||||
json.dump(out_eval, f)
|
||||
else:
|
||||
print(json.dumps(out_eval, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
OPTS = parse_args()
|
||||
if OPTS.out_image_dir:
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
main()
|
||||
@@ -1,128 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Datasets Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" SQuAD v2 metric. """
|
||||
|
||||
import datasets
|
||||
|
||||
from .evaluate import (
|
||||
apply_no_ans_threshold,
|
||||
find_all_best_thresh,
|
||||
get_raw_scores,
|
||||
make_eval_dict,
|
||||
make_qid_to_has_ans,
|
||||
merge_eval,
|
||||
)
|
||||
|
||||
|
||||
_CITATION = """\
|
||||
@inproceedings{Rajpurkar2016SQuAD10,
|
||||
title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},
|
||||
author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},
|
||||
booktitle={EMNLP},
|
||||
year={2016}
|
||||
}
|
||||
"""
|
||||
|
||||
_DESCRIPTION = """
|
||||
This metric wrap the official scoring script for version 2 of the Stanford Question
|
||||
Answering Dataset (SQuAD).
|
||||
|
||||
Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by
|
||||
crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,
|
||||
from the corresponding reading passage, or the question might be unanswerable.
|
||||
|
||||
SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions
|
||||
written adversarially by crowdworkers to look similar to answerable ones.
|
||||
To do well on SQuAD2.0, systems must not only answer questions when possible, but also
|
||||
determine when no answer is supported by the paragraph and abstain from answering.
|
||||
"""
|
||||
|
||||
_KWARGS_DESCRIPTION = """
|
||||
Computes SQuAD v2 scores (F1 and EM).
|
||||
Args:
|
||||
predictions: List of triple for question-answers to score with the following elements:
|
||||
- the question-answer 'id' field as given in the references (see below)
|
||||
- the text of the answer
|
||||
- the probability that the question has no answer
|
||||
references: List of question-answers dictionaries with the following key-values:
|
||||
- 'id': id of the question-answer pair (see above),
|
||||
- 'answers': a list of Dict {'text': text of the answer as a string}
|
||||
no_answer_threshold: float
|
||||
Probability threshold to decide that a question has no answer.
|
||||
Returns:
|
||||
'exact': Exact match (the normalized answer exactly match the gold answer)
|
||||
'f1': The F-score of predicted tokens versus the gold answer
|
||||
'total': Number of score considered
|
||||
'HasAns_exact': Exact match (the normalized answer exactly match the gold answer)
|
||||
'HasAns_f1': The F-score of predicted tokens versus the gold answer
|
||||
'HasAns_total': Number of score considered
|
||||
'NoAns_exact': Exact match (the normalized answer exactly match the gold answer)
|
||||
'NoAns_f1': The F-score of predicted tokens versus the gold answer
|
||||
'NoAns_total': Number of score considered
|
||||
'best_exact': Best exact match (with varying threshold)
|
||||
'best_exact_thresh': No-answer probability threshold associated to the best exact match
|
||||
'best_f1': Best F1 (with varying threshold)
|
||||
'best_f1_thresh': No-answer probability threshold associated to the best F1
|
||||
"""
|
||||
|
||||
|
||||
class SquadV2(datasets.Metric):
|
||||
def _info(self):
|
||||
return datasets.MetricInfo(
|
||||
description=_DESCRIPTION,
|
||||
citation=_CITATION,
|
||||
inputs_description=_KWARGS_DESCRIPTION,
|
||||
features=datasets.Features(
|
||||
{
|
||||
"predictions": {
|
||||
"id": datasets.Value("string"),
|
||||
"prediction_text": datasets.Value("string"),
|
||||
"no_answer_probability": datasets.Value("float32"),
|
||||
},
|
||||
"references": {
|
||||
"id": datasets.Value("string"),
|
||||
"answers": datasets.features.Sequence(
|
||||
{"text": datasets.Value("string"), "answer_start": datasets.Value("int32")}
|
||||
),
|
||||
},
|
||||
}
|
||||
),
|
||||
codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"],
|
||||
reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"],
|
||||
)
|
||||
|
||||
def _compute(self, predictions, references, no_answer_threshold=1.0):
|
||||
no_answer_probabilities = dict((p["id"], p["no_answer_probability"]) for p in predictions)
|
||||
dataset = [{"paragraphs": [{"qas": references}]}]
|
||||
predictions = dict((p["id"], p["prediction_text"]) for p in predictions)
|
||||
|
||||
qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False
|
||||
has_ans_qids = [k for k, v in qid_to_has_ans.items() if v]
|
||||
no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v]
|
||||
|
||||
exact_raw, f1_raw = get_raw_scores(dataset, predictions)
|
||||
exact_thresh = apply_no_ans_threshold(exact_raw, no_answer_probabilities, qid_to_has_ans, no_answer_threshold)
|
||||
f1_thresh = apply_no_ans_threshold(f1_raw, no_answer_probabilities, qid_to_has_ans, no_answer_threshold)
|
||||
out_eval = make_eval_dict(exact_thresh, f1_thresh)
|
||||
|
||||
if has_ans_qids:
|
||||
has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids)
|
||||
merge_eval(out_eval, has_ans_eval, "HasAns")
|
||||
if no_ans_qids:
|
||||
no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids)
|
||||
merge_eval(out_eval, no_ans_eval, "NoAns")
|
||||
find_all_best_thresh(out_eval, predictions, exact_raw, f1_raw, no_answer_probabilities, qid_to_has_ans)
|
||||
|
||||
return out_eval
|
||||
@@ -1,104 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Team All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
A subclass of `Trainer` specific to Question-Answering tasks
|
||||
"""
|
||||
|
||||
from transformers import Trainer, is_datasets_available, is_torch_tpu_available
|
||||
from transformers.trainer_utils import PredictionOutput
|
||||
|
||||
|
||||
if is_datasets_available():
|
||||
import datasets
|
||||
|
||||
if is_torch_tpu_available():
|
||||
import torch_xla.core.xla_model as xm
|
||||
import torch_xla.debug.metrics as met
|
||||
|
||||
|
||||
class QuestionAnsweringTrainer(Trainer):
|
||||
def __init__(self, *args, eval_examples=None, post_process_function=None, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.eval_examples = eval_examples
|
||||
self.post_process_function = post_process_function
|
||||
|
||||
def evaluate(self, eval_dataset=None, eval_examples=None, ignore_keys=None):
|
||||
eval_dataset = self.eval_dataset if eval_dataset is None else eval_dataset
|
||||
eval_dataloader = self.get_eval_dataloader(eval_dataset)
|
||||
eval_examples = self.eval_examples if eval_examples is None else eval_examples
|
||||
|
||||
# Temporarily disable metric computation, we will do it in the loop here.
|
||||
compute_metrics = self.compute_metrics
|
||||
self.compute_metrics = None
|
||||
try:
|
||||
output = self.prediction_loop(
|
||||
eval_dataloader,
|
||||
description="Evaluation",
|
||||
# No point gathering the predictions if there are no metrics, otherwise we defer to
|
||||
# self.args.prediction_loss_only
|
||||
prediction_loss_only=True if compute_metrics is None else None,
|
||||
ignore_keys=ignore_keys,
|
||||
)
|
||||
finally:
|
||||
self.compute_metrics = compute_metrics
|
||||
|
||||
# We might have removed columns from the dataset so we put them back.
|
||||
if isinstance(eval_dataset, datasets.Dataset):
|
||||
eval_dataset.set_format(type=eval_dataset.format["type"], columns=list(eval_dataset.features.keys()))
|
||||
|
||||
if self.post_process_function is not None and self.compute_metrics is not None:
|
||||
eval_preds = self.post_process_function(eval_examples, eval_dataset, output.predictions)
|
||||
metrics = self.compute_metrics(eval_preds)
|
||||
|
||||
self.log(metrics)
|
||||
else:
|
||||
metrics = {}
|
||||
|
||||
if self.args.tpu_metrics_debug or self.args.debug:
|
||||
# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
|
||||
xm.master_print(met.metrics_report())
|
||||
|
||||
self.control = self.callback_handler.on_evaluate(self.args, self.state, self.control, metrics)
|
||||
return metrics
|
||||
|
||||
def predict(self, test_dataset, test_examples, ignore_keys=None):
|
||||
test_dataloader = self.get_test_dataloader(test_dataset)
|
||||
|
||||
# Temporarily disable metric computation, we will do it in the loop here.
|
||||
compute_metrics = self.compute_metrics
|
||||
self.compute_metrics = None
|
||||
try:
|
||||
output = self.prediction_loop(
|
||||
test_dataloader,
|
||||
description="Evaluation",
|
||||
# No point gathering the predictions if there are no metrics, otherwise we defer to
|
||||
# self.args.prediction_loss_only
|
||||
prediction_loss_only=True if compute_metrics is None else None,
|
||||
ignore_keys=ignore_keys,
|
||||
)
|
||||
finally:
|
||||
self.compute_metrics = compute_metrics
|
||||
|
||||
if self.post_process_function is None or self.compute_metrics is None:
|
||||
return output
|
||||
|
||||
# We might have removed columns from the dataset so we put them back.
|
||||
if isinstance(test_dataset, datasets.Dataset):
|
||||
test_dataset.set_format(type=test_dataset.format["type"], columns=list(test_dataset.features.keys()))
|
||||
|
||||
eval_preds = self.post_process_function(test_examples, test_dataset, output.predictions)
|
||||
metrics = self.compute_metrics(eval_preds)
|
||||
|
||||
return PredictionOutput(predictions=eval_preds.predictions, label_ids=eval_preds.label_ids, metrics=metrics)
|
||||
@@ -1,429 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Team All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Post-processing utilities for question answering.
|
||||
"""
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def postprocess_qa_predictions(
|
||||
examples,
|
||||
features,
|
||||
predictions: Tuple[np.ndarray, np.ndarray],
|
||||
version_2_with_negative: bool = False,
|
||||
n_best_size: int = 20,
|
||||
max_answer_length: int = 30,
|
||||
null_score_diff_threshold: float = 0.0,
|
||||
output_dir: Optional[str] = None,
|
||||
prefix: Optional[str] = None,
|
||||
is_world_process_zero: bool = True,
|
||||
):
|
||||
"""
|
||||
Post-processes the predictions of a question-answering model to convert them to answers that are substrings of the
|
||||
original contexts. This is the base postprocessing functions for models that only return start and end logits.
|
||||
|
||||
Args:
|
||||
examples: The non-preprocessed dataset (see the main script for more information).
|
||||
features: The processed dataset (see the main script for more information).
|
||||
predictions (:obj:`Tuple[np.ndarray, np.ndarray]`):
|
||||
The predictions of the model: two arrays containing the start logits and the end logits respectively. Its
|
||||
first dimension must match the number of elements of :obj:`features`.
|
||||
version_2_with_negative (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
||||
Whether or not the underlying dataset contains examples with no answers.
|
||||
n_best_size (:obj:`int`, `optional`, defaults to 20):
|
||||
The total number of n-best predictions to generate when looking for an answer.
|
||||
max_answer_length (:obj:`int`, `optional`, defaults to 30):
|
||||
The maximum length of an answer that can be generated. This is needed because the start and end predictions
|
||||
are not conditioned on one another.
|
||||
null_score_diff_threshold (:obj:`float`, `optional`, defaults to 0):
|
||||
The threshold used to select the null answer: if the best answer has a score that is less than the score of
|
||||
the null answer minus this threshold, the null answer is selected for this example (note that the score of
|
||||
the null answer for an example giving several features is the minimum of the scores for the null answer on
|
||||
each feature: all features must be aligned on the fact they `want` to predict a null answer).
|
||||
|
||||
Only useful when :obj:`version_2_with_negative` is :obj:`True`.
|
||||
output_dir (:obj:`str`, `optional`):
|
||||
If provided, the dictionaries of predictions, n_best predictions (with their scores and logits) and, if
|
||||
:obj:`version_2_with_negative=True`, the dictionary of the scores differences between best and null
|
||||
answers, are saved in `output_dir`.
|
||||
prefix (:obj:`str`, `optional`):
|
||||
If provided, the dictionaries mentioned above are saved with `prefix` added to their names.
|
||||
is_world_process_zero (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether this process is the main process or not (used to determine if logging/saves should be done).
|
||||
"""
|
||||
assert len(predictions) == 2, "`predictions` should be a tuple with two elements (start_logits, end_logits)."
|
||||
all_start_logits, all_end_logits = predictions
|
||||
|
||||
assert len(predictions[0]) == len(
|
||||
features
|
||||
), f"Got {len(predictions[0])} predicitions and {len(features)} features."
|
||||
|
||||
# Build a map example to its corresponding features.
|
||||
example_id_to_index = {k: i for i, k in enumerate(examples["id"])}
|
||||
features_per_example = collections.defaultdict(list)
|
||||
for i, feature in enumerate(features):
|
||||
features_per_example[example_id_to_index[feature["example_id"]]].append(i)
|
||||
|
||||
# The dictionaries we have to fill.
|
||||
all_predictions = collections.OrderedDict()
|
||||
all_nbest_json = collections.OrderedDict()
|
||||
if version_2_with_negative:
|
||||
scores_diff_json = collections.OrderedDict()
|
||||
|
||||
# Logging.
|
||||
logger.setLevel(logging.INFO if is_world_process_zero else logging.WARN)
|
||||
logger.info(f"Post-processing {len(examples)} example predictions split into {len(features)} features.")
|
||||
|
||||
# Let's loop over all the examples!
|
||||
for example_index, example in enumerate(tqdm(examples)):
|
||||
# Those are the indices of the features associated to the current example.
|
||||
feature_indices = features_per_example[example_index]
|
||||
|
||||
min_null_prediction = None
|
||||
prelim_predictions = []
|
||||
|
||||
# Looping through all the features associated to the current example.
|
||||
for feature_index in feature_indices:
|
||||
# We grab the predictions of the model for this feature.
|
||||
start_logits = all_start_logits[feature_index]
|
||||
end_logits = all_end_logits[feature_index]
|
||||
# This is what will allow us to map some the positions in our logits to span of texts in the original
|
||||
# context.
|
||||
offset_mapping = features[feature_index]["offset_mapping"]
|
||||
# Optional `token_is_max_context`, if provided we will remove answers that do not have the maximum context
|
||||
# available in the current feature.
|
||||
token_is_max_context = features[feature_index].get("token_is_max_context", None)
|
||||
|
||||
# Update minimum null prediction.
|
||||
feature_null_score = start_logits[0] + end_logits[0]
|
||||
if min_null_prediction is None or min_null_prediction["score"] < feature_null_score:
|
||||
min_null_prediction = {
|
||||
"offsets": (0, 0),
|
||||
"score": feature_null_score,
|
||||
"start_logit": start_logits[0],
|
||||
"end_logit": end_logits[0],
|
||||
}
|
||||
|
||||
# Go through all possibilities for the `n_best_size` greater start and end logits.
|
||||
start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()
|
||||
end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()
|
||||
for start_index in start_indexes:
|
||||
for end_index in end_indexes:
|
||||
# Don't consider out-of-scope answers, either because the indices are out of bounds or correspond
|
||||
# to part of the input_ids that are not in the context.
|
||||
if (
|
||||
start_index >= len(offset_mapping)
|
||||
or end_index >= len(offset_mapping)
|
||||
or offset_mapping[start_index] is None
|
||||
or offset_mapping[end_index] is None
|
||||
):
|
||||
continue
|
||||
# Don't consider answers with a length that is either < 0 or > max_answer_length.
|
||||
if end_index < start_index or end_index - start_index + 1 > max_answer_length:
|
||||
continue
|
||||
# Don't consider answer that don't have the maximum context available (if such information is
|
||||
# provided).
|
||||
if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False):
|
||||
continue
|
||||
prelim_predictions.append(
|
||||
{
|
||||
"offsets": (offset_mapping[start_index][0], offset_mapping[end_index][1]),
|
||||
"score": start_logits[start_index] + end_logits[end_index],
|
||||
"start_logit": start_logits[start_index],
|
||||
"end_logit": end_logits[end_index],
|
||||
}
|
||||
)
|
||||
if version_2_with_negative:
|
||||
# Add the minimum null prediction
|
||||
prelim_predictions.append(min_null_prediction)
|
||||
null_score = min_null_prediction["score"]
|
||||
|
||||
# Only keep the best `n_best_size` predictions.
|
||||
predictions = sorted(prelim_predictions, key=lambda x: x["score"], reverse=True)[:n_best_size]
|
||||
|
||||
# Add back the minimum null prediction if it was removed because of its low score.
|
||||
if version_2_with_negative and not any(p["offsets"] == (0, 0) for p in predictions):
|
||||
predictions.append(min_null_prediction)
|
||||
|
||||
# Use the offsets to gather the answer text in the original context.
|
||||
context = example["context"]
|
||||
for pred in predictions:
|
||||
offsets = pred.pop("offsets")
|
||||
pred["text"] = context[offsets[0] : offsets[1]]
|
||||
|
||||
# In the very rare edge case we have not a single non-null prediction, we create a fake prediction to avoid
|
||||
# failure.
|
||||
if len(predictions) == 0 or (len(predictions) == 1 and predictions[0]["text"] == ""):
|
||||
predictions.insert(0, {"text": "empty", "start_logit": 0.0, "end_logit": 0.0, "score": 0.0})
|
||||
|
||||
# Compute the softmax of all scores (we do it with numpy to stay independent from torch/tf in this file, using
|
||||
# the LogSumExp trick).
|
||||
scores = np.array([pred.pop("score") for pred in predictions])
|
||||
exp_scores = np.exp(scores - np.max(scores))
|
||||
probs = exp_scores / exp_scores.sum()
|
||||
|
||||
# Include the probabilities in our predictions.
|
||||
for prob, pred in zip(probs, predictions):
|
||||
pred["probability"] = prob
|
||||
|
||||
# Pick the best prediction. If the null answer is not possible, this is easy.
|
||||
if not version_2_with_negative:
|
||||
all_predictions[example["id"]] = predictions[0]["text"]
|
||||
else:
|
||||
# Otherwise we first need to find the best non-empty prediction.
|
||||
i = 0
|
||||
while predictions[i]["text"] == "":
|
||||
i += 1
|
||||
best_non_null_pred = predictions[i]
|
||||
|
||||
# Then we compare to the null prediction using the threshold.
|
||||
score_diff = null_score - best_non_null_pred["start_logit"] - best_non_null_pred["end_logit"]
|
||||
scores_diff_json[example["id"]] = float(score_diff) # To be JSON-serializable.
|
||||
if score_diff > null_score_diff_threshold:
|
||||
all_predictions[example["id"]] = ""
|
||||
else:
|
||||
all_predictions[example["id"]] = best_non_null_pred["text"]
|
||||
|
||||
# Make `predictions` JSON-serializable by casting np.float back to float.
|
||||
all_nbest_json[example["id"]] = [
|
||||
{k: (float(v) if isinstance(v, (np.float32, np.float64)) else v) for k, v in pred.items()}
|
||||
for pred in predictions
|
||||
]
|
||||
|
||||
# If we have an output_dir, let's save all those dicts.
|
||||
if output_dir is not None:
|
||||
assert os.path.isdir(output_dir), f"{output_dir} is not a directory."
|
||||
|
||||
prediction_file = os.path.join(
|
||||
output_dir, "predictions.json" if prefix is None else f"predictions_{prefix}".json
|
||||
)
|
||||
nbest_file = os.path.join(
|
||||
output_dir, "nbest_predictions.json" if prefix is None else f"nbest_predictions_{prefix}".json
|
||||
)
|
||||
if version_2_with_negative:
|
||||
null_odds_file = os.path.join(
|
||||
output_dir, "null_odds.json" if prefix is None else f"null_odds_{prefix}".json
|
||||
)
|
||||
|
||||
logger.info(f"Saving predictions to {prediction_file}.")
|
||||
with open(prediction_file, "w") as writer:
|
||||
writer.write(json.dumps(all_predictions, indent=4) + "\n")
|
||||
logger.info(f"Saving nbest_preds to {nbest_file}.")
|
||||
with open(nbest_file, "w") as writer:
|
||||
writer.write(json.dumps(all_nbest_json, indent=4) + "\n")
|
||||
if version_2_with_negative:
|
||||
logger.info(f"Saving null_odds to {null_odds_file}.")
|
||||
with open(null_odds_file, "w") as writer:
|
||||
writer.write(json.dumps(scores_diff_json, indent=4) + "\n")
|
||||
|
||||
return all_predictions
|
||||
|
||||
|
||||
def postprocess_qa_predictions_with_beam_search(
|
||||
examples,
|
||||
features,
|
||||
predictions: Tuple[np.ndarray, np.ndarray],
|
||||
version_2_with_negative: bool = False,
|
||||
n_best_size: int = 20,
|
||||
max_answer_length: int = 30,
|
||||
start_n_top: int = 5,
|
||||
end_n_top: int = 5,
|
||||
output_dir: Optional[str] = None,
|
||||
prefix: Optional[str] = None,
|
||||
is_world_process_zero: bool = True,
|
||||
):
|
||||
"""
|
||||
Post-processes the predictions of a question-answering model with beam search to convert them to answers that are substrings of the
|
||||
original contexts. This is the postprocessing functions for models that return start and end logits, indices, as well as
|
||||
cls token predictions.
|
||||
|
||||
Args:
|
||||
examples: The non-preprocessed dataset (see the main script for more information).
|
||||
features: The processed dataset (see the main script for more information).
|
||||
predictions (:obj:`Tuple[np.ndarray, np.ndarray]`):
|
||||
The predictions of the model: two arrays containing the start logits and the end logits respectively. Its
|
||||
first dimension must match the number of elements of :obj:`features`.
|
||||
version_2_with_negative (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
||||
Whether or not the underlying dataset contains examples with no answers.
|
||||
n_best_size (:obj:`int`, `optional`, defaults to 20):
|
||||
The total number of n-best predictions to generate when looking for an answer.
|
||||
max_answer_length (:obj:`int`, `optional`, defaults to 30):
|
||||
The maximum length of an answer that can be generated. This is needed because the start and end predictions
|
||||
are not conditioned on one another.
|
||||
start_n_top (:obj:`int`, `optional`, defaults to 5):
|
||||
The number of top start logits too keep when searching for the :obj:`n_best_size` predictions.
|
||||
end_n_top (:obj:`int`, `optional`, defaults to 5):
|
||||
The number of top end logits too keep when searching for the :obj:`n_best_size` predictions.
|
||||
output_dir (:obj:`str`, `optional`):
|
||||
If provided, the dictionaries of predictions, n_best predictions (with their scores and logits) and, if
|
||||
:obj:`version_2_with_negative=True`, the dictionary of the scores differences between best and null
|
||||
answers, are saved in `output_dir`.
|
||||
prefix (:obj:`str`, `optional`):
|
||||
If provided, the dictionaries mentioned above are saved with `prefix` added to their names.
|
||||
is_world_process_zero (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether this process is the main process or not (used to determine if logging/saves should be done).
|
||||
"""
|
||||
assert len(predictions) == 5, "`predictions` should be a tuple with five elements."
|
||||
start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits = predictions
|
||||
|
||||
assert len(predictions[0]) == len(
|
||||
features
|
||||
), f"Got {len(predictions[0])} predicitions and {len(features)} features."
|
||||
|
||||
# Build a map example to its corresponding features.
|
||||
example_id_to_index = {k: i for i, k in enumerate(examples["id"])}
|
||||
features_per_example = collections.defaultdict(list)
|
||||
for i, feature in enumerate(features):
|
||||
features_per_example[example_id_to_index[feature["example_id"]]].append(i)
|
||||
|
||||
# The dictionaries we have to fill.
|
||||
all_predictions = collections.OrderedDict()
|
||||
all_nbest_json = collections.OrderedDict()
|
||||
scores_diff_json = collections.OrderedDict() if version_2_with_negative else None
|
||||
|
||||
# Logging.
|
||||
logger.setLevel(logging.INFO if is_world_process_zero else logging.WARN)
|
||||
logger.info(f"Post-processing {len(examples)} example predictions split into {len(features)} features.")
|
||||
|
||||
# Let's loop over all the examples!
|
||||
for example_index, example in enumerate(tqdm(examples)):
|
||||
# Those are the indices of the features associated to the current example.
|
||||
feature_indices = features_per_example[example_index]
|
||||
|
||||
min_null_score = None
|
||||
prelim_predictions = []
|
||||
|
||||
# Looping through all the features associated to the current example.
|
||||
for feature_index in feature_indices:
|
||||
# We grab the predictions of the model for this feature.
|
||||
start_log_prob = start_top_log_probs[feature_index]
|
||||
start_indexes = start_top_index[feature_index]
|
||||
end_log_prob = end_top_log_probs[feature_index]
|
||||
end_indexes = end_top_index[feature_index]
|
||||
feature_null_score = cls_logits[feature_index]
|
||||
# This is what will allow us to map some the positions in our logits to span of texts in the original
|
||||
# context.
|
||||
offset_mapping = features[feature_index]["offset_mapping"]
|
||||
# Optional `token_is_max_context`, if provided we will remove answers that do not have the maximum context
|
||||
# available in the current feature.
|
||||
token_is_max_context = features[feature_index].get("token_is_max_context", None)
|
||||
|
||||
# Update minimum null prediction
|
||||
if min_null_score is None or feature_null_score < min_null_score:
|
||||
min_null_score = feature_null_score
|
||||
|
||||
# Go through all possibilities for the `n_start_top`/`n_end_top` greater start and end logits.
|
||||
for i in range(start_n_top):
|
||||
for j in range(end_n_top):
|
||||
start_index = start_indexes[i]
|
||||
j_index = i * end_n_top + j
|
||||
end_index = end_indexes[j_index]
|
||||
# Don't consider out-of-scope answers (last part of the test should be unnecessary because of the
|
||||
# p_mask but let's not take any risk)
|
||||
if (
|
||||
start_index >= len(offset_mapping)
|
||||
or end_index >= len(offset_mapping)
|
||||
or offset_mapping[start_index] is None
|
||||
or offset_mapping[end_index] is None
|
||||
):
|
||||
continue
|
||||
# Don't consider answers with a length negative or > max_answer_length.
|
||||
if end_index < start_index or end_index - start_index + 1 > max_answer_length:
|
||||
continue
|
||||
# Don't consider answer that don't have the maximum context available (if such information is
|
||||
# provided).
|
||||
if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False):
|
||||
continue
|
||||
prelim_predictions.append(
|
||||
{
|
||||
"offsets": (offset_mapping[start_index][0], offset_mapping[end_index][1]),
|
||||
"score": start_log_prob[i] + end_log_prob[j_index],
|
||||
"start_log_prob": start_log_prob[i],
|
||||
"end_log_prob": end_log_prob[j_index],
|
||||
}
|
||||
)
|
||||
|
||||
# Only keep the best `n_best_size` predictions.
|
||||
predictions = sorted(prelim_predictions, key=lambda x: x["score"], reverse=True)[:n_best_size]
|
||||
|
||||
# Use the offsets to gather the answer text in the original context.
|
||||
context = example["context"]
|
||||
for pred in predictions:
|
||||
offsets = pred.pop("offsets")
|
||||
pred["text"] = context[offsets[0] : offsets[1]]
|
||||
|
||||
# In the very rare edge case we have not a single non-null prediction, we create a fake prediction to avoid
|
||||
# failure.
|
||||
if len(predictions) == 0:
|
||||
predictions.insert(0, {"text": "", "start_logit": -1e-6, "end_logit": -1e-6, "score": -2e-6})
|
||||
|
||||
# Compute the softmax of all scores (we do it with numpy to stay independent from torch/tf in this file, using
|
||||
# the LogSumExp trick).
|
||||
scores = np.array([pred.pop("score") for pred in predictions])
|
||||
exp_scores = np.exp(scores - np.max(scores))
|
||||
probs = exp_scores / exp_scores.sum()
|
||||
|
||||
# Include the probabilities in our predictions.
|
||||
for prob, pred in zip(probs, predictions):
|
||||
pred["probability"] = prob
|
||||
|
||||
# Pick the best prediction and set the probability for the null answer.
|
||||
all_predictions[example["id"]] = predictions[0]["text"]
|
||||
if version_2_with_negative:
|
||||
scores_diff_json[example["id"]] = float(min_null_score)
|
||||
|
||||
# Make `predictions` JSON-serializable by casting np.float back to float.
|
||||
all_nbest_json[example["id"]] = [
|
||||
{k: (float(v) if isinstance(v, (np.float32, np.float64)) else v) for k, v in pred.items()}
|
||||
for pred in predictions
|
||||
]
|
||||
|
||||
# If we have an output_dir, let's save all those dicts.
|
||||
if output_dir is not None:
|
||||
assert os.path.isdir(output_dir), f"{output_dir} is not a directory."
|
||||
|
||||
prediction_file = os.path.join(
|
||||
output_dir, "predictions.json" if prefix is None else f"predictions_{prefix}".json
|
||||
)
|
||||
nbest_file = os.path.join(
|
||||
output_dir, "nbest_predictions.json" if prefix is None else f"nbest_predictions_{prefix}".json
|
||||
)
|
||||
if version_2_with_negative:
|
||||
null_odds_file = os.path.join(
|
||||
output_dir, "null_odds.json" if prefix is None else f"null_odds_{prefix}".json
|
||||
)
|
||||
|
||||
print(f"Saving predictions to {prediction_file}.")
|
||||
with open(prediction_file, "w") as writer:
|
||||
writer.write(json.dumps(all_predictions, indent=4) + "\n")
|
||||
print(f"Saving nbest_preds to {nbest_file}.")
|
||||
with open(nbest_file, "w") as writer:
|
||||
writer.write(json.dumps(all_nbest_json, indent=4) + "\n")
|
||||
if version_2_with_negative:
|
||||
print(f"Saving null_odds to {null_odds_file}.")
|
||||
with open(null_odds_file, "w") as writer:
|
||||
writer.write(json.dumps(scores_diff_json, indent=4) + "\n")
|
||||
|
||||
return all_predictions, scores_diff_json
|
||||
+12
-11
@@ -46,7 +46,7 @@ if SRC_DIRS is not None:
|
||||
import run_mlm
|
||||
import run_ner
|
||||
import run_pl_glue
|
||||
import run_qa as run_squad
|
||||
import run_squad
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
@@ -213,8 +213,8 @@ class ExamplesTests(TestCasePlus):
|
||||
--do_eval
|
||||
--warmup_steps=2
|
||||
--learning_rate=2e-4
|
||||
--per_device_train_batch_size=2
|
||||
--per_device_eval_batch_size=2
|
||||
--per_gpu_train_batch_size=2
|
||||
--per_gpu_eval_batch_size=2
|
||||
--num_train_epochs=2
|
||||
""".split()
|
||||
|
||||
@@ -235,25 +235,26 @@ class ExamplesTests(TestCasePlus):
|
||||
tmp_dir = self.get_auto_remove_tmp_dir()
|
||||
testargs = f"""
|
||||
run_squad.py
|
||||
--model_name_or_path bert-base-uncased
|
||||
--version_2_with_negative
|
||||
--train_file tests/fixtures/tests_samples/SQUAD/sample.json
|
||||
--validation_file tests/fixtures/tests_samples/SQUAD/sample.json
|
||||
--model_type=distilbert
|
||||
--model_name_or_path=sshleifer/tiny-distilbert-base-cased-distilled-squad
|
||||
--data_dir=./tests/fixtures/tests_samples/SQUAD
|
||||
--output_dir {tmp_dir}
|
||||
--overwrite_output_dir
|
||||
--max_steps=10
|
||||
--warmup_steps=2
|
||||
--do_train
|
||||
--do_eval
|
||||
--version_2_with_negative
|
||||
--learning_rate=2e-4
|
||||
--per_device_train_batch_size=2
|
||||
--per_device_eval_batch_size=1
|
||||
--per_gpu_train_batch_size=2
|
||||
--per_gpu_eval_batch_size=1
|
||||
--seed=42
|
||||
""".split()
|
||||
|
||||
with patch.object(sys, "argv", testargs):
|
||||
result = run_squad.main()
|
||||
self.assertGreaterEqual(result["f1"], 30)
|
||||
self.assertGreaterEqual(result["exact"], 30)
|
||||
self.assertGreaterEqual(result["f1"], 25)
|
||||
self.assertGreaterEqual(result["exact"], 21)
|
||||
|
||||
@require_torch_non_multi_gpu_but_fix_me
|
||||
def test_generation(self):
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
---
|
||||
language: code
|
||||
thumbnail: https://cdn-media.huggingface.co/CodeBERTa/CodeBERTa.png
|
||||
tags:
|
||||
- test
|
||||
datasets:
|
||||
- code_search_net
|
||||
- aeslc
|
||||
metrics:
|
||||
- bertscore
|
||||
---
|
||||
|
||||
# CodeBERTa-language-id: The World’s fanciest programming language identification algo 🤯
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
---
|
||||
language: code
|
||||
thumbnail: https://cdn-media.huggingface.co/CodeBERTa/CodeBERTa.png
|
||||
datasets:
|
||||
- code_search_net
|
||||
---
|
||||
|
||||
# CodeBERTa
|
||||
|
||||
+32
-654
@@ -3,7 +3,6 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "YKdSeUmVSXah",
|
||||
"pycharm": {
|
||||
"is_executing": false,
|
||||
"name": "#%% md\n"
|
||||
@@ -60,7 +59,6 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "TFHTP6CFSXai",
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
@@ -80,7 +78,6 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "KnT3Jn6fSXai",
|
||||
"pycharm": {
|
||||
"is_executing": false,
|
||||
"name": "#%% code\n"
|
||||
@@ -95,13 +92,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "UIQGDTIDSXai",
|
||||
"outputId": "9851454a-c898-4fba-a389-9b16462a27c1",
|
||||
"pycharm": {
|
||||
"is_executing": false,
|
||||
"name": "#%% code\n"
|
||||
@@ -111,13 +103,11 @@
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<torch.autograd.grad_mode.set_grad_enabled at 0x7f9c03e5b3c8>"
|
||||
"<torch.autograd.grad_mode.set_grad_enabled at 0x7f10b441e890>"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
@@ -130,9 +120,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"id": "1xMDTHQXSXai",
|
||||
"pycharm": {
|
||||
"is_executing": false,
|
||||
"name": "#%% code\n"
|
||||
@@ -151,7 +140,6 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "l6EcynhYSXai",
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
@@ -164,13 +152,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "XgkFg52fSXai",
|
||||
"outputId": "94b569d4-5415-4327-f39e-c9541b0a53e0",
|
||||
"pycharm": {
|
||||
"is_executing": false,
|
||||
"name": "#%% code\n"
|
||||
@@ -206,17 +189,13 @@
|
||||
"print(\"Tokens PyTorch: {}\".format(tokens_pt))\n",
|
||||
"\n",
|
||||
"# Now we're ready to go through BERT with out input\n",
|
||||
"outputs = model(tokens_pt)\n",
|
||||
"last_hidden_state = outputs.last_hidden_state\n",
|
||||
"pooler_output = outputs.pooler_output\n",
|
||||
"\n",
|
||||
"print(\"Token wise output: {}, Pooled output: {}\".format(last_hidden_state.shape, pooler_output.shape))"
|
||||
"outputs, pooled = model(tokens_pt)\n",
|
||||
"print(\"Token wise output: {}, Pooled output: {}\".format(outputs.shape, pooled.shape))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "lBbvwNKXSXaj",
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
@@ -236,7 +215,6 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "DCxuDWH2SXaj",
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
@@ -249,13 +227,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "sgcNCdXUSXaj",
|
||||
"outputId": "af2fb928-7c17-475b-cf81-89cfc4b1d9e5",
|
||||
"pycharm": {
|
||||
"is_executing": false,
|
||||
"name": "#%% code\n"
|
||||
@@ -287,18 +260,13 @@
|
||||
"for key, value in tokens_pt2.items():\n",
|
||||
" print(\"{}:\\n\\t{}\".format(key, value))\n",
|
||||
"\n",
|
||||
"outputs2 = model(**tokens_pt2)\n",
|
||||
"last_hidden_state2 = outputs2.last_hidden_state\n",
|
||||
"pooler_output2 = outputs2.pooler_output\n",
|
||||
"\n",
|
||||
"print(\"Difference with previous code: ({}, {})\".format((last_hidden_state2 - last_hidden_state).sum(), (pooler_output2 - pooler_output).sum()))"
|
||||
"outputs2, pooled2 = model(**tokens_pt2)\n",
|
||||
"print(\"Difference with previous code: ({}, {})\".format((outputs2 - outputs).sum(), (pooled2 - pooled).sum()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "gC-7xGYPSXal"
|
||||
},
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"As you can see above, calling the tokenizer provides a convenient way to generate all the required parameters\n",
|
||||
"that will go through the model. \n",
|
||||
@@ -311,13 +279,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "Pl2HIcwDSXal",
|
||||
"outputId": "22e5d010-47a9-4a12-a67d-208e5016157e",
|
||||
"pycharm": {
|
||||
"is_executing": false
|
||||
}
|
||||
@@ -357,13 +320,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "1NtvWOgzSXam",
|
||||
"outputId": "e66c47d0-e106-408d-d01c-9ac194ca3ec6",
|
||||
"pycharm": {
|
||||
"is_executing": false
|
||||
}
|
||||
@@ -400,9 +358,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "vkRYm2HESXan"
|
||||
},
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Frameworks interoperability\n",
|
||||
"\n",
|
||||
@@ -414,9 +370,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"id": "Kubwm-wJSXan",
|
||||
"pycharm": {
|
||||
"is_executing": false
|
||||
}
|
||||
@@ -432,13 +387,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": 9,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "lJ13tlzOSXan",
|
||||
"outputId": "1e4ac151-a8fc-4b34-946a-da0bc44ed0e6",
|
||||
"pycharm": {
|
||||
"is_executing": false
|
||||
}
|
||||
@@ -448,8 +398,8 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"last_hidden_state differences: 1.0094e-05\n",
|
||||
"pooler_output differences: 7.2969e-07\n"
|
||||
"output differences: 1.6236e-05\n",
|
||||
"pooled differences: -1.3039e-08\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -463,14 +413,13 @@
|
||||
"\n",
|
||||
"# Models outputs 2 values (The value for each tokens, the pooled representation of the input sentence)\n",
|
||||
"# Here we compare the output differences between PyTorch and TensorFlow.\n",
|
||||
"for name in [\"last_hidden_state\", \"pooler_output\"]:\n",
|
||||
" print(\"{} differences: {:.5}\".format(name, (output_tf[name].numpy() - output_pt[name].numpy()).sum()))"
|
||||
"for name, o_tf, o_pt in zip([\"output\", \"pooled\"], output_tf, output_pt):\n",
|
||||
" print(\"{} differences: {:.5}\".format(name, (o_tf.numpy() - o_pt.numpy()).sum()))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "CQf_fpApSXao",
|
||||
"pycharm": {
|
||||
"name": "#%% md\n"
|
||||
}
|
||||
@@ -493,85 +442,21 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 185,
|
||||
"referenced_widgets": [
|
||||
"fcffccb427714665bec7d621d00d4ce3",
|
||||
"7aa02ef05fe64489ad6c969dd92d1b07",
|
||||
"817f53e3fa5c43e29ad1b4410c3df7db",
|
||||
"50a441d7a43c4a809a09505b36e83375",
|
||||
"497ba6a585a147459f1346c0661d5c94",
|
||||
"a18c5319739141af9a255bccf25f6884",
|
||||
"cf319210c8134cdba487525c49e4813b",
|
||||
"f1f0272d9bea4e9fad3be8646b45d629",
|
||||
"530b39d56f6b4e0caae3317855c4bcf4",
|
||||
"c5e735694f2c4813a1d6f0d867119f67",
|
||||
"8d53b8dc213f405d8187f3c1f005826d",
|
||||
"d492afe626804d95a5cfac0550913190",
|
||||
"a657a312068b43529afed2050bce572f",
|
||||
"fe230ff13a82400f97cf6f292e8851ba",
|
||||
"be97cf2269d748f3b1a916b5376f7736",
|
||||
"74bd90a09da74db5bcbbe86f044bd664"
|
||||
]
|
||||
},
|
||||
"id": "wfxMOXb-SXao",
|
||||
"outputId": "fa667556-fbf2-4c86-fc7e-9e3d3ec9da88",
|
||||
"pycharm": {
|
||||
"is_executing": false
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "fcffccb427714665bec7d621d00d4ce3",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(FloatProgress(value=0.0, description='Downloading', max=411.0, style=ProgressStyle(description_…"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "530b39d56f6b4e0caae3317855c4bcf4",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HBox(children=(FloatProgress(value=0.0, description='Downloading', max=263273408.0, style=ProgressStyle(descri…"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"CPU times: user 64.4 ms, sys: 0 ns, total: 64.4 ms\n",
|
||||
"Wall time: 72.9 ms\n",
|
||||
"CPU times: user 130 ms, sys: 124 µs, total: 130 ms\n",
|
||||
"Wall time: 131 ms\n"
|
||||
"CPU times: user 232 ms, sys: 0 ns, total: 232 ms\n",
|
||||
"Wall time: 21.1 ms\n",
|
||||
"CPU times: user 511 ms, sys: 0 ns, total: 511 ms\n",
|
||||
"Wall time: 43.9 ms\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -591,9 +476,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "7lSIc7FbSXao"
|
||||
},
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Community provided models\n",
|
||||
"\n",
|
||||
@@ -607,13 +490,8 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 11,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "cxLYnadGSXao",
|
||||
"outputId": "70ab584a-e795-490a-8c6a-06e034b3df3d",
|
||||
"pycharm": {
|
||||
"is_executing": false
|
||||
}
|
||||
@@ -627,7 +505,7 @@
|
||||
"Tokens (str) : ['[CLS]', 'Hug', '##ging', 'Fac', '##e', 'ist', 'eine', 'französische', 'Firma', 'mit', 'Sitz', 'in', 'New', '-', 'York', '.', '[SEP]']\n",
|
||||
"Tokens (attn_mask): [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
|
||||
"\n",
|
||||
"Token wise output: torch.Size([1, 17, 768]), Pooled output: torch.Size([1, 768])\n"
|
||||
"Token wise output: torch.Size([1, 7, 768]), Pooled output: torch.Size([1, 768])\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -645,19 +523,13 @@
|
||||
"print(\"Tokens (attn_mask): {}\".format(de_input['attention_mask'].tolist()[0]))\n",
|
||||
"print()\n",
|
||||
"\n",
|
||||
"outputs_de = de_bert(**de_input)\n",
|
||||
"last_hidden_state_de = outputs_de.last_hidden_state\n",
|
||||
"pooler_output_de = outputs_de.pooler_output\n",
|
||||
"output_de, pooled_de = de_bert(**de_input)\n",
|
||||
"\n",
|
||||
"print(\"Token wise output: {}, Pooled output: {}\".format(last_hidden_state_de.shape, pooler_output_de.shape))"
|
||||
"print(\"Token wise output: {}, Pooled output: {}\".format(outputs.shape, pooled.shape))"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "02-transformers.ipynb",
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
@@ -673,7 +545,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
"version": "3.7.4"
|
||||
},
|
||||
"pycharm": {
|
||||
"stem_cell": {
|
||||
@@ -683,502 +555,8 @@
|
||||
},
|
||||
"source": []
|
||||
}
|
||||
},
|
||||
"widgets": {
|
||||
"application/vnd.jupyter.widget-state+json": {
|
||||
"497ba6a585a147459f1346c0661d5c94": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "ProgressStyleModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "ProgressStyleModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "StyleView",
|
||||
"bar_color": null,
|
||||
"description_width": "initial"
|
||||
}
|
||||
},
|
||||
"50a441d7a43c4a809a09505b36e83375": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "HTMLModel",
|
||||
"state": {
|
||||
"_dom_classes": [],
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "HTMLModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/controls",
|
||||
"_view_module_version": "1.5.0",
|
||||
"_view_name": "HTMLView",
|
||||
"description": "",
|
||||
"description_tooltip": null,
|
||||
"layout": "IPY_MODEL_f1f0272d9bea4e9fad3be8646b45d629",
|
||||
"placeholder": "",
|
||||
"style": "IPY_MODEL_cf319210c8134cdba487525c49e4813b",
|
||||
"value": " 411/411 [00:19<00:00, 21.5B/s]"
|
||||
}
|
||||
},
|
||||
"530b39d56f6b4e0caae3317855c4bcf4": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "HBoxModel",
|
||||
"state": {
|
||||
"_dom_classes": [],
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "HBoxModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/controls",
|
||||
"_view_module_version": "1.5.0",
|
||||
"_view_name": "HBoxView",
|
||||
"box_style": "",
|
||||
"children": [
|
||||
"IPY_MODEL_8d53b8dc213f405d8187f3c1f005826d",
|
||||
"IPY_MODEL_d492afe626804d95a5cfac0550913190"
|
||||
],
|
||||
"layout": "IPY_MODEL_c5e735694f2c4813a1d6f0d867119f67"
|
||||
}
|
||||
},
|
||||
"74bd90a09da74db5bcbbe86f044bd664": {
|
||||
"model_module": "@jupyter-widgets/base",
|
||||
"model_name": "LayoutModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/base",
|
||||
"_model_module_version": "1.2.0",
|
||||
"_model_name": "LayoutModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "LayoutView",
|
||||
"align_content": null,
|
||||
"align_items": null,
|
||||
"align_self": null,
|
||||
"border": null,
|
||||
"bottom": null,
|
||||
"display": null,
|
||||
"flex": null,
|
||||
"flex_flow": null,
|
||||
"grid_area": null,
|
||||
"grid_auto_columns": null,
|
||||
"grid_auto_flow": null,
|
||||
"grid_auto_rows": null,
|
||||
"grid_column": null,
|
||||
"grid_gap": null,
|
||||
"grid_row": null,
|
||||
"grid_template_areas": null,
|
||||
"grid_template_columns": null,
|
||||
"grid_template_rows": null,
|
||||
"height": null,
|
||||
"justify_content": null,
|
||||
"justify_items": null,
|
||||
"left": null,
|
||||
"margin": null,
|
||||
"max_height": null,
|
||||
"max_width": null,
|
||||
"min_height": null,
|
||||
"min_width": null,
|
||||
"object_fit": null,
|
||||
"object_position": null,
|
||||
"order": null,
|
||||
"overflow": null,
|
||||
"overflow_x": null,
|
||||
"overflow_y": null,
|
||||
"padding": null,
|
||||
"right": null,
|
||||
"top": null,
|
||||
"visibility": null,
|
||||
"width": null
|
||||
}
|
||||
},
|
||||
"7aa02ef05fe64489ad6c969dd92d1b07": {
|
||||
"model_module": "@jupyter-widgets/base",
|
||||
"model_name": "LayoutModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/base",
|
||||
"_model_module_version": "1.2.0",
|
||||
"_model_name": "LayoutModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "LayoutView",
|
||||
"align_content": null,
|
||||
"align_items": null,
|
||||
"align_self": null,
|
||||
"border": null,
|
||||
"bottom": null,
|
||||
"display": null,
|
||||
"flex": null,
|
||||
"flex_flow": null,
|
||||
"grid_area": null,
|
||||
"grid_auto_columns": null,
|
||||
"grid_auto_flow": null,
|
||||
"grid_auto_rows": null,
|
||||
"grid_column": null,
|
||||
"grid_gap": null,
|
||||
"grid_row": null,
|
||||
"grid_template_areas": null,
|
||||
"grid_template_columns": null,
|
||||
"grid_template_rows": null,
|
||||
"height": null,
|
||||
"justify_content": null,
|
||||
"justify_items": null,
|
||||
"left": null,
|
||||
"margin": null,
|
||||
"max_height": null,
|
||||
"max_width": null,
|
||||
"min_height": null,
|
||||
"min_width": null,
|
||||
"object_fit": null,
|
||||
"object_position": null,
|
||||
"order": null,
|
||||
"overflow": null,
|
||||
"overflow_x": null,
|
||||
"overflow_y": null,
|
||||
"padding": null,
|
||||
"right": null,
|
||||
"top": null,
|
||||
"visibility": null,
|
||||
"width": null
|
||||
}
|
||||
},
|
||||
"817f53e3fa5c43e29ad1b4410c3df7db": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "FloatProgressModel",
|
||||
"state": {
|
||||
"_dom_classes": [],
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "FloatProgressModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/controls",
|
||||
"_view_module_version": "1.5.0",
|
||||
"_view_name": "ProgressView",
|
||||
"bar_style": "success",
|
||||
"description": "Downloading: 100%",
|
||||
"description_tooltip": null,
|
||||
"layout": "IPY_MODEL_a18c5319739141af9a255bccf25f6884",
|
||||
"max": 411,
|
||||
"min": 0,
|
||||
"orientation": "horizontal",
|
||||
"style": "IPY_MODEL_497ba6a585a147459f1346c0661d5c94",
|
||||
"value": 411
|
||||
}
|
||||
},
|
||||
"8d53b8dc213f405d8187f3c1f005826d": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "FloatProgressModel",
|
||||
"state": {
|
||||
"_dom_classes": [],
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "FloatProgressModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/controls",
|
||||
"_view_module_version": "1.5.0",
|
||||
"_view_name": "ProgressView",
|
||||
"bar_style": "success",
|
||||
"description": "Downloading: 100%",
|
||||
"description_tooltip": null,
|
||||
"layout": "IPY_MODEL_fe230ff13a82400f97cf6f292e8851ba",
|
||||
"max": 263273408,
|
||||
"min": 0,
|
||||
"orientation": "horizontal",
|
||||
"style": "IPY_MODEL_a657a312068b43529afed2050bce572f",
|
||||
"value": 263273408
|
||||
}
|
||||
},
|
||||
"a18c5319739141af9a255bccf25f6884": {
|
||||
"model_module": "@jupyter-widgets/base",
|
||||
"model_name": "LayoutModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/base",
|
||||
"_model_module_version": "1.2.0",
|
||||
"_model_name": "LayoutModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "LayoutView",
|
||||
"align_content": null,
|
||||
"align_items": null,
|
||||
"align_self": null,
|
||||
"border": null,
|
||||
"bottom": null,
|
||||
"display": null,
|
||||
"flex": null,
|
||||
"flex_flow": null,
|
||||
"grid_area": null,
|
||||
"grid_auto_columns": null,
|
||||
"grid_auto_flow": null,
|
||||
"grid_auto_rows": null,
|
||||
"grid_column": null,
|
||||
"grid_gap": null,
|
||||
"grid_row": null,
|
||||
"grid_template_areas": null,
|
||||
"grid_template_columns": null,
|
||||
"grid_template_rows": null,
|
||||
"height": null,
|
||||
"justify_content": null,
|
||||
"justify_items": null,
|
||||
"left": null,
|
||||
"margin": null,
|
||||
"max_height": null,
|
||||
"max_width": null,
|
||||
"min_height": null,
|
||||
"min_width": null,
|
||||
"object_fit": null,
|
||||
"object_position": null,
|
||||
"order": null,
|
||||
"overflow": null,
|
||||
"overflow_x": null,
|
||||
"overflow_y": null,
|
||||
"padding": null,
|
||||
"right": null,
|
||||
"top": null,
|
||||
"visibility": null,
|
||||
"width": null
|
||||
}
|
||||
},
|
||||
"a657a312068b43529afed2050bce572f": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "ProgressStyleModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "ProgressStyleModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "StyleView",
|
||||
"bar_color": null,
|
||||
"description_width": "initial"
|
||||
}
|
||||
},
|
||||
"be97cf2269d748f3b1a916b5376f7736": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "DescriptionStyleModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "DescriptionStyleModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "StyleView",
|
||||
"description_width": ""
|
||||
}
|
||||
},
|
||||
"c5e735694f2c4813a1d6f0d867119f67": {
|
||||
"model_module": "@jupyter-widgets/base",
|
||||
"model_name": "LayoutModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/base",
|
||||
"_model_module_version": "1.2.0",
|
||||
"_model_name": "LayoutModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "LayoutView",
|
||||
"align_content": null,
|
||||
"align_items": null,
|
||||
"align_self": null,
|
||||
"border": null,
|
||||
"bottom": null,
|
||||
"display": null,
|
||||
"flex": null,
|
||||
"flex_flow": null,
|
||||
"grid_area": null,
|
||||
"grid_auto_columns": null,
|
||||
"grid_auto_flow": null,
|
||||
"grid_auto_rows": null,
|
||||
"grid_column": null,
|
||||
"grid_gap": null,
|
||||
"grid_row": null,
|
||||
"grid_template_areas": null,
|
||||
"grid_template_columns": null,
|
||||
"grid_template_rows": null,
|
||||
"height": null,
|
||||
"justify_content": null,
|
||||
"justify_items": null,
|
||||
"left": null,
|
||||
"margin": null,
|
||||
"max_height": null,
|
||||
"max_width": null,
|
||||
"min_height": null,
|
||||
"min_width": null,
|
||||
"object_fit": null,
|
||||
"object_position": null,
|
||||
"order": null,
|
||||
"overflow": null,
|
||||
"overflow_x": null,
|
||||
"overflow_y": null,
|
||||
"padding": null,
|
||||
"right": null,
|
||||
"top": null,
|
||||
"visibility": null,
|
||||
"width": null
|
||||
}
|
||||
},
|
||||
"cf319210c8134cdba487525c49e4813b": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "DescriptionStyleModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "DescriptionStyleModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "StyleView",
|
||||
"description_width": ""
|
||||
}
|
||||
},
|
||||
"d492afe626804d95a5cfac0550913190": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "HTMLModel",
|
||||
"state": {
|
||||
"_dom_classes": [],
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "HTMLModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/controls",
|
||||
"_view_module_version": "1.5.0",
|
||||
"_view_name": "HTMLView",
|
||||
"description": "",
|
||||
"description_tooltip": null,
|
||||
"layout": "IPY_MODEL_74bd90a09da74db5bcbbe86f044bd664",
|
||||
"placeholder": "",
|
||||
"style": "IPY_MODEL_be97cf2269d748f3b1a916b5376f7736",
|
||||
"value": " 263M/263M [00:06<00:00, 43.5MB/s]"
|
||||
}
|
||||
},
|
||||
"f1f0272d9bea4e9fad3be8646b45d629": {
|
||||
"model_module": "@jupyter-widgets/base",
|
||||
"model_name": "LayoutModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/base",
|
||||
"_model_module_version": "1.2.0",
|
||||
"_model_name": "LayoutModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "LayoutView",
|
||||
"align_content": null,
|
||||
"align_items": null,
|
||||
"align_self": null,
|
||||
"border": null,
|
||||
"bottom": null,
|
||||
"display": null,
|
||||
"flex": null,
|
||||
"flex_flow": null,
|
||||
"grid_area": null,
|
||||
"grid_auto_columns": null,
|
||||
"grid_auto_flow": null,
|
||||
"grid_auto_rows": null,
|
||||
"grid_column": null,
|
||||
"grid_gap": null,
|
||||
"grid_row": null,
|
||||
"grid_template_areas": null,
|
||||
"grid_template_columns": null,
|
||||
"grid_template_rows": null,
|
||||
"height": null,
|
||||
"justify_content": null,
|
||||
"justify_items": null,
|
||||
"left": null,
|
||||
"margin": null,
|
||||
"max_height": null,
|
||||
"max_width": null,
|
||||
"min_height": null,
|
||||
"min_width": null,
|
||||
"object_fit": null,
|
||||
"object_position": null,
|
||||
"order": null,
|
||||
"overflow": null,
|
||||
"overflow_x": null,
|
||||
"overflow_y": null,
|
||||
"padding": null,
|
||||
"right": null,
|
||||
"top": null,
|
||||
"visibility": null,
|
||||
"width": null
|
||||
}
|
||||
},
|
||||
"fcffccb427714665bec7d621d00d4ce3": {
|
||||
"model_module": "@jupyter-widgets/controls",
|
||||
"model_name": "HBoxModel",
|
||||
"state": {
|
||||
"_dom_classes": [],
|
||||
"_model_module": "@jupyter-widgets/controls",
|
||||
"_model_module_version": "1.5.0",
|
||||
"_model_name": "HBoxModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/controls",
|
||||
"_view_module_version": "1.5.0",
|
||||
"_view_name": "HBoxView",
|
||||
"box_style": "",
|
||||
"children": [
|
||||
"IPY_MODEL_817f53e3fa5c43e29ad1b4410c3df7db",
|
||||
"IPY_MODEL_50a441d7a43c4a809a09505b36e83375"
|
||||
],
|
||||
"layout": "IPY_MODEL_7aa02ef05fe64489ad6c969dd92d1b07"
|
||||
}
|
||||
},
|
||||
"fe230ff13a82400f97cf6f292e8851ba": {
|
||||
"model_module": "@jupyter-widgets/base",
|
||||
"model_name": "LayoutModel",
|
||||
"state": {
|
||||
"_model_module": "@jupyter-widgets/base",
|
||||
"_model_module_version": "1.2.0",
|
||||
"_model_name": "LayoutModel",
|
||||
"_view_count": null,
|
||||
"_view_module": "@jupyter-widgets/base",
|
||||
"_view_module_version": "1.2.0",
|
||||
"_view_name": "LayoutView",
|
||||
"align_content": null,
|
||||
"align_items": null,
|
||||
"align_self": null,
|
||||
"border": null,
|
||||
"bottom": null,
|
||||
"display": null,
|
||||
"flex": null,
|
||||
"flex_flow": null,
|
||||
"grid_area": null,
|
||||
"grid_auto_columns": null,
|
||||
"grid_auto_flow": null,
|
||||
"grid_auto_rows": null,
|
||||
"grid_column": null,
|
||||
"grid_gap": null,
|
||||
"grid_row": null,
|
||||
"grid_template_areas": null,
|
||||
"grid_template_columns": null,
|
||||
"grid_template_rows": null,
|
||||
"height": null,
|
||||
"justify_content": null,
|
||||
"justify_items": null,
|
||||
"left": null,
|
||||
"margin": null,
|
||||
"max_height": null,
|
||||
"max_width": null,
|
||||
"min_height": null,
|
||||
"min_width": null,
|
||||
"object_fit": null,
|
||||
"object_position": null,
|
||||
"order": null,
|
||||
"overflow": null,
|
||||
"overflow_x": null,
|
||||
"overflow_y": null,
|
||||
"padding": null,
|
||||
"right": null,
|
||||
"top": null,
|
||||
"visibility": null,
|
||||
"width": null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
@@ -152,7 +152,6 @@ from .models.marian import MarianConfig
|
||||
from .models.mbart import MBartConfig
|
||||
from .models.mmbt import MMBTConfig
|
||||
from .models.mobilebert import MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MobileBertConfig, MobileBertTokenizer
|
||||
from .models.mpnet import MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP, MPNetConfig, MPNetTokenizer
|
||||
from .models.mt5 import MT5Config
|
||||
from .models.openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, OpenAIGPTTokenizer
|
||||
from .models.pegasus import PegasusConfig
|
||||
@@ -257,7 +256,6 @@ if is_tokenizers_available():
|
||||
from .models.lxmert import LxmertTokenizerFast
|
||||
from .models.mbart import MBartTokenizerFast
|
||||
from .models.mobilebert import MobileBertTokenizerFast
|
||||
from .models.mpnet import MPNetTokenizerFast
|
||||
from .models.mt5 import MT5TokenizerFast
|
||||
from .models.openai import OpenAIGPTTokenizerFast
|
||||
from .models.pegasus import PegasusTokenizerFast
|
||||
@@ -322,14 +320,12 @@ if is_torch_available():
|
||||
)
|
||||
from .generation_beam_search import BeamScorer, BeamSearchScorer
|
||||
from .generation_logits_process import (
|
||||
HammingDiversityLogitsProcessor,
|
||||
LogitsProcessor,
|
||||
LogitsProcessorList,
|
||||
LogitsWarper,
|
||||
MinLengthLogitsProcessor,
|
||||
NoBadWordsLogitsProcessor,
|
||||
NoRepeatNGramLogitsProcessor,
|
||||
PrefixConstrainedLogitsProcessor,
|
||||
RepetitionPenaltyLogitsProcessor,
|
||||
TemperatureLogitsWarper,
|
||||
TopKLogitsWarper,
|
||||
@@ -379,7 +375,6 @@ if is_torch_available():
|
||||
BartForQuestionAnswering,
|
||||
BartForSequenceClassification,
|
||||
BartModel,
|
||||
BartPretrainedModel,
|
||||
PretrainedBartModel,
|
||||
)
|
||||
from .models.bert import (
|
||||
@@ -534,17 +529,6 @@ if is_torch_available():
|
||||
MobileBertPreTrainedModel,
|
||||
load_tf_weights_in_mobilebert,
|
||||
)
|
||||
from .models.mpnet import (
|
||||
MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
MPNetForMaskedLM,
|
||||
MPNetForMultipleChoice,
|
||||
MPNetForQuestionAnswering,
|
||||
MPNetForSequenceClassification,
|
||||
MPNetForTokenClassification,
|
||||
MPNetLayer,
|
||||
MPNetModel,
|
||||
MPNetPreTrainedModel,
|
||||
)
|
||||
from .models.mt5 import MT5EncoderModel, MT5ForConditionalGeneration, MT5Model
|
||||
from .models.openai import (
|
||||
OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
@@ -606,12 +590,12 @@ if is_torch_available():
|
||||
T5PreTrainedModel,
|
||||
load_tf_weights_in_t5,
|
||||
)
|
||||
from .models.tapas import (
|
||||
from .models.tapas import(
|
||||
TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
TapasModel,
|
||||
TapasForMaskedLM,
|
||||
TapasForQuestionAnswering,
|
||||
TapasForSequenceClassification,
|
||||
TapasModel,
|
||||
)
|
||||
from .models.transfo_xl import (
|
||||
TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
@@ -852,17 +836,6 @@ if is_tf_available():
|
||||
TFMobileBertModel,
|
||||
TFMobileBertPreTrainedModel,
|
||||
)
|
||||
from .models.mpnet import (
|
||||
TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
TFMPNetForMaskedLM,
|
||||
TFMPNetForMultipleChoice,
|
||||
TFMPNetForQuestionAnswering,
|
||||
TFMPNetForSequenceClassification,
|
||||
TFMPNetForTokenClassification,
|
||||
TFMPNetMainLayer,
|
||||
TFMPNetModel,
|
||||
TFMPNetPreTrainedModel,
|
||||
)
|
||||
from .models.mt5 import TFMT5EncoderModel, TFMT5ForConditionalGeneration, TFMT5Model
|
||||
from .models.openai import (
|
||||
TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
@@ -945,7 +918,7 @@ else:
|
||||
|
||||
if is_flax_available():
|
||||
from .models.auto import FLAX_MODEL_MAPPING, FlaxAutoModel
|
||||
from .models.bert import FlaxBertForMaskedLM, FlaxBertModel
|
||||
from .models.bert import FlaxBertModel
|
||||
from .models.roberta import FlaxRobertaModel
|
||||
else:
|
||||
# Import the same objects as dummies to get them in the namespace.
|
||||
|
||||
@@ -19,18 +19,12 @@ from argparse import ArgumentParser, Namespace
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from cookiecutter.main import cookiecutter
|
||||
from transformers.commands import BaseTransformersCLICommand
|
||||
|
||||
from ..utils import logging
|
||||
|
||||
|
||||
try:
|
||||
from cookiecutter.main import cookiecutter
|
||||
|
||||
_has_cookiecutter = True
|
||||
except ImportError:
|
||||
_has_cookiecutter = False
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
|
||||
@@ -55,11 +49,6 @@ class AddNewModelCommand(BaseTransformersCLICommand):
|
||||
self._path = path
|
||||
|
||||
def run(self):
|
||||
if not _has_cookiecutter:
|
||||
raise ImportError(
|
||||
"Model creation dependencies are required to use the `add_new_model` command. Install them by running "
|
||||
"the folowing at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n"
|
||||
)
|
||||
# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
|
||||
directories = [directory for directory in os.listdir() if "cookiecutter-template-" == directory[:22]]
|
||||
if len(directories) > 0:
|
||||
@@ -164,11 +153,6 @@ class AddNewModelCommand(BaseTransformersCLICommand):
|
||||
f"{model_dir}/tokenization_{lowercase_model_name}.py",
|
||||
)
|
||||
|
||||
shutil.move(
|
||||
f"{directory}/tokenization_fast_{lowercase_model_name}.py",
|
||||
f"{model_dir}/tokenization_{lowercase_model_name}_fast.py",
|
||||
)
|
||||
|
||||
from os import fdopen, remove
|
||||
from shutil import copymode, move
|
||||
from tempfile import mkstemp
|
||||
|
||||
@@ -95,12 +95,6 @@ class PretrainedConfig(object):
|
||||
sentences are finished per batch or not.
|
||||
- **num_beams** (:obj:`int`, `optional`, defaults to 1) -- Number of beams for beam search that will be used by
|
||||
default in the :obj:`generate` method of the model. 1 means no beam search.
|
||||
- **num_beam_groups** (:obj:`int`, `optional`, defaults to 1) -- Number of groups to divide :obj:`num_beams`
|
||||
into in order to ensure diversity among different groups of beams that will be used by default in the
|
||||
:obj:`generate` method of the model. 1 means no group beam search.
|
||||
- **diversity_penalty** (:obj:`float`, `optional`, defaults to 0.0) -- Value to control diversity for group
|
||||
beam search. that will be used by default in the :obj:`generate` method of the model. 0 means no diversity
|
||||
penalty. The higher the penalty, the more diverse are the outputs.
|
||||
- **temperature** (:obj:`float`, `optional`, defaults to 1) -- The value used to module the next token
|
||||
probabilities that will be used by default in the :obj:`generate` method of the model. Must be strictly
|
||||
positive.
|
||||
@@ -191,8 +185,6 @@ class PretrainedConfig(object):
|
||||
self.do_sample = kwargs.pop("do_sample", False)
|
||||
self.early_stopping = kwargs.pop("early_stopping", False)
|
||||
self.num_beams = kwargs.pop("num_beams", 1)
|
||||
self.num_beam_groups = kwargs.pop("num_beam_groups", 1)
|
||||
self.diversity_penalty = kwargs.pop("diversity_penalty", 0.0)
|
||||
self.temperature = kwargs.pop("temperature", 1.0)
|
||||
self.top_k = kwargs.pop("top_k", 50)
|
||||
self.top_p = kwargs.pop("top_p", 1.0)
|
||||
|
||||
@@ -28,7 +28,7 @@ from .utils import DataProcessor
|
||||
|
||||
|
||||
# Store the tokenizers which insert 2 separators tokens
|
||||
MULTI_SEP_TOKENS_TOKENIZERS_SET = {"roberta", "camembert", "bart", "mpnet"}
|
||||
MULTI_SEP_TOKENS_TOKENIZERS_SET = {"roberta", "camembert", "bart"}
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
|
||||
@@ -216,15 +216,6 @@ except ImportError:
|
||||
_tokenizers_available = False
|
||||
|
||||
|
||||
try:
|
||||
import pandas # noqa: F401
|
||||
|
||||
_pandas_available = True
|
||||
|
||||
except ImportError:
|
||||
_pandas_available = False
|
||||
|
||||
|
||||
try:
|
||||
import torch_scatter
|
||||
|
||||
@@ -352,10 +343,6 @@ def is_scatter_available():
|
||||
return _scatter_available
|
||||
|
||||
|
||||
def is_pandas_available():
|
||||
return _pandas_available
|
||||
|
||||
|
||||
def torch_only_method(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
if not _torch_available:
|
||||
@@ -893,7 +880,7 @@ def add_code_sample_docstrings(
|
||||
elif "MaskedLM" in model_class or model_class in ["FlaubertWithLMHeadModel", "XLMWithLMHeadModel"]:
|
||||
doc_kwargs["mask"] = "[MASK]" if mask is None else mask
|
||||
code_sample = TF_MASKED_LM_SAMPLE if is_tf_class else PT_MASKED_LM_SAMPLE
|
||||
elif "LMHead" in model_class or "CausalLM" in model_class:
|
||||
elif "LMHead" in model_class:
|
||||
code_sample = TF_CAUSAL_LM_SAMPLE if is_tf_class else PT_CAUSAL_LM_SAMPLE
|
||||
elif "Model" in model_class or "Encoder" in model_class:
|
||||
code_sample = TF_BASE_MODEL_SAMPLE if is_tf_class else PT_BASE_MODEL_SAMPLE
|
||||
|
||||
@@ -122,12 +122,9 @@ class BeamSearchScorer(BeamScorer):
|
||||
Adapted in part from `Facebook's XLM beam search code
|
||||
<https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529>`__.
|
||||
|
||||
Reference for the diverse beam search algorithm and implementation `Ashwin Kalyan's DBS implementation
|
||||
<https://github.com/ashwinkalyan/dbs/blob/master/dbs/beam_utils.lua>`__
|
||||
|
||||
Args:
|
||||
batch_size (:obj:`int`):
|
||||
Batch Size of :obj:`input_ids` for which standard beam search decoding is run in parallel.
|
||||
Batch Size of :obj:`input_ids` for which beam search decoding is run in parallel.
|
||||
max_length (:obj:`int`):
|
||||
The maximum length of the sequence to be generated.
|
||||
num_beams (:obj:`int`):
|
||||
@@ -144,9 +141,6 @@ class BeamSearchScorer(BeamScorer):
|
||||
num_beam_hyps_to_keep (:obj:`int`, `optional`, defaults to 1):
|
||||
The number of beam hypotheses that shall be returned upon calling
|
||||
:meth:`~transformer.BeamSearchScorer.finalize`.
|
||||
num_beam_groups (:obj:`int`):
|
||||
Number of groups to divide :obj:`num_beams` into in order to ensure diversity among different groups of
|
||||
beams. See `this paper <https://arxiv.org/pdf/1610.02424.pdf>`__ for more details.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -158,7 +152,6 @@ class BeamSearchScorer(BeamScorer):
|
||||
length_penalty: Optional[float] = 1.0,
|
||||
do_early_stopping: Optional[bool] = False,
|
||||
num_beam_hyps_to_keep: Optional[int] = 1,
|
||||
num_beam_groups: Optional[int] = 1,
|
||||
):
|
||||
self.max_length = max_length
|
||||
self.num_beams = num_beams
|
||||
@@ -166,8 +159,6 @@ class BeamSearchScorer(BeamScorer):
|
||||
self.length_penalty = length_penalty
|
||||
self.do_early_stopping = do_early_stopping
|
||||
self.num_beam_hyps_to_keep = num_beam_hyps_to_keep
|
||||
self.num_beam_groups = num_beam_groups
|
||||
self.group_size = self.num_beams // self.num_beam_groups
|
||||
|
||||
self._is_init = False
|
||||
self._beam_hyps = [
|
||||
@@ -186,12 +177,6 @@ class BeamSearchScorer(BeamScorer):
|
||||
f"`num_beams` has to be an integer strictly greater than 1, but is {num_beams}. For `num_beams` == 1, one should make use of `greedy_search` instead."
|
||||
)
|
||||
|
||||
if not isinstance(num_beam_groups, int) or (num_beam_groups > num_beams) or (num_beams % num_beam_groups != 0):
|
||||
raise ValueError(
|
||||
f"`num_beam_groups` has to be an integer smaller or equal than `num_beams` and `num_beams` "
|
||||
f"has to be divisible by `num_beam_groups`, but is {num_beam_groups} with `num_beams` being {num_beams}."
|
||||
)
|
||||
|
||||
@property
|
||||
def is_done(self) -> bool:
|
||||
return self._done.all()
|
||||
@@ -207,12 +192,12 @@ class BeamSearchScorer(BeamScorer):
|
||||
) -> Tuple[torch.Tensor]:
|
||||
cur_len = input_ids.shape[-1]
|
||||
batch_size = len(self._beam_hyps)
|
||||
assert batch_size == (input_ids.shape[0] // self.group_size)
|
||||
assert batch_size == (input_ids.shape[0] // self.num_beams)
|
||||
|
||||
device = input_ids.device
|
||||
next_beam_scores = torch.zeros((batch_size, self.group_size), dtype=next_scores.dtype, device=device)
|
||||
next_beam_tokens = torch.zeros((batch_size, self.group_size), dtype=next_tokens.dtype, device=device)
|
||||
next_beam_indices = torch.zeros((batch_size, self.group_size), dtype=next_indices.dtype, device=device)
|
||||
next_beam_scores = torch.zeros((batch_size, self.num_beams), dtype=next_scores.dtype, device=device)
|
||||
next_beam_tokens = torch.zeros((batch_size, self.num_beams), dtype=next_tokens.dtype, device=device)
|
||||
next_beam_indices = torch.zeros((batch_size, self.num_beams), dtype=next_indices.dtype, device=device)
|
||||
|
||||
for batch_idx, beam_hyp in enumerate(self._beam_hyps):
|
||||
if self._done[batch_idx]:
|
||||
@@ -233,11 +218,11 @@ class BeamSearchScorer(BeamScorer):
|
||||
for beam_token_rank, (next_token, next_score, next_index) in enumerate(
|
||||
zip(next_tokens[batch_idx], next_scores[batch_idx], next_indices[batch_idx])
|
||||
):
|
||||
batch_beam_idx = batch_idx * self.group_size + next_index
|
||||
batch_beam_idx = batch_idx * self.num_beams + next_index
|
||||
# add to generated hypotheses if end of sentence
|
||||
if (eos_token_id is not None) and (next_token.item() == eos_token_id):
|
||||
# if beam_token does not belong to top num_beams tokens, it should not be added
|
||||
is_beam_token_worse_than_top_num_beams = beam_token_rank >= self.group_size
|
||||
is_beam_token_worse_than_top_num_beams = beam_token_rank >= self.num_beams
|
||||
if is_beam_token_worse_than_top_num_beams:
|
||||
continue
|
||||
beam_hyp.add(
|
||||
@@ -252,12 +237,12 @@ class BeamSearchScorer(BeamScorer):
|
||||
beam_idx += 1
|
||||
|
||||
# once the beam for next step is full, don't add more tokens to it.
|
||||
if beam_idx == self.group_size:
|
||||
if beam_idx == self.num_beams:
|
||||
break
|
||||
|
||||
if beam_idx < self.group_size:
|
||||
if beam_idx < self.num_beams:
|
||||
raise ValueError(
|
||||
f"At most {self.group_size} tokens in {next_tokens[batch_idx]} can be equal to `eos_token_id: {eos_token_id}`. Make sure {next_tokens[batch_idx]} are corrected."
|
||||
f"At most {self.num_beams} tokens in {next_tokens[batch_idx]} can be equal to `eos_token_id: {eos_token_id}`. Make sure {next_tokens[batch_idx]} are corrected."
|
||||
)
|
||||
|
||||
# Check if we are done so that we can save a pad step if all(done)
|
||||
@@ -289,8 +274,7 @@ class BeamSearchScorer(BeamScorer):
|
||||
if self._done[batch_idx]:
|
||||
continue
|
||||
|
||||
# all open beam hypotheses are added to the beam hypothesis
|
||||
# beam hypothesis class automatically keeps the best beams
|
||||
# need to add best num_beams hypotheses to generated hyps
|
||||
for beam_id in range(self.num_beams):
|
||||
batch_beam_idx = batch_idx * self.num_beams + beam_id
|
||||
final_score = final_beam_scores[batch_beam_idx].item()
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import inspect
|
||||
import math
|
||||
from abc import ABC
|
||||
from typing import Callable, Iterable, List
|
||||
@@ -38,8 +37,6 @@ LOGITS_PROCESSOR_INPUTS_DOCSTRING = r"""
|
||||
scores (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.vocab_size)`):
|
||||
Prediction scores of a language modeling head. These can be scores for each vocabulary token before SoftMax
|
||||
or scores for each vocabulary token after SoftMax.
|
||||
kwargs:
|
||||
Additional logits processor specific kwargs.
|
||||
|
||||
Return:
|
||||
:obj:`torch.FloatTensor` of shape :obj:`(batch_size, config.vocab_size)`: The processed prediction scores.
|
||||
@@ -78,16 +75,9 @@ class LogitsProcessorList(list):
|
||||
"""
|
||||
|
||||
@add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
|
||||
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> torch.FloatTensor:
|
||||
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
|
||||
for processor in self:
|
||||
function_args = inspect.signature(processor.__call__).parameters
|
||||
if len(function_args) > 2:
|
||||
assert all(
|
||||
arg in kwargs for arg in list(function_args.keys())[2:]
|
||||
), f"Make sure that all the required parameters: {list(function_args.keys())} for {processor.__class__} are passed to the logits processor."
|
||||
scores = processor(input_ids, scores, **kwargs)
|
||||
else:
|
||||
scores = processor(input_ids, scores)
|
||||
scores = processor(input_ids, scores)
|
||||
return scores
|
||||
|
||||
|
||||
@@ -410,65 +400,3 @@ class PrefixConstrainedLogitsProcessor(LogitsProcessor):
|
||||
mask[batch_id * self._num_beams + beam_id, self._prefix_allowed_tokens_fn(batch_id, sent)] = 0
|
||||
|
||||
return scores + mask
|
||||
|
||||
|
||||
class HammingDiversityLogitsProcessor(LogitsProcessor):
|
||||
r"""
|
||||
:class:`transformers.LogitsProcessor` that enforces diverse beam search. Note that this logits processor is only
|
||||
effective for :meth:`transformers.PretrainedModel.group_beam_search`. See `Diverse Beam Search: Decoding Diverse
|
||||
Solutions from Neural Sequence Models <https://arxiv.org/pdf/1610.02424.pdf>`__ for more details.
|
||||
|
||||
Args:
|
||||
diversity_penalty (:obj:`float`):
|
||||
This value is subtracted from a beam's score if it generates a token same as any beam from other group at a
|
||||
particular time. Note that :obj:`diversity_penalty` is only effective if ``group beam search`` is enabled.
|
||||
num_beams (:obj:`int`):
|
||||
Number of beams used for group beam search. See `this paper <https://arxiv.org/pdf/1610.02424.pdf>`__ for
|
||||
more details.
|
||||
num_beam_groups (:obj:`int`):
|
||||
Number of groups to divide :obj:`num_beams` into in order to ensure diversity among different groups of
|
||||
beams. See `this paper <https://arxiv.org/pdf/1610.02424.pdf>`__ for more details.
|
||||
"""
|
||||
|
||||
def __init__(self, diversity_penalty: float, num_beams: int, num_beam_groups: int):
|
||||
if not isinstance(diversity_penalty, float) or (not diversity_penalty > 0.0):
|
||||
raise ValueError("`diversity_penalty` should be a float strictly larger than 0.")
|
||||
self._diversity_penalty = diversity_penalty
|
||||
if not isinstance(num_beams, int) or num_beams < 2:
|
||||
raise ValueError("`num_beams` should be an integer strictly larger than 1.")
|
||||
self._num_beams = num_beams
|
||||
if not isinstance(num_beam_groups, int) or num_beam_groups < 2:
|
||||
raise ValueError("`num_beam_groups` should be an integer strictly larger than 1.")
|
||||
if num_beam_groups > num_beams:
|
||||
raise ValueError("`beam_groups` has to be smaller or equal to `num_beams`.")
|
||||
if num_beam_groups > num_beams:
|
||||
raise ValueError("`beam_groups` has to be smaller or equal to `num_beams`")
|
||||
self._num_sub_beams = num_beams // num_beam_groups
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
input_ids: torch.LongTensor,
|
||||
scores: torch.FloatTensor,
|
||||
current_tokens: torch.LongTensor,
|
||||
beam_group_idx: int,
|
||||
) -> torch.FloatTensor:
|
||||
# hamming diversity: penalise using same token in current group which was used in previous groups at
|
||||
# the same time step
|
||||
batch_size = current_tokens.shape[0] // self._num_beams
|
||||
group_start_idx = beam_group_idx * self._num_sub_beams
|
||||
group_end_idx = min(group_start_idx + self._num_sub_beams, self._num_beams)
|
||||
group_size = group_end_idx - group_start_idx
|
||||
vocab_size = scores.shape[-1]
|
||||
|
||||
if group_start_idx == 0:
|
||||
return scores
|
||||
|
||||
for batch_idx in range(batch_size):
|
||||
# predicted tokens of last time step of previous groups
|
||||
previous_group_tokens = current_tokens[
|
||||
batch_idx * self._num_beams : batch_idx * self._num_beams + group_start_idx
|
||||
]
|
||||
token_frequency = torch.bincount(previous_group_tokens, minlength=vocab_size).to(scores.device)
|
||||
scores[batch_idx * group_size : (batch_idx + 1) * group_size] -= self._diversity_penalty * token_frequency
|
||||
|
||||
return scores
|
||||
@@ -22,7 +22,6 @@ from torch.nn import functional as F
|
||||
from .file_utils import ModelOutput
|
||||
from .generation_beam_search import BeamScorer, BeamSearchScorer
|
||||
from .generation_logits_process import (
|
||||
HammingDiversityLogitsProcessor,
|
||||
LogitsProcessorList,
|
||||
MinLengthLogitsProcessor,
|
||||
NoBadWordsLogitsProcessor,
|
||||
@@ -262,8 +261,6 @@ class GenerationMixin:
|
||||
eos_token_id: int,
|
||||
prefix_allowed_tokens_fn: Callable[[int, torch.Tensor], List[int]],
|
||||
num_beams: int,
|
||||
num_beam_groups: int,
|
||||
diversity_penalty: float,
|
||||
) -> LogitsProcessorList:
|
||||
"""
|
||||
This class returns a :obj:`~transformers.LogitsProcessorList` list object that contains all relevant
|
||||
@@ -278,18 +275,11 @@ class GenerationMixin:
|
||||
bad_words_ids = bad_words_ids if bad_words_ids is not None else self.config.bad_words_ids
|
||||
min_length = min_length if min_length is not None else self.config.min_length
|
||||
eos_token_id = eos_token_id if eos_token_id is not None else self.config.eos_token_id
|
||||
diversity_penalty = diversity_penalty if diversity_penalty is not None else self.config.diversity_penalty
|
||||
# instantiate processors list
|
||||
processors = LogitsProcessorList()
|
||||
|
||||
# the following idea is largely copied from this PR: https://github.com/huggingface/transformers/pull/5420/files
|
||||
# all samplers can be found in `generation_utils_samplers.py`
|
||||
if diversity_penalty is not None and diversity_penalty > 0.0:
|
||||
processors.append(
|
||||
HammingDiversityLogitsProcessor(
|
||||
diversity_penalty=diversity_penalty, num_beams=num_beams, num_beam_groups=num_beam_groups
|
||||
)
|
||||
)
|
||||
if repetition_penalty is not None and repetition_penalty != 1.0:
|
||||
processors.append(RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty))
|
||||
if no_repeat_ngram_size is not None and no_repeat_ngram_size > 0:
|
||||
@@ -324,8 +314,6 @@ class GenerationMixin:
|
||||
num_return_sequences: Optional[int] = None,
|
||||
decoder_start_token_id: Optional[int] = None,
|
||||
use_cache: Optional[bool] = None,
|
||||
num_beam_groups: Optional[int] = None,
|
||||
diversity_penalty: Optional[float] = None,
|
||||
prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None,
|
||||
**model_kwargs
|
||||
) -> torch.LongTensor:
|
||||
@@ -393,13 +381,6 @@ class GenerationMixin:
|
||||
use_cache: (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not the model should use the past last key/values attentions (if applicable to the model) to
|
||||
speed up decoding.
|
||||
num_beam_groups (:obj:`int`, `optional`, defaults to 1):
|
||||
Number of groups to divide :obj:`num_beams` into in order to ensure diversity among different groups of
|
||||
beams. `this paper <https://arxiv.org/pdf/1610.02424.pdf>`__ for more details.
|
||||
diversity_penalty (:obj:`float`, `optional`, defaults to 0.0):
|
||||
This value is subtracted from a beam's score if it generates a token same as any beam from other group
|
||||
at a particular time. Note that :obj:`diversity_penalty` is only effective if ``group beam search`` is
|
||||
enabled.
|
||||
prefix_allowed_tokens_fn: (:obj:`Callable[[int, torch.Tensor], List[int]]`, `optional`):
|
||||
If provided, this function constraints the beam search to allowed tokens only at each step. If not
|
||||
provided no constraint is applied. This function takes 2 arguments :obj:`inputs_ids` and the batch ID
|
||||
@@ -472,7 +453,6 @@ class GenerationMixin:
|
||||
|
||||
# set init values
|
||||
num_beams = num_beams if num_beams is not None else self.config.num_beams
|
||||
num_beam_groups = num_beam_groups if num_beam_groups is not None else self.config.num_beam_groups
|
||||
max_length = max_length if max_length is not None else self.config.max_length
|
||||
do_sample = do_sample if do_sample is not None else self.config.do_sample
|
||||
num_return_sequences = (
|
||||
@@ -511,17 +491,10 @@ class GenerationMixin:
|
||||
raise ValueError("Make sure that `model_kwargs` include `encoder_outputs` of type `ModelOutput`.")
|
||||
|
||||
# determine generation mode
|
||||
is_greedy_gen_mode = (num_beams == 1) and (num_beam_groups == 1) and do_sample is False
|
||||
is_sample_gen_mode = (num_beams == 1) and (num_beam_groups == 1) and do_sample is True
|
||||
is_beam_gen_mode = (num_beams > 1) and (num_beam_groups == 1) and do_sample is False
|
||||
is_beam_sample_gen_mode = (num_beams > 1) and (num_beam_groups == 1) and do_sample is True
|
||||
is_group_beam_gen_mode = (num_beams > 1) and (num_beam_groups > 1)
|
||||
if num_beam_groups > num_beams:
|
||||
raise ValueError("`num_beam_groups` has to be smaller or equal to `num_beams`")
|
||||
if is_group_beam_gen_mode and do_sample is True:
|
||||
raise ValueError(
|
||||
"Diverse beam search cannot be used in sampling mode. Make sure that `do_sample` is set to `False`."
|
||||
)
|
||||
is_greedy_gen_mode = (num_beams == 1) and do_sample is False
|
||||
is_sample_gen_mode = (num_beams == 1) and do_sample is True
|
||||
is_beam_gen_mode = (num_beams > 1) and do_sample is False
|
||||
is_beam_sample_gen_mode = (num_beams > 1) and do_sample is True
|
||||
|
||||
# set model_kwargs
|
||||
model_kwargs["use_cache"] = use_cache
|
||||
@@ -535,8 +508,6 @@ class GenerationMixin:
|
||||
eos_token_id=eos_token_id,
|
||||
prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
|
||||
num_beams=num_beams,
|
||||
num_beam_groups=num_beam_groups,
|
||||
diversity_penalty=diversity_penalty,
|
||||
)
|
||||
|
||||
if is_greedy_gen_mode:
|
||||
@@ -648,42 +619,6 @@ class GenerationMixin:
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
elif is_group_beam_gen_mode:
|
||||
batch_size = input_ids.shape[0]
|
||||
|
||||
length_penalty = length_penalty if length_penalty is not None else self.config.length_penalty
|
||||
early_stopping = early_stopping if early_stopping is not None else self.config.early_stopping
|
||||
|
||||
if num_return_sequences > num_beams:
|
||||
raise ValueError("`num_return_sequences` has to be smaller or equal to `num_beams`.")
|
||||
|
||||
if num_beams % num_beam_groups != 0:
|
||||
raise ValueError("`num_beams` should be divisible by `num_beam_groups` for group beam search.")
|
||||
|
||||
diverse_beam_scorer = BeamSearchScorer(
|
||||
batch_size=batch_size,
|
||||
max_length=max_length,
|
||||
num_beams=num_beams,
|
||||
device=self.device,
|
||||
length_penalty=length_penalty,
|
||||
do_early_stopping=early_stopping,
|
||||
num_beam_hyps_to_keep=num_return_sequences,
|
||||
num_beam_groups=num_beam_groups,
|
||||
)
|
||||
# interleave with `num_beams`
|
||||
input_ids, model_kwargs = self._expand_inputs_for_generation(
|
||||
input_ids, expand_size=num_beams, is_encoder_decoder=self.config.is_encoder_decoder, **model_kwargs
|
||||
)
|
||||
return self.group_beam_search(
|
||||
input_ids,
|
||||
diverse_beam_scorer,
|
||||
logits_processor=logits_processor,
|
||||
max_length=max_length,
|
||||
pad_token_id=pad_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
def greedy_search(
|
||||
self,
|
||||
input_ids: torch.LongTensor,
|
||||
@@ -1273,213 +1208,6 @@ class GenerationMixin:
|
||||
|
||||
return decoded
|
||||
|
||||
def group_beam_search(
|
||||
self,
|
||||
input_ids: torch.LongTensor,
|
||||
beam_scorer: BeamScorer,
|
||||
logits_processor: Optional[LogitsProcessorList] = None,
|
||||
max_length: Optional[int] = None,
|
||||
pad_token_id: Optional[int] = None,
|
||||
eos_token_id: Optional[int] = None,
|
||||
**model_kwargs
|
||||
):
|
||||
r"""
|
||||
Generates sequences for models with a language modeling head using beam search decoding.
|
||||
|
||||
Parameters:
|
||||
|
||||
input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
|
||||
The sequence used as a prompt for the generation. If :obj:`None` the method initializes it as an empty
|
||||
:obj:`torch.LongTensor` of shape :obj:`(1,)`.
|
||||
beam_scorer (:obj:`BeamScorer`):
|
||||
An derived instance of :class:`~transformers.BeamScorer` that defines how beam hypotheses are
|
||||
constructed, stored and sorted during generation. For more information, the documentation of
|
||||
:class:`~transformers.BeamScorer` should be read.
|
||||
logits_processor (:obj:`LogitsProcessorList`, `optional`):
|
||||
An instance of :class:`~transformers.LogitsProcessorList`. List of instances of class derived from
|
||||
:class:`~transformers.LogitsProcessor` used to modify the prediction scores of the language modeling
|
||||
head applied at each generation step.
|
||||
max_length (:obj:`int`, `optional`, defaults to 20):
|
||||
The maximum length of the sequence to be generated.
|
||||
pad_token_id (:obj:`int`, `optional`):
|
||||
The id of the `padding` token.
|
||||
eos_token_id (:obj:`int`, `optional`):
|
||||
The id of the `end-of-sequence` token.
|
||||
model_kwargs:
|
||||
Additional model specific kwargs that will be forwarded to the :obj:`forward` function of the model. If
|
||||
model is an encoder-decoder model the kwargs should include :obj:`encoder_outputs`.
|
||||
|
||||
Return:
|
||||
:obj:`torch.LongTensor` of shape :obj:`(batch_size * num_return_sequences, sequence_length)`: The generated
|
||||
sequences. The second dimension (sequence_length) is either equal to :obj:`max_length` or shorter if all
|
||||
batches finished early due to the :obj:`eos_token_id`.
|
||||
|
||||
Examples::
|
||||
|
||||
>>> from transformers import (
|
||||
... AutoTokenizer,
|
||||
... AutoModelForSeq2SeqLM,
|
||||
... LogitsProcessorList,
|
||||
... MinLengthLogitsProcessor,
|
||||
... HammingDiversityLogitsProcessor,
|
||||
... BeamSearchScorer,
|
||||
... )
|
||||
>>> import torch
|
||||
|
||||
>>> tokenizer = AutoTokenizer.from_pretrained("t5-base")
|
||||
>>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
|
||||
|
||||
>>> encoder_input_str = "translate English to German: How old are you?"
|
||||
>>> encoder_input_ids = tokenizer(encoder_input_str, return_tensors="pt").input_ids
|
||||
|
||||
|
||||
>>> # lets run diverse beam search using 6 beams
|
||||
>>> num_beams = 6
|
||||
>>> # define decoder start token ids
|
||||
>>> input_ids = torch.ones((num_beams, 1), device=model.device, dtype=torch.long)
|
||||
>>> input_ids = input_ids * model.config.decoder_start_token_id
|
||||
|
||||
>>> # add encoder_outputs to model keyword arguments
|
||||
>>> model_kwargs = {
|
||||
... "encoder_outputs": model.get_encoder()(encoder_input_ids.repeat_interleave(num_beams, dim=0), return_dict=True)
|
||||
... }
|
||||
|
||||
>>> # instantiate beam scorer
|
||||
>>> beam_scorer = BeamSearchScorer(
|
||||
... batch_size=1,
|
||||
... max_length=model.config.max_length,
|
||||
... num_beams=num_beams,
|
||||
... device=model.device,
|
||||
... num_beam_groups=3
|
||||
... )
|
||||
|
||||
>>> # instantiate logits processors
|
||||
>>> logits_processor = LogitsProcessorList([
|
||||
... HammingDiversityLogitsProcessor(5.5, num_beams=6, num_beam_groups=3),
|
||||
... MinLengthLogitsProcessor(5, eos_token_id=model.config.eos_token_id),
|
||||
... ])
|
||||
|
||||
>>> outputs = model.group_beam_search(input_ids, beam_scorer, logits_processor=logits_processor, **model_kwargs)
|
||||
|
||||
>>> print("Generated:", tokenizer.batch_decode(outputs, skip_special_tokens=True))
|
||||
"""
|
||||
|
||||
# init values
|
||||
logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
|
||||
max_length = max_length if max_length is not None else self.config.max_length
|
||||
pad_token_id = pad_token_id if pad_token_id is not None else self.config.pad_token_id
|
||||
eos_token_id = eos_token_id if eos_token_id is not None else self.config.eos_token_id
|
||||
|
||||
batch_size = len(beam_scorer._beam_hyps)
|
||||
num_beams = beam_scorer.num_beams
|
||||
num_beam_groups = beam_scorer.num_beam_groups
|
||||
num_sub_beams = num_beams // num_beam_groups
|
||||
device = input_ids.device
|
||||
|
||||
batch_beam_size, cur_len = input_ids.shape
|
||||
|
||||
assert (
|
||||
num_beams * batch_size == batch_beam_size
|
||||
), f"Batch dimension of `input_ids` should be {num_beams * batch_size}, but is {batch_beam_size}."
|
||||
|
||||
beam_scores = torch.full((batch_size, num_beams), -1e9, dtype=torch.float, device=device)
|
||||
# initialise score of first beam of each group with 0 and the rest with 1e-9. This ensures that the beams in
|
||||
# the same group don't produce same tokens everytime.
|
||||
beam_scores[:, ::num_sub_beams] = 0
|
||||
beam_scores = beam_scores.view((batch_size * num_beams,))
|
||||
|
||||
while cur_len < max_length:
|
||||
# predicted tokens in cur_len step
|
||||
current_tokens = torch.zeros(batch_size * num_beams, dtype=input_ids.dtype, device=device)
|
||||
|
||||
# indices which will form the beams in the next time step
|
||||
reordering_indices = torch.zeros(batch_size * num_beams, dtype=torch.long, device=device)
|
||||
|
||||
# do one decoder step on all beams of all sentences in batch
|
||||
model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
|
||||
outputs = self(**model_inputs, return_dict=True)
|
||||
|
||||
for beam_group_idx in range(num_beam_groups):
|
||||
group_start_idx = beam_group_idx * num_sub_beams
|
||||
group_end_idx = min(group_start_idx + num_sub_beams, num_beams)
|
||||
group_size = group_end_idx - group_start_idx
|
||||
|
||||
# indices of beams of current group among all sentences in batch
|
||||
batch_group_indices = []
|
||||
for batch_idx in range(batch_size):
|
||||
batch_group_indices.extend(
|
||||
[batch_idx * num_beams + idx for idx in range(group_start_idx, group_end_idx)]
|
||||
)
|
||||
group_input_ids = input_ids[batch_group_indices]
|
||||
|
||||
# select outputs of beams of current group only
|
||||
next_token_logits = outputs.logits[batch_group_indices, -1, :]
|
||||
|
||||
# adjust tokens for Bart, *e.g.*
|
||||
next_token_logits = self.adjust_logits_during_generation(
|
||||
next_token_logits, cur_len=cur_len, max_length=max_length
|
||||
)
|
||||
|
||||
next_token_scores = F.log_softmax(next_token_logits, dim=-1) # (batch_size * group_size, vocab_size)
|
||||
vocab_size = next_token_scores.shape[-1]
|
||||
|
||||
next_token_scores = logits_processor(
|
||||
group_input_ids, next_token_scores, current_tokens=current_tokens, beam_group_idx=beam_group_idx
|
||||
)
|
||||
next_token_scores = next_token_scores + beam_scores[batch_group_indices].unsqueeze(-1).expand_as(
|
||||
next_token_scores
|
||||
)
|
||||
# reshape for beam search
|
||||
|
||||
next_token_scores = next_token_scores.view(batch_size, group_size * vocab_size)
|
||||
|
||||
next_token_scores, next_tokens = torch.topk(
|
||||
next_token_scores, 2 * group_size, dim=1, largest=True, sorted=True
|
||||
)
|
||||
|
||||
next_indices = next_tokens // vocab_size
|
||||
next_tokens = next_tokens % vocab_size
|
||||
|
||||
# stateless
|
||||
beam_outputs = beam_scorer.process(
|
||||
group_input_ids,
|
||||
next_token_scores,
|
||||
next_tokens,
|
||||
next_indices,
|
||||
pad_token_id=pad_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
)
|
||||
beam_scores[batch_group_indices] = beam_outputs["next_beam_scores"]
|
||||
beam_next_tokens = beam_outputs["next_beam_tokens"]
|
||||
beam_idx = beam_outputs["next_beam_indices"]
|
||||
|
||||
input_ids[batch_group_indices] = group_input_ids[beam_idx]
|
||||
group_input_ids = torch.cat([group_input_ids[beam_idx, :], beam_next_tokens.unsqueeze(-1)], dim=-1)
|
||||
current_tokens[batch_group_indices] = group_input_ids[:, -1]
|
||||
|
||||
# (beam_idx // group_size) -> batch_idx
|
||||
# (beam_idx % group_size) -> offset of idx inside the group
|
||||
reordering_indices[batch_group_indices] = (
|
||||
num_beams * (beam_idx // group_size) + group_start_idx + (beam_idx % group_size)
|
||||
)
|
||||
|
||||
model_kwargs = self._update_model_kwargs_for_generation(
|
||||
outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder
|
||||
)
|
||||
if model_kwargs["past"] is not None:
|
||||
model_kwargs["past"] = self._reorder_cache(model_kwargs["past"], reordering_indices)
|
||||
|
||||
input_ids = torch.cat([input_ids, current_tokens.unsqueeze(-1)], dim=-1)
|
||||
cur_len = cur_len + 1
|
||||
if beam_scorer.is_done:
|
||||
break
|
||||
|
||||
decoded = beam_scorer.finalize(
|
||||
input_ids, beam_scores, next_tokens, next_indices, pad_token_id=pad_token_id, eos_token_id=eos_token_id
|
||||
)
|
||||
|
||||
return decoded
|
||||
|
||||
|
||||
def top_k_top_p_filtering(
|
||||
logits: torch.FloatTensor,
|
||||
|
||||
@@ -65,12 +65,13 @@ class FlaxPreTrainedModel(ABC):
|
||||
base_model_prefix = ""
|
||||
model_class = None
|
||||
|
||||
def __init__(
|
||||
self, config: PretrainedConfig, module: nn.Module, params: Dict, seed: int = 0, dtype: jnp.dtype = jnp.float32
|
||||
):
|
||||
def __init__(self, config: PretrainedConfig, module: nn.Module, params: Dict, seed: int = 0):
|
||||
if config is None:
|
||||
raise ValueError("config cannot be None")
|
||||
|
||||
if module is None:
|
||||
raise ValueError("module cannot be None")
|
||||
|
||||
if params is None:
|
||||
raise ValueError("state cannot be None")
|
||||
|
||||
@@ -81,23 +82,19 @@ class FlaxPreTrainedModel(ABC):
|
||||
# Those are public as their type is generic to every derived classes.
|
||||
self.key = PRNGKey(seed)
|
||||
self.params = params
|
||||
self.dtype = dtype
|
||||
self.model = module
|
||||
|
||||
@property
|
||||
def config(self) -> PretrainedConfig:
|
||||
return self._config
|
||||
|
||||
@property
|
||||
def module(self) -> nn.Module:
|
||||
return self._module
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def convert_from_pytorch(pt_state: Dict, config: PretrainedConfig) -> Dict:
|
||||
raise NotImplementedError()
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, pretrained_model_name_or_path, dtype: jnp.dtype = jnp.float32, *model_args, **kwargs):
|
||||
def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
|
||||
r"""
|
||||
Instantiate a pretrained Flax model from a pre-trained model configuration.
|
||||
"""
|
||||
@@ -130,9 +127,6 @@ class FlaxPreTrainedModel(ABC):
|
||||
else:
|
||||
model_kwargs = kwargs
|
||||
|
||||
# Add the dtype to model_kwargs
|
||||
model_kwargs["dtype"] = dtype
|
||||
|
||||
# Load model
|
||||
if pretrained_model_name_or_path is not None:
|
||||
if os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
|
||||
|
||||
@@ -41,7 +41,7 @@ class BaseModelOutput(ModelOutput):
|
||||
heads.
|
||||
"""
|
||||
|
||||
last_hidden_state: torch.FloatTensor = None
|
||||
last_hidden_state: torch.FloatTensor
|
||||
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
||||
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
|
||||
@@ -71,7 +71,7 @@ class BaseModelOutputWithPooling(ModelOutput):
|
||||
heads.
|
||||
"""
|
||||
|
||||
last_hidden_state: torch.FloatTensor = None
|
||||
last_hidden_state: torch.FloatTensor
|
||||
pooler_output: torch.FloatTensor = None
|
||||
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
||||
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
@@ -107,7 +107,7 @@ class BaseModelOutputWithPast(ModelOutput):
|
||||
heads.
|
||||
"""
|
||||
|
||||
last_hidden_state: torch.FloatTensor = None
|
||||
last_hidden_state: torch.FloatTensor
|
||||
past_key_values: Optional[List[torch.FloatTensor]] = None
|
||||
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
||||
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
@@ -140,7 +140,7 @@ class BaseModelOutputWithCrossAttentions(ModelOutput):
|
||||
weighted average in the cross-attention heads.
|
||||
"""
|
||||
|
||||
last_hidden_state: torch.FloatTensor = None
|
||||
last_hidden_state: torch.FloatTensor
|
||||
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
||||
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
cross_attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
@@ -177,7 +177,7 @@ class BaseModelOutputWithPoolingAndCrossAttentions(ModelOutput):
|
||||
weighted average in the cross-attention heads.
|
||||
"""
|
||||
|
||||
last_hidden_state: torch.FloatTensor = None
|
||||
last_hidden_state: torch.FloatTensor
|
||||
pooler_output: torch.FloatTensor = None
|
||||
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
||||
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
@@ -220,7 +220,7 @@ class BaseModelOutputWithPastAndCrossAttentions(ModelOutput):
|
||||
weighted average in the cross-attention heads.
|
||||
"""
|
||||
|
||||
last_hidden_state: torch.FloatTensor = None
|
||||
last_hidden_state: torch.FloatTensor
|
||||
past_key_values: Optional[List[torch.FloatTensor]] = None
|
||||
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
||||
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
@@ -277,7 +277,7 @@ class Seq2SeqModelOutput(ModelOutput):
|
||||
self-attention heads.
|
||||
"""
|
||||
|
||||
last_hidden_state: torch.FloatTensor = None
|
||||
last_hidden_state: torch.FloatTensor
|
||||
past_key_values: Optional[List[torch.FloatTensor]] = None
|
||||
decoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
||||
decoder_attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
@@ -310,7 +310,7 @@ class CausalLMOutput(ModelOutput):
|
||||
heads.
|
||||
"""
|
||||
|
||||
loss: Optional[torch.FloatTensor] = None
|
||||
loss: Optional[torch.FloatTensor]
|
||||
logits: torch.FloatTensor = None
|
||||
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
||||
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
@@ -381,7 +381,7 @@ class CausalLMOutputWithCrossAttentions(ModelOutput):
|
||||
cross-attention heads.
|
||||
"""
|
||||
|
||||
loss: Optional[torch.FloatTensor] = None
|
||||
loss: Optional[torch.FloatTensor]
|
||||
logits: torch.FloatTensor = None
|
||||
hidden_states: Optional[Tuple[torch.FloatTensor]] = None
|
||||
attentions: Optional[Tuple[torch.FloatTensor]] = None
|
||||
|
||||
@@ -40,7 +40,6 @@ from ..lxmert.configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
||||
from ..marian.configuration_marian import MarianConfig
|
||||
from ..mbart.configuration_mbart import MBART_PRETRAINED_CONFIG_ARCHIVE_MAP, MBartConfig
|
||||
from ..mobilebert.configuration_mobilebert import MobileBertConfig
|
||||
from ..mpnet.configuration_mpnet import MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP, MPNetConfig
|
||||
from ..mt5.configuration_mt5 import MT5Config
|
||||
from ..openai.configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
|
||||
from ..pegasus.configuration_pegasus import PegasusConfig
|
||||
@@ -95,7 +94,6 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict(
|
||||
SQUEEZEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
||||
XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
||||
PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
||||
MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
||||
TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
||||
]
|
||||
for key, value, in pretrained_map.items()
|
||||
@@ -117,7 +115,6 @@ CONFIG_MAPPING = OrderedDict(
|
||||
("pegasus", PegasusConfig),
|
||||
("marian", MarianConfig),
|
||||
("mbart", MBartConfig),
|
||||
("mpnet", MPNetConfig),
|
||||
("bart", BartConfig),
|
||||
("blenderbot", BlenderbotConfig),
|
||||
("reformer", ReformerConfig),
|
||||
@@ -187,7 +184,6 @@ MODEL_NAMES_MAPPING = OrderedDict(
|
||||
("xlm-prophetnet", "XLMProphetNet"),
|
||||
("prophetnet", "ProphetNet"),
|
||||
("mt5", "mT5"),
|
||||
("mpnet", "MPNet"),
|
||||
("tapas", "TAPAS"),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -122,14 +122,6 @@ from ..mobilebert.modeling_mobilebert import (
|
||||
MobileBertForTokenClassification,
|
||||
MobileBertModel,
|
||||
)
|
||||
from ..mpnet.modeling_mpnet import (
|
||||
MPNetForMaskedLM,
|
||||
MPNetForMultipleChoice,
|
||||
MPNetForQuestionAnswering,
|
||||
MPNetForSequenceClassification,
|
||||
MPNetForTokenClassification,
|
||||
MPNetModel,
|
||||
)
|
||||
from ..mt5.modeling_mt5 import MT5ForConditionalGeneration, MT5Model
|
||||
from ..openai.modeling_openai import OpenAIGPTForSequenceClassification, OpenAIGPTLMHeadModel, OpenAIGPTModel
|
||||
from ..pegasus.modeling_pegasus import PegasusForConditionalGeneration
|
||||
@@ -166,10 +158,10 @@ from ..squeezebert.modeling_squeezebert import (
|
||||
)
|
||||
from ..t5.modeling_t5 import T5ForConditionalGeneration, T5Model
|
||||
from ..tapas.modeling_tapas import (
|
||||
TapasModel,
|
||||
TapasForMaskedLM,
|
||||
TapasForQuestionAnswering,
|
||||
TapasForSequenceClassification,
|
||||
TapasModel,
|
||||
)
|
||||
from ..transfo_xl.modeling_transfo_xl import TransfoXLForSequenceClassification, TransfoXLLMHeadModel, TransfoXLModel
|
||||
from ..xlm.modeling_xlm import (
|
||||
@@ -226,7 +218,6 @@ from .configuration_auto import (
|
||||
MarianConfig,
|
||||
MBartConfig,
|
||||
MobileBertConfig,
|
||||
MPNetConfig,
|
||||
MT5Config,
|
||||
OpenAIGPTConfig,
|
||||
PegasusConfig,
|
||||
@@ -283,7 +274,6 @@ MODEL_MAPPING = OrderedDict(
|
||||
(DPRConfig, DPRQuestionEncoder),
|
||||
(XLMProphetNetConfig, XLMProphetNetModel),
|
||||
(ProphetNetConfig, ProphetNetModel),
|
||||
(MPNetConfig, MPNetModel),
|
||||
(TapasConfig, TapasModel),
|
||||
]
|
||||
)
|
||||
@@ -315,7 +305,6 @@ MODEL_FOR_PRETRAINING_MAPPING = OrderedDict(
|
||||
(ElectraConfig, ElectraForPreTraining),
|
||||
(LxmertConfig, LxmertForPreTraining),
|
||||
(FunnelConfig, FunnelForPreTraining),
|
||||
(MPNetConfig, MPNetForMaskedLM),
|
||||
(TapasConfig, TapasForMaskedLM),
|
||||
]
|
||||
)
|
||||
@@ -348,7 +337,6 @@ MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
|
||||
(EncoderDecoderConfig, EncoderDecoderModel),
|
||||
(ReformerConfig, ReformerModelWithLMHead),
|
||||
(FunnelConfig, FunnelForMaskedLM),
|
||||
(MPNetConfig, MPNetForMaskedLM),
|
||||
(TapasConfig, TapasForMaskedLM),
|
||||
]
|
||||
)
|
||||
@@ -395,7 +383,6 @@ MODEL_FOR_MASKED_LM_MAPPING = OrderedDict(
|
||||
(ElectraConfig, ElectraForMaskedLM),
|
||||
(ReformerConfig, ReformerForMaskedLM),
|
||||
(FunnelConfig, FunnelForMaskedLM),
|
||||
(MPNetConfig, MPNetForMaskedLM),
|
||||
(TapasConfig, TapasForMaskedLM),
|
||||
]
|
||||
)
|
||||
@@ -441,7 +428,6 @@ MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
|
||||
(ReformerConfig, ReformerForSequenceClassification),
|
||||
(CTRLConfig, CTRLForSequenceClassification),
|
||||
(TransfoXLConfig, TransfoXLForSequenceClassification),
|
||||
(MPNetConfig, MPNetForSequenceClassification),
|
||||
(TapasConfig, TapasForSequenceClassification),
|
||||
]
|
||||
)
|
||||
@@ -466,7 +452,6 @@ MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
|
||||
(ReformerConfig, ReformerForQuestionAnswering),
|
||||
(FunnelConfig, FunnelForQuestionAnswering),
|
||||
(LxmertConfig, LxmertForQuestionAnswering),
|
||||
(MPNetConfig, MPNetForQuestionAnswering),
|
||||
(TapasConfig, TapasForQuestionAnswering),
|
||||
]
|
||||
)
|
||||
@@ -490,7 +475,6 @@ MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = OrderedDict(
|
||||
(ElectraConfig, ElectraForTokenClassification),
|
||||
(FlaubertConfig, FlaubertForTokenClassification),
|
||||
(FunnelConfig, FunnelForTokenClassification),
|
||||
(MPNetConfig, MPNetForTokenClassification),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -511,7 +495,6 @@ MODEL_FOR_MULTIPLE_CHOICE_MAPPING = OrderedDict(
|
||||
(XLMConfig, XLMForMultipleChoice),
|
||||
(FlaubertConfig, FlaubertForMultipleChoice),
|
||||
(FunnelConfig, FunnelForMultipleChoice),
|
||||
(MPNetConfig, MPNetForMultipleChoice),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -111,14 +111,6 @@ from ..mobilebert.modeling_tf_mobilebert import (
|
||||
TFMobileBertForTokenClassification,
|
||||
TFMobileBertModel,
|
||||
)
|
||||
from ..mpnet.modeling_tf_mpnet import (
|
||||
TFMPNetForMaskedLM,
|
||||
TFMPNetForMultipleChoice,
|
||||
TFMPNetForQuestionAnswering,
|
||||
TFMPNetForSequenceClassification,
|
||||
TFMPNetForTokenClassification,
|
||||
TFMPNetModel,
|
||||
)
|
||||
from ..mt5.modeling_tf_mt5 import TFMT5ForConditionalGeneration, TFMT5Model
|
||||
from ..openai.modeling_tf_openai import TFOpenAIGPTLMHeadModel, TFOpenAIGPTModel
|
||||
from ..pegasus.modeling_tf_pegasus import TFPegasusForConditionalGeneration
|
||||
@@ -175,7 +167,6 @@ from .configuration_auto import (
|
||||
MarianConfig,
|
||||
MBartConfig,
|
||||
MobileBertConfig,
|
||||
MPNetConfig,
|
||||
MT5Config,
|
||||
OpenAIGPTConfig,
|
||||
PegasusConfig,
|
||||
@@ -217,7 +208,6 @@ TF_MODEL_MAPPING = OrderedDict(
|
||||
(ElectraConfig, TFElectraModel),
|
||||
(FunnelConfig, TFFunnelModel),
|
||||
(DPRConfig, TFDPRQuestionEncoder),
|
||||
(MPNetConfig, TFMPNetModel),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -243,7 +233,6 @@ TF_MODEL_FOR_PRETRAINING_MAPPING = OrderedDict(
|
||||
(CTRLConfig, TFCTRLLMHeadModel),
|
||||
(ElectraConfig, TFElectraForPreTraining),
|
||||
(FunnelConfig, TFFunnelForPreTraining),
|
||||
(MPNetConfig, TFMPNetForMaskedLM),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -270,7 +259,6 @@ TF_MODEL_WITH_LM_HEAD_MAPPING = OrderedDict(
|
||||
(CTRLConfig, TFCTRLLMHeadModel),
|
||||
(ElectraConfig, TFElectraForMaskedLM),
|
||||
(FunnelConfig, TFFunnelForMaskedLM),
|
||||
(MPNetConfig, TFMPNetForMaskedLM),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -305,7 +293,6 @@ TF_MODEL_FOR_MASKED_LM_MAPPING = OrderedDict(
|
||||
(XLMConfig, TFXLMWithLMHeadModel),
|
||||
(ElectraConfig, TFElectraForMaskedLM),
|
||||
(FunnelConfig, TFFunnelForMaskedLM),
|
||||
(MPNetConfig, TFMPNetForMaskedLM),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -340,7 +327,6 @@ TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = OrderedDict(
|
||||
(ElectraConfig, TFElectraForSequenceClassification),
|
||||
(FunnelConfig, TFFunnelForSequenceClassification),
|
||||
(GPT2Config, TFGPT2ForSequenceClassification),
|
||||
(MPNetConfig, TFMPNetForSequenceClassification),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -360,7 +346,6 @@ TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING = OrderedDict(
|
||||
(XLMConfig, TFXLMForQuestionAnsweringSimple),
|
||||
(ElectraConfig, TFElectraForQuestionAnswering),
|
||||
(FunnelConfig, TFFunnelForQuestionAnswering),
|
||||
(MPNetConfig, TFMPNetForQuestionAnswering),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -380,7 +365,6 @@ TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = OrderedDict(
|
||||
(XLNetConfig, TFXLNetForTokenClassification),
|
||||
(ElectraConfig, TFElectraForTokenClassification),
|
||||
(FunnelConfig, TFFunnelForTokenClassification),
|
||||
(MPNetConfig, TFMPNetForTokenClassification),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -400,7 +384,6 @@ TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING = OrderedDict(
|
||||
(AlbertConfig, TFAlbertForMultipleChoice),
|
||||
(ElectraConfig, TFElectraForMultipleChoice),
|
||||
(FunnelConfig, TFFunnelForMultipleChoice),
|
||||
(MPNetConfig, TFMPNetForMultipleChoice),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -39,7 +39,6 @@ from ..layoutlm.tokenization_layoutlm import LayoutLMTokenizer
|
||||
from ..longformer.tokenization_longformer import LongformerTokenizer
|
||||
from ..lxmert.tokenization_lxmert import LxmertTokenizer
|
||||
from ..mobilebert.tokenization_mobilebert import MobileBertTokenizer
|
||||
from ..mpnet.tokenization_mpnet import MPNetTokenizer
|
||||
from ..openai.tokenization_openai import OpenAIGPTTokenizer
|
||||
from ..phobert.tokenization_phobert import PhobertTokenizer
|
||||
from ..prophetnet.tokenization_prophetnet import ProphetNetTokenizer
|
||||
@@ -74,7 +73,6 @@ from .configuration_auto import (
|
||||
MarianConfig,
|
||||
MBartConfig,
|
||||
MobileBertConfig,
|
||||
MPNetConfig,
|
||||
MT5Config,
|
||||
OpenAIGPTConfig,
|
||||
PegasusConfig,
|
||||
@@ -141,7 +139,6 @@ if is_tokenizers_available():
|
||||
from ..lxmert.tokenization_lxmert_fast import LxmertTokenizerFast
|
||||
from ..mbart.tokenization_mbart_fast import MBartTokenizerFast
|
||||
from ..mobilebert.tokenization_mobilebert_fast import MobileBertTokenizerFast
|
||||
from ..mpnet.tokenization_mpnet_fast import MPNetTokenizerFast
|
||||
from ..mt5 import MT5TokenizerFast
|
||||
from ..openai.tokenization_openai_fast import OpenAIGPTTokenizerFast
|
||||
from ..pegasus.tokenization_pegasus_fast import PegasusTokenizerFast
|
||||
@@ -169,7 +166,6 @@ else:
|
||||
LxmertTokenizerFast = None
|
||||
MBartTokenizerFast = None
|
||||
MobileBertTokenizerFast = None
|
||||
MPNetTokenizerFast = None
|
||||
MT5TokenizerFast = None
|
||||
OpenAIGPTTokenizerFast = None
|
||||
PegasusTokenizerFast = None
|
||||
@@ -224,7 +220,6 @@ TOKENIZER_MAPPING = OrderedDict(
|
||||
(RagConfig, (RagTokenizer, None)),
|
||||
(XLMProphetNetConfig, (XLMProphetNetTokenizer, None)),
|
||||
(ProphetNetConfig, (ProphetNetTokenizer, None)),
|
||||
(MPNetConfig, (MPNetTokenizer, MPNetTokenizerFast)),
|
||||
(TapasConfig, (TapasTokenizer, None)),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -31,7 +31,6 @@ if is_torch_available():
|
||||
BartForQuestionAnswering,
|
||||
BartForSequenceClassification,
|
||||
BartModel,
|
||||
BartPretrainedModel,
|
||||
PretrainedBartModel,
|
||||
)
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -577,9 +577,9 @@ class TFBartDecoder(tf.keras.layers.Layer):
|
||||
encoder_padding_mask = invert_mask(encoder_padding_mask)
|
||||
|
||||
# embed positions
|
||||
positions = self.embed_positions(input_ids, use_cache=(use_cache and decoder_cached_states is not None))
|
||||
positions = self.embed_positions(input_ids, use_cache=use_cache)
|
||||
|
||||
if use_cache and decoder_cached_states is not None:
|
||||
if use_cache:
|
||||
input_ids = input_ids[:, -1:]
|
||||
positions = positions[:, -1:]
|
||||
|
||||
@@ -964,7 +964,7 @@ class TFBartModel(TFPretrainedBartModel):
|
||||
else self.config.output_hidden_states
|
||||
)
|
||||
|
||||
if not use_cache or past_key_values is None:
|
||||
if not inputs["use_cache"]:
|
||||
inputs["decoder_input_ids"], decoder_padding_mask, causal_mask = self._prepare_bart_decoder_inputs(
|
||||
inputs["input_ids"],
|
||||
decoder_input_ids=inputs["decoder_input_ids"],
|
||||
@@ -1154,7 +1154,6 @@ class TFBartForConditionalGeneration(TFPretrainedBartModel):
|
||||
assert (
|
||||
decoder_cached_states
|
||||
), f"decoder cached states must be truthy. got {decoder_cached_states} from the 2nd element of past"
|
||||
|
||||
assert isinstance(
|
||||
encoder_outputs, TFBaseModelOutput
|
||||
), f"encoder_outputs should be a TFBaseModelOutput, Instead got {type(encoder_outputs)}."
|
||||
|
||||
@@ -59,4 +59,4 @@ if is_tf_available():
|
||||
)
|
||||
|
||||
if is_flax_available():
|
||||
from .modeling_flax_bert import FlaxBertForMaskedLM, FlaxBertModel
|
||||
from .modeling_flax_bert import FlaxBertModel
|
||||
@@ -13,14 +13,13 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Callable, Dict, Tuple
|
||||
from typing import Callable, Dict
|
||||
|
||||
import numpy as np
|
||||
|
||||
import flax.linen as nn
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
from jax.random import PRNGKey
|
||||
|
||||
from ...file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
|
||||
from ...modeling_flax_utils import FlaxPreTrainedModel, gelu
|
||||
@@ -102,8 +101,8 @@ class FlaxBertLayerNorm(nn.Module):
|
||||
scale: bool = True # If True, multiply by scale (gamma). When the next layer is linear
|
||||
# (also e.g. nn.relu), this can be disabled since the scaling will be
|
||||
# done by the next layer.
|
||||
scale_init: Callable[..., np.ndarray] = jax.nn.initializers.ones
|
||||
bias_init: Callable[..., np.ndarray] = jax.nn.initializers.zeros
|
||||
bias_init: jnp.ndarray = nn.initializers.zeros
|
||||
scale_init: jnp.ndarray = nn.initializers.ones
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, x):
|
||||
@@ -123,13 +122,11 @@ class FlaxBertLayerNorm(nn.Module):
|
||||
mean2 = jnp.mean(jax.lax.square(x), axis=-1, keepdims=True)
|
||||
var = mean2 - jax.lax.square(mean)
|
||||
mul = jax.lax.rsqrt(var + self.epsilon)
|
||||
|
||||
if self.scale:
|
||||
mul = mul * jnp.asarray(self.param("gamma", self.scale_init, (features,)))
|
||||
mul = mul * jnp.asarray(self.param("gamma", self.scale_init, (features,)), self.dtype)
|
||||
y = (x - mean) * mul
|
||||
|
||||
if self.bias:
|
||||
y = y + jnp.asarray(self.param("beta", self.bias_init, (features,)))
|
||||
y = y + jnp.asarray(self.param("beta", self.bias_init, (features,)), self.dtype)
|
||||
return y
|
||||
|
||||
|
||||
@@ -141,9 +138,7 @@ class FlaxBertEmbedding(nn.Module):
|
||||
|
||||
vocab_size: int
|
||||
hidden_size: int
|
||||
kernel_init_scale: float = 0.2
|
||||
emb_init: Callable[..., np.ndarray] = jax.nn.initializers.normal(stddev=kernel_init_scale)
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
emb_init: Callable[..., np.ndarray] = nn.initializers.normal(stddev=0.1)
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, inputs):
|
||||
@@ -158,105 +153,63 @@ class FlaxBertEmbeddings(nn.Module):
|
||||
hidden_size: int
|
||||
type_vocab_size: int
|
||||
max_length: int
|
||||
kernel_init_scale: float = 0.2
|
||||
dropout_rate: float = 0.0
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, input_ids, token_type_ids, position_ids, attention_mask, deterministic: bool = True):
|
||||
def __call__(self, input_ids, token_type_ids, position_ids, attention_mask):
|
||||
|
||||
# Embed
|
||||
w_emb = FlaxBertEmbedding(
|
||||
self.vocab_size,
|
||||
self.hidden_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
name="word_embeddings",
|
||||
dtype=self.dtype,
|
||||
)(jnp.atleast_2d(input_ids.astype("i4")))
|
||||
p_emb = FlaxBertEmbedding(
|
||||
self.max_length,
|
||||
self.hidden_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
name="position_embeddings",
|
||||
dtype=self.dtype,
|
||||
)(jnp.atleast_2d(position_ids.astype("i4")))
|
||||
t_emb = FlaxBertEmbedding(
|
||||
self.type_vocab_size,
|
||||
self.hidden_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
name="token_type_embeddings",
|
||||
dtype=self.dtype,
|
||||
)(jnp.atleast_2d(token_type_ids.astype("i4")))
|
||||
w_emb = FlaxBertEmbedding(self.vocab_size, self.hidden_size, name="word_embeddings")(
|
||||
jnp.atleast_2d(input_ids.astype("i4"))
|
||||
)
|
||||
p_emb = FlaxBertEmbedding(self.max_length, self.hidden_size, name="position_embeddings")(
|
||||
jnp.atleast_2d(position_ids.astype("i4"))
|
||||
)
|
||||
t_emb = FlaxBertEmbedding(self.type_vocab_size, self.hidden_size, name="token_type_embeddings")(
|
||||
jnp.atleast_2d(token_type_ids.astype("i4"))
|
||||
)
|
||||
|
||||
# Sum all embeddings
|
||||
summed_emb = w_emb + jnp.broadcast_to(p_emb, w_emb.shape) + t_emb
|
||||
|
||||
# Layer Norm
|
||||
layer_norm = FlaxBertLayerNorm(name="layer_norm", dtype=self.dtype)(summed_emb)
|
||||
embeddings = nn.Dropout(rate=self.dropout_rate)(layer_norm, deterministic=deterministic)
|
||||
return embeddings
|
||||
layer_norm = FlaxBertLayerNorm(name="layer_norm")(summed_emb)
|
||||
|
||||
return layer_norm
|
||||
|
||||
|
||||
class FlaxBertAttention(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state, attention_mask, deterministic: bool = True):
|
||||
def __call__(self, hidden_state, attention_mask):
|
||||
# Attention mask comes in as attention_mask.shape == (*batch_sizes, kv_length)
|
||||
# FLAX expects: attention_mask.shape == (*batch_sizes, 1, 1, kv_length) such that it is broadcastable
|
||||
# with attn_weights.shape == (*batch_sizes, num_heads, q_length, kv_length)
|
||||
attention_mask = jnp.expand_dims(attention_mask, axis=(-3, -2))
|
||||
self_att = nn.attention.SelfAttention(
|
||||
num_heads=self.num_heads,
|
||||
qkv_features=self.head_size,
|
||||
dropout_rate=self.dropout_rate,
|
||||
deterministic=deterministic,
|
||||
kernel_init=jax.nn.initializers.normal(self.kernel_init_scale, self.dtype),
|
||||
bias_init=jax.nn.initializers.zeros,
|
||||
name="self",
|
||||
dtype=self.dtype,
|
||||
)(hidden_state, attention_mask)
|
||||
self_att = nn.attention.SelfAttention(num_heads=self.num_heads, qkv_features=self.head_size, name="self")(
|
||||
hidden_state, attention_mask
|
||||
)
|
||||
|
||||
layer_norm = FlaxBertLayerNorm(name="layer_norm", dtype=self.dtype)(self_att + hidden_state)
|
||||
layer_norm = FlaxBertLayerNorm(name="layer_norm")(self_att + hidden_state)
|
||||
return layer_norm
|
||||
|
||||
|
||||
class FlaxBertIntermediate(nn.Module):
|
||||
output_size: int
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state):
|
||||
# TODO: Add ACT2FN reference to change activation function
|
||||
dense = nn.Dense(
|
||||
features=self.output_size,
|
||||
kernel_init=jax.nn.initializers.normal(self.kernel_init_scale, self.dtype),
|
||||
name="dense",
|
||||
dtype=self.dtype,
|
||||
)(hidden_state)
|
||||
dense = nn.Dense(features=self.output_size, name="dense")(hidden_state)
|
||||
return gelu(dense)
|
||||
|
||||
|
||||
class FlaxBertOutput(nn.Module):
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, intermediate_output, attention_output, deterministic: bool = True):
|
||||
hidden_state = nn.Dense(
|
||||
attention_output.shape[-1],
|
||||
kernel_init=jax.nn.initializers.normal(self.kernel_init_scale, self.dtype),
|
||||
name="dense",
|
||||
dtype=self.dtype,
|
||||
)(intermediate_output)
|
||||
hidden_state = nn.Dropout(rate=self.dropout_rate)(hidden_state, deterministic=deterministic)
|
||||
hidden_state = FlaxBertLayerNorm(name="layer_norm", dtype=self.dtype)(hidden_state + attention_output)
|
||||
def __call__(self, intermediate_output, attention_output):
|
||||
hidden_state = nn.Dense(attention_output.shape[-1], name="dense")(intermediate_output)
|
||||
hidden_state = FlaxBertLayerNorm(name="layer_norm")(hidden_state + attention_output)
|
||||
return hidden_state
|
||||
|
||||
|
||||
@@ -264,26 +217,12 @@ class FlaxBertLayer(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
intermediate_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state, attention_mask, deterministic: bool = True):
|
||||
attention = FlaxBertAttention(
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name="attention",
|
||||
dtype=self.dtype,
|
||||
)(hidden_state, attention_mask, deterministic=deterministic)
|
||||
intermediate = FlaxBertIntermediate(
|
||||
self.intermediate_size, kernel_init_scale=self.kernel_init_scale, name="intermediate", dtype=self.dtype
|
||||
)(attention)
|
||||
output = FlaxBertOutput(
|
||||
kernel_init_scale=self.kernel_init_scale, dropout_rate=self.dropout_rate, name="output", dtype=self.dtype
|
||||
)(intermediate, attention, deterministic=deterministic)
|
||||
def __call__(self, hidden_state, attention_mask):
|
||||
attention = FlaxBertAttention(self.num_heads, self.head_size, name="attention")(hidden_state, attention_mask)
|
||||
intermediate = FlaxBertIntermediate(self.intermediate_size, name="intermediate")(attention)
|
||||
output = FlaxBertOutput(name="output")(intermediate, attention)
|
||||
|
||||
return output
|
||||
|
||||
@@ -297,12 +236,9 @@ class FlaxBertLayerCollection(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
intermediate_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, inputs, attention_mask, deterministic: bool = True):
|
||||
def __call__(self, inputs, attention_mask):
|
||||
assert self.num_layers > 0, f"num_layers should be >= 1, got ({self.num_layers})"
|
||||
|
||||
# Initialize input / output
|
||||
@@ -310,16 +246,8 @@ class FlaxBertLayerCollection(nn.Module):
|
||||
|
||||
# Forward over all encoders
|
||||
for i in range(self.num_layers):
|
||||
layer = FlaxBertLayer(
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
self.intermediate_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name=f"{i}",
|
||||
dtype=self.dtype,
|
||||
)
|
||||
input_i = layer(input_i, attention_mask, deterministic=deterministic)
|
||||
layer = FlaxBertLayer(self.num_heads, self.head_size, self.intermediate_size, name=f"{i}")
|
||||
input_i = layer(input_i, attention_mask)
|
||||
return input_i
|
||||
|
||||
|
||||
@@ -328,39 +256,21 @@ class FlaxBertEncoder(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
intermediate_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state, attention_mask, deterministic: bool = True):
|
||||
def __call__(self, hidden_state, attention_mask):
|
||||
layer = FlaxBertLayerCollection(
|
||||
self.num_layers,
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
self.intermediate_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name="layer",
|
||||
dtype=self.dtype,
|
||||
)(hidden_state, attention_mask, deterministic=deterministic)
|
||||
self.num_layers, self.num_heads, self.head_size, self.intermediate_size, name="layer"
|
||||
)(hidden_state, attention_mask)
|
||||
return layer
|
||||
|
||||
|
||||
class FlaxBertPooler(nn.Module):
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state):
|
||||
cls_token = hidden_state[:, 0]
|
||||
out = nn.Dense(
|
||||
hidden_state.shape[-1],
|
||||
kernel_init=jax.nn.initializers.normal(self.kernel_init_scale, self.dtype),
|
||||
name="dense",
|
||||
dtype=self.dtype,
|
||||
)(cls_token)
|
||||
return nn.tanh(out)
|
||||
out = nn.Dense(hidden_state.shape[-1], name="dense")(cls_token)
|
||||
return jax.lax.tanh(out)
|
||||
|
||||
|
||||
class FlaxBertModule(nn.Module):
|
||||
@@ -372,104 +282,24 @@ class FlaxBertModule(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
intermediate_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, input_ids, attention_mask, token_type_ids, position_ids, deterministic: bool = True):
|
||||
def __call__(self, input_ids, attention_mask, token_type_ids, position_ids):
|
||||
|
||||
# Embedding
|
||||
embeddings = FlaxBertEmbeddings(
|
||||
self.vocab_size,
|
||||
self.hidden_size,
|
||||
self.type_vocab_size,
|
||||
self.max_length,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name="embeddings",
|
||||
dtype=self.dtype,
|
||||
)(input_ids, token_type_ids, position_ids, attention_mask, deterministic=deterministic)
|
||||
self.vocab_size, self.hidden_size, self.type_vocab_size, self.max_length, name="embeddings"
|
||||
)(input_ids, token_type_ids, position_ids, attention_mask)
|
||||
|
||||
# N stacked encoding layers
|
||||
encoder = FlaxBertEncoder(
|
||||
self.num_encoder_layers,
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
self.intermediate_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name="encoder",
|
||||
dtype=self.dtype,
|
||||
)(embeddings, attention_mask, deterministic=deterministic)
|
||||
self.num_encoder_layers, self.num_heads, self.head_size, self.intermediate_size, name="encoder"
|
||||
)(embeddings, attention_mask)
|
||||
|
||||
pooled = FlaxBertPooler(kernel_init_scale=self.kernel_init_scale, name="pooler", dtype=self.dtype)(encoder)
|
||||
pooled = FlaxBertPooler(name="pooler")(encoder)
|
||||
return encoder, pooled
|
||||
|
||||
|
||||
class FlaxBertPredictionHeadTransform(nn.Module):
|
||||
dtype: jnp.dtype = jnp.float32
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_states):
|
||||
hidden_states = nn.Dense(hidden_states.shape[-1], name="dense", dtype=self.dtype)(hidden_states)
|
||||
hidden_states = nn.elu(hidden_states) # TODO: ACT2FN[config.hidden_act]
|
||||
return FlaxBertLayerNorm(name="LayerNorm", dtype=self.dtype)(hidden_states)
|
||||
|
||||
|
||||
class FlaxBertLMPredictionHead(nn.Module):
|
||||
vocab_size: int
|
||||
dtype: jnp.dtype = jnp.float32
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_states):
|
||||
# TODO: The output weights are the same as the input embeddings, but there is
|
||||
# an output-only bias for each token.
|
||||
# Need a link between the two variables so that the bias is correctly
|
||||
# resized with `resize_token_embeddings`
|
||||
|
||||
hidden_states = FlaxBertPredictionHeadTransform(name="transform", dtype=self.dtype)(hidden_states)
|
||||
hidden_states = nn.Dense(self.vocab_size, name="decoder", dtype=self.dtype)(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class FlaxBertOnlyMLMHead(nn.Module):
|
||||
vocab_size: int
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
head_size: int
|
||||
num_heads: int
|
||||
num_encoder_layers: int
|
||||
type_vocab_size: int
|
||||
max_length: int
|
||||
dropout_rate: float = 0.0
|
||||
dtype: jnp.dtype = jnp.float32
|
||||
|
||||
@nn.compact
|
||||
def __call__(
|
||||
self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, deterministic: bool = True
|
||||
):
|
||||
# Model
|
||||
encoder, pooled = FlaxBertModule(
|
||||
vocab_size=self.vocab_size,
|
||||
type_vocab_size=self.type_vocab_size,
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=self.intermediate_size,
|
||||
head_size=self.hidden_size,
|
||||
num_heads=self.num_heads,
|
||||
num_encoder_layers=self.num_encoder_layers,
|
||||
max_length=self.max_length,
|
||||
dropout_rate=self.dropout_rate,
|
||||
dtype=self.dtype,
|
||||
)(input_ids, attention_mask, token_type_ids, position_ids, deterministic=deterministic)
|
||||
|
||||
# Compute the prediction scores
|
||||
encoder = nn.Dropout(rate=self.dropout_rate)(encoder, deterministic=deterministic)
|
||||
logits = FlaxBertLMPredictionHead(vocab_size=self.vocab_size, name="predictions", dtype=self.dtype)(encoder)
|
||||
|
||||
return logits, pooled
|
||||
|
||||
|
||||
@add_start_docstrings(
|
||||
"The bare Bert Model transformer outputting raw hidden-states without any specific head on top.",
|
||||
BERT_START_DOCSTRING,
|
||||
@@ -555,8 +385,8 @@ class FlaxBertModel(FlaxPreTrainedModel):
|
||||
|
||||
return jax_state
|
||||
|
||||
def __init__(self, config: BertConfig, state: dict, seed: int = 0, dtype: jnp.dtype = jnp.float32):
|
||||
module = FlaxBertModule(
|
||||
def __init__(self, config: BertConfig, state: dict, seed: int = 0, **kwargs):
|
||||
model = FlaxBertModule(
|
||||
vocab_size=config.vocab_size,
|
||||
hidden_size=config.hidden_size,
|
||||
type_vocab_size=config.type_vocab_size,
|
||||
@@ -565,43 +395,16 @@ class FlaxBertModel(FlaxPreTrainedModel):
|
||||
num_heads=config.num_attention_heads,
|
||||
head_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
dropout_rate=config.hidden_dropout_prob,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
super().__init__(config, module, state, seed)
|
||||
super().__init__(config, model, state, seed)
|
||||
|
||||
@property
|
||||
def module(self) -> nn.Module:
|
||||
return self._module
|
||||
|
||||
@add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
|
||||
def __call__(
|
||||
self,
|
||||
input_ids,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
params: dict = None,
|
||||
dropout_rng: PRNGKey = None,
|
||||
train: bool = False,
|
||||
):
|
||||
input_ids, attention_mask, token_type_ids, position_ids = self._check_inputs(
|
||||
input_ids, attention_mask, token_type_ids, position_ids
|
||||
)
|
||||
|
||||
# Handle any PRNG if needed
|
||||
rngs = {}
|
||||
if dropout_rng is not None:
|
||||
rngs["dropout"] = dropout_rng
|
||||
|
||||
return self.module.apply(
|
||||
{"params": params or self.params},
|
||||
jnp.array(input_ids, dtype="i4"),
|
||||
jnp.array(attention_mask, dtype="i4"),
|
||||
jnp.array(token_type_ids, dtype="i4"),
|
||||
jnp.array(position_ids, dtype="i4"),
|
||||
not train,
|
||||
rngs=rngs,
|
||||
)
|
||||
|
||||
def _check_inputs(self, input_ids, attention_mask, token_type_ids, position_ids):
|
||||
def __call__(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None):
|
||||
if token_type_ids is None:
|
||||
token_type_ids = jnp.ones_like(input_ids)
|
||||
|
||||
@@ -611,62 +414,10 @@ class FlaxBertModel(FlaxPreTrainedModel):
|
||||
if attention_mask is None:
|
||||
attention_mask = jnp.ones_like(input_ids)
|
||||
|
||||
return input_ids, attention_mask, token_type_ids, position_ids
|
||||
|
||||
def init(self, rng: jax.random.PRNGKey, input_shape: Tuple):
|
||||
input_ids, attention_mask, token_type_ids, position_ids = self._check_inputs(
|
||||
jnp.zeros(input_shape, dtype="i4"), None, None, None
|
||||
)
|
||||
|
||||
params_rng, dropout_rng = jax.random.split(rng)
|
||||
rngs = {"params": params_rng, "dropout": dropout_rng}
|
||||
|
||||
self.params = self.module.init(rngs, input_ids, attention_mask, token_type_ids, position_ids)["params"]
|
||||
|
||||
|
||||
class FlaxBertForMaskedLM(FlaxBertModel):
|
||||
def __init__(self, config: BertConfig, state: dict, seed: int = 0, dtype: jnp.dtype = jnp.float32, **kwargs):
|
||||
super().__init__(config, state, seed, dtype)
|
||||
|
||||
self._module = FlaxBertOnlyMLMHead(
|
||||
vocab_size=config.vocab_size,
|
||||
type_vocab_size=config.type_vocab_size,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
head_size=config.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
num_encoder_layers=config.num_hidden_layers,
|
||||
max_length=config.max_length,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
input_ids,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
params: dict = None,
|
||||
dropout_rng: PRNGKey = None,
|
||||
train: bool = False,
|
||||
):
|
||||
input_ids, attention_mask, token_type_ids, position_ids = self._check_inputs(
|
||||
input_ids, attention_mask, token_type_ids, position_ids
|
||||
)
|
||||
|
||||
# Handle any PRNG if needed
|
||||
rngs = {}
|
||||
if dropout_rng is not None:
|
||||
rngs["dropout"] = dropout_rng
|
||||
|
||||
pooled, logits = self.module.apply(
|
||||
{"params": params or self.params},
|
||||
return self.model.apply(
|
||||
{"params": self.params},
|
||||
jnp.array(input_ids, dtype="i4"),
|
||||
jnp.array(attention_mask, dtype="i4"),
|
||||
jnp.array(token_type_ids, dtype="i4"),
|
||||
jnp.array(position_ids, dtype="i4"),
|
||||
not train,
|
||||
rngs=rngs,
|
||||
)
|
||||
|
||||
return logits, pooled
|
||||
@@ -251,6 +251,8 @@ class MobileBertSelfAttention(nn.Module):
|
||||
value_tensor,
|
||||
attention_mask=None,
|
||||
head_mask=None,
|
||||
encoder_hidden_states=None,
|
||||
encoder_attention_mask=None,
|
||||
output_attentions=None,
|
||||
):
|
||||
mixed_query_layer = self.query(query_tensor)
|
||||
@@ -333,6 +335,8 @@ class MobileBertAttention(nn.Module):
|
||||
layer_input,
|
||||
attention_mask=None,
|
||||
head_mask=None,
|
||||
encoder_hidden_states=None,
|
||||
encoder_attention_mask=None,
|
||||
output_attentions=None,
|
||||
):
|
||||
self_outputs = self.self(
|
||||
@@ -341,6 +345,8 @@ class MobileBertAttention(nn.Module):
|
||||
value_tensor,
|
||||
attention_mask,
|
||||
head_mask,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
output_attentions,
|
||||
)
|
||||
# Run a linear projection of `hidden_size` then add a residual
|
||||
@@ -492,6 +498,8 @@ class MobileBertLayer(nn.Module):
|
||||
hidden_states,
|
||||
attention_mask=None,
|
||||
head_mask=None,
|
||||
encoder_hidden_states=None,
|
||||
encoder_attention_mask=None,
|
||||
output_attentions=None,
|
||||
):
|
||||
if self.use_bottleneck:
|
||||
@@ -546,6 +554,8 @@ class MobileBertEncoder(nn.Module):
|
||||
hidden_states,
|
||||
attention_mask=None,
|
||||
head_mask=None,
|
||||
encoder_hidden_states=None,
|
||||
encoder_attention_mask=None,
|
||||
output_attentions=False,
|
||||
output_hidden_states=False,
|
||||
return_dict=True,
|
||||
@@ -560,6 +570,8 @@ class MobileBertEncoder(nn.Module):
|
||||
hidden_states,
|
||||
attention_mask,
|
||||
head_mask[i],
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
output_attentions,
|
||||
)
|
||||
hidden_states = layer_outputs[0]
|
||||
@@ -771,6 +783,16 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
|
||||
Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation.
|
||||
This is useful if you want more control over how to convert :obj:`input_ids` indices into associated
|
||||
vectors than the model's internal embedding lookup matrix.
|
||||
encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
|
||||
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
|
||||
the model is configured as a decoder.
|
||||
encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
|
||||
Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
|
||||
the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
|
||||
|
||||
- 1 for tokens that are **not masked**,
|
||||
- 0 for tokens that are **masked**.
|
||||
|
||||
output_attentions (:obj:`bool`, `optional`):
|
||||
Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned
|
||||
tensors for more detail.
|
||||
@@ -830,6 +852,8 @@ class MobileBertModel(MobileBertPreTrainedModel):
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
encoder_hidden_states=None,
|
||||
encoder_attention_mask=None,
|
||||
output_hidden_states=None,
|
||||
output_attentions=None,
|
||||
return_dict=None,
|
||||
@@ -862,6 +886,17 @@ class MobileBertModel(MobileBertPreTrainedModel):
|
||||
attention_mask, input_shape, self.device
|
||||
)
|
||||
|
||||
# If a 2D ou 3D attention mask is provided for the cross-attention
|
||||
# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
|
||||
if self.config.is_decoder and encoder_hidden_states is not None:
|
||||
encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
|
||||
encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
|
||||
if encoder_attention_mask is None:
|
||||
encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
|
||||
encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
|
||||
else:
|
||||
encoder_extended_attention_mask = None
|
||||
|
||||
# Prepare head mask if needed
|
||||
# 1.0 in head_mask indicate we keep the head
|
||||
# attention_probs has shape bsz x n_heads x N x N
|
||||
@@ -876,6 +911,8 @@ class MobileBertModel(MobileBertPreTrainedModel):
|
||||
embedding_output,
|
||||
attention_mask=extended_attention_mask,
|
||||
head_mask=head_mask,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_extended_attention_mask,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
@@ -1046,6 +1083,8 @@ class MobileBertForMaskedLM(MobileBertPreTrainedModel):
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
labels=None,
|
||||
encoder_hidden_states=None,
|
||||
encoder_attention_mask=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
@@ -1065,6 +1104,8 @@ class MobileBertForMaskedLM(MobileBertPreTrainedModel):
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
# flake8: noqa
|
||||
# There's no way to ignore "F401 '...' imported but unused" warnings in this
|
||||
# module, but to preserve other warnings. So, don't check this module at all.
|
||||
|
||||
from ...file_utils import is_flax_available, is_tf_available, is_tokenizers_available, is_torch_available
|
||||
from .configuration_mpnet import MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP, MPNetConfig
|
||||
from .tokenization_mpnet import MPNetTokenizer
|
||||
|
||||
|
||||
if is_tokenizers_available():
|
||||
from .tokenization_mpnet_fast import MPNetTokenizerFast
|
||||
|
||||
if is_torch_available():
|
||||
from .modeling_mpnet import (
|
||||
MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
MPNetForMaskedLM,
|
||||
MPNetForMultipleChoice,
|
||||
MPNetForQuestionAnswering,
|
||||
MPNetForSequenceClassification,
|
||||
MPNetForTokenClassification,
|
||||
MPNetLayer,
|
||||
MPNetModel,
|
||||
MPNetPreTrainedModel,
|
||||
)
|
||||
|
||||
if is_tf_available():
|
||||
from .modeling_tf_mpnet import (
|
||||
TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
TFMPNetEmbeddings,
|
||||
TFMPNetForMaskedLM,
|
||||
TFMPNetForMultipleChoice,
|
||||
TFMPNetForQuestionAnswering,
|
||||
TFMPNetForSequenceClassification,
|
||||
TFMPNetForTokenClassification,
|
||||
TFMPNetMainLayer,
|
||||
TFMPNetModel,
|
||||
TFMPNetPreTrainedModel,
|
||||
)
|
||||
@@ -1,116 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The HuggingFace Inc. team, Microsoft Corporation.
|
||||
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" MPNet model configuration """
|
||||
|
||||
from ...configuration_utils import PretrainedConfig
|
||||
from ...utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
|
||||
"microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/config.json",
|
||||
}
|
||||
|
||||
|
||||
class MPNetConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a :class:`~transformers.MPNetModel` or a
|
||||
:class:`~transformers.TFMPNetModel`. It is used to instantiate a MPNet model according to the specified arguments,
|
||||
defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
|
||||
to that of the MPNet `mpnet-base <https://huggingface.co/mpnet-base>`__ architecture.
|
||||
|
||||
Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model
|
||||
outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information.
|
||||
|
||||
Args:
|
||||
vocab_size (:obj:`int`, `optional`, defaults to 30527):
|
||||
Vocabulary size of the MPNet model. Defines the number of different tokens that can be represented by the
|
||||
:obj:`inputs_ids` passed when calling :class:`~transformers.MPNetModel` or
|
||||
:class:`~transformers.TFMPNetModel`.
|
||||
hidden_size (:obj:`int`, `optional`, defaults to 768):
|
||||
Dimensionality of the encoder layers and the pooler layer.
|
||||
num_hidden_layers (:obj:`int`, `optional`, defaults to 12):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
num_attention_heads (:obj:`int`, `optional`, defaults to 12):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
intermediate_size (:obj:`int`, `optional`, defaults to 3072):
|
||||
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
|
||||
hidden_act (:obj:`str` or :obj:`Callable`, `optional`, defaults to :obj:`"gelu"`):
|
||||
The non-linear activation function (function or string) in the encoder and pooler. If string,
|
||||
:obj:`"gelu"`, :obj:`"relu"`, :obj:`"silu"` and :obj:`"gelu_new"` are supported.
|
||||
hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1):
|
||||
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
|
||||
attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1):
|
||||
The dropout ratio for the attention probabilities.
|
||||
max_position_embeddings (:obj:`int`, `optional`, defaults to 512):
|
||||
The maximum sequence length that this model might ever be used with. Typically set this to something large
|
||||
just in case (e.g., 512 or 1024 or 2048).
|
||||
initializer_range (:obj:`float`, `optional`, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_eps (:obj:`float`, `optional`, defaults to 1e-12):
|
||||
The epsilon used by the layer normalization layers.
|
||||
relative_attention_num_buckets (:obj:`int`, `optional`, defaults to 32):
|
||||
The number of buckets to use for each attention layer.
|
||||
|
||||
Examples::
|
||||
|
||||
>>> from transformers import MPNetModel, MPNetConfig
|
||||
|
||||
>>> # Initializing a MPNet mpnet-base style configuration
|
||||
>>> configuration = MPNetConfig()
|
||||
|
||||
>>> # Initializing a model from the mpnet-base style configuration
|
||||
>>> model = MPNetModel(configuration)
|
||||
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
"""
|
||||
model_type = "mpnet"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=30527,
|
||||
hidden_size=768,
|
||||
num_hidden_layers=12,
|
||||
num_attention_heads=12,
|
||||
intermediate_size=3072,
|
||||
hidden_act="gelu",
|
||||
hidden_dropout_prob=0.1,
|
||||
attention_probs_dropout_prob=0.1,
|
||||
max_position_embeddings=512,
|
||||
initializer_range=0.02,
|
||||
layer_norm_eps=1e-12,
|
||||
relative_attention_num_buckets=32,
|
||||
pad_token_id=1,
|
||||
bos_token_id=0,
|
||||
eos_token_id=2,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
||||
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.intermediate_size = intermediate_size
|
||||
self.hidden_dropout_prob = hidden_dropout_prob
|
||||
self.attention_probs_dropout_prob = attention_probs_dropout_prob
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
self.relative_attention_num_buckets = relative_attention_num_buckets
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,528 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The HuggingFace Inc. team, Microsoft Corporation.
|
||||
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Tokenization classes for MPNet."""
|
||||
|
||||
import collections
|
||||
import os
|
||||
import unicodedata
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from ...tokenization_utils import AddedToken, PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
|
||||
from ...utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
|
||||
|
||||
PRETRAINED_VOCAB_FILES_MAP = {
|
||||
"vocab_file": {
|
||||
"microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt",
|
||||
}
|
||||
}
|
||||
|
||||
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
|
||||
"microsoft/mpnet-base": 512,
|
||||
}
|
||||
|
||||
PRETRAINED_INIT_CONFIGURATION = {
|
||||
"microsoft/mpnet-base": {"do_lower_case": True},
|
||||
}
|
||||
|
||||
|
||||
def load_vocab(vocab_file):
|
||||
"""Loads a vocabulary file into a dictionary."""
|
||||
vocab = collections.OrderedDict()
|
||||
with open(vocab_file, "r", encoding="utf-8") as reader:
|
||||
tokens = reader.readlines()
|
||||
for index, token in enumerate(tokens):
|
||||
token = token.rstrip("\n")
|
||||
vocab[token] = index
|
||||
return vocab
|
||||
|
||||
|
||||
def whitespace_tokenize(text):
|
||||
"""Runs basic whitespace cleaning and splitting on a piece of text."""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return []
|
||||
tokens = text.split()
|
||||
return tokens
|
||||
|
||||
|
||||
class MPNetTokenizer(PreTrainedTokenizer):
|
||||
"""
|
||||
|
||||
This tokenizer inherits from :class:`~transformers.BertTokenizer` which contains most of the methods. Users should
|
||||
refer to the superclass for more information regarding methods.
|
||||
|
||||
Args:
|
||||
vocab_file (:obj:`str`):
|
||||
Path to the vocabulary file.
|
||||
do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to lowercase the input when tokenizing.
|
||||
do_basic_tokenize (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to do basic tokenization before WordPiece.
|
||||
never_split (:obj:`Iterable`, `optional`):
|
||||
Collection of tokens which will never be split during tokenization. Only has an effect when
|
||||
:obj:`do_basic_tokenize=True`
|
||||
bos_token (:obj:`str`, `optional`, defaults to :obj:`"<s>"`):
|
||||
The beginning of sequence token that was used during pre-training. Can be used a sequence classifier token.
|
||||
|
||||
.. note::
|
||||
|
||||
When building a sequence using special tokens, this is not the token that is used for the beginning of
|
||||
sequence. The token used is the :obj:`cls_token`.
|
||||
eos_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
|
||||
The end of sequence token.
|
||||
|
||||
.. note::
|
||||
|
||||
When building a sequence using special tokens, this is not the token that is used for the end of
|
||||
sequence. The token used is the :obj:`sep_token`.
|
||||
sep_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
|
||||
The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
|
||||
sequence classification or for a text and a question for question answering. It is also used as the last
|
||||
token of a sequence built with special tokens.
|
||||
cls_token (:obj:`str`, `optional`, defaults to :obj:`"<s>"`):
|
||||
The classifier token which is used when doing sequence classification (classification of the whole sequence
|
||||
instead of per-token classification). It is the first token of the sequence when built with special tokens.
|
||||
unk_token (:obj:`str`, `optional`, defaults to :obj:`"[UNK]"`):
|
||||
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
|
||||
token instead.
|
||||
pad_token (:obj:`str`, `optional`, defaults to :obj:`"<pad>"`):
|
||||
The token used for padding, for example when batching sequences of different lengths.
|
||||
mask_token (:obj:`str`, `optional`, defaults to :obj:`"<mask>"`):
|
||||
The token used for masking values. This is the token used when training this model with masked language
|
||||
modeling. This is the token which the model will try to predict.
|
||||
tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to tokenize Chinese characters.
|
||||
|
||||
This should likely be deactivated for Japanese (see this `issue
|
||||
<https://github.com/huggingface/transformers/issues/328>`__).
|
||||
strip_accents: (:obj:`bool`, `optional`):
|
||||
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
|
||||
value for :obj:`lowercase` (as in the original BERT).
|
||||
"""
|
||||
|
||||
vocab_files_names = VOCAB_FILES_NAMES
|
||||
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
|
||||
pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
|
||||
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_file,
|
||||
do_lower_case=True,
|
||||
do_basic_tokenize=True,
|
||||
never_split=None,
|
||||
bos_token="<s>",
|
||||
eos_token="</s>",
|
||||
sep_token="</s>",
|
||||
cls_token="<s>",
|
||||
unk_token="[UNK]",
|
||||
pad_token="<pad>",
|
||||
mask_token="<mask>",
|
||||
tokenize_chinese_chars=True,
|
||||
strip_accents=None,
|
||||
**kwargs
|
||||
):
|
||||
bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
|
||||
eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
|
||||
sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token
|
||||
cls_token = AddedToken(cls_token, lstrip=False, rstrip=False) if isinstance(cls_token, str) else cls_token
|
||||
unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
|
||||
pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
|
||||
|
||||
# Mask token behave like a normal word, i.e. include the space before it
|
||||
mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token
|
||||
|
||||
super().__init__(
|
||||
do_lower_case=do_lower_case,
|
||||
do_basic_tokenize=do_basic_tokenize,
|
||||
never_split=never_split,
|
||||
bos_token=bos_token,
|
||||
eos_token=eos_token,
|
||||
unk_token=unk_token,
|
||||
sep_token=sep_token,
|
||||
cls_token=cls_token,
|
||||
pad_token=pad_token,
|
||||
mask_token=mask_token,
|
||||
tokenize_chinese_chars=tokenize_chinese_chars,
|
||||
strip_accents=strip_accents,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if not os.path.isfile(vocab_file):
|
||||
raise ValueError(
|
||||
"Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained "
|
||||
"model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file)
|
||||
)
|
||||
self.vocab = load_vocab(vocab_file)
|
||||
self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])
|
||||
self.do_basic_tokenize = do_basic_tokenize
|
||||
if do_basic_tokenize:
|
||||
self.basic_tokenizer = BasicTokenizer(
|
||||
do_lower_case=do_lower_case,
|
||||
never_split=never_split,
|
||||
tokenize_chinese_chars=tokenize_chinese_chars,
|
||||
strip_accents=strip_accents,
|
||||
)
|
||||
self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=self.unk_token)
|
||||
|
||||
@property
|
||||
def do_lower_case(self):
|
||||
return self.basic_tokenizer.do_lower_case
|
||||
|
||||
@property
|
||||
def vocab_size(self):
|
||||
return len(self.vocab)
|
||||
|
||||
def get_vocab(self):
|
||||
return dict(self.vocab, **self.added_tokens_encoder)
|
||||
|
||||
def _tokenize(self, text):
|
||||
split_tokens = []
|
||||
if self.do_basic_tokenize:
|
||||
for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens):
|
||||
|
||||
# If the token is part of the never_split set
|
||||
if token in self.basic_tokenizer.never_split:
|
||||
split_tokens.append(token)
|
||||
else:
|
||||
split_tokens += self.wordpiece_tokenizer.tokenize(token)
|
||||
else:
|
||||
split_tokens = self.wordpiece_tokenizer.tokenize(text)
|
||||
return split_tokens
|
||||
|
||||
def _convert_token_to_id(self, token):
|
||||
""" Converts a token (str) in an id using the vocab. """
|
||||
return self.vocab.get(token, self.vocab.get(self.unk_token))
|
||||
|
||||
def _convert_id_to_token(self, index):
|
||||
"""Converts an index (integer) in a token (str) using the vocab."""
|
||||
return self.ids_to_tokens.get(index, self.unk_token)
|
||||
|
||||
def convert_tokens_to_string(self, tokens):
|
||||
""" Converts a sequence of tokens (string) in a single string. """
|
||||
out_string = " ".join(tokens).replace(" ##", "").strip()
|
||||
return out_string
|
||||
|
||||
def build_inputs_with_special_tokens(
|
||||
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
|
||||
) -> List[int]:
|
||||
"""
|
||||
Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
|
||||
adding special tokens. A MPNet sequence has the following format:
|
||||
|
||||
- single sequence: ``<s> X </s>``
|
||||
- pair of sequences: ``<s> A </s></s> B </s>``
|
||||
|
||||
Args:
|
||||
token_ids_0 (:obj:`List[int]`):
|
||||
List of IDs to which the special tokens will be added
|
||||
token_ids_1 (:obj:`List[int]`, `optional`, defaults to :obj:`None`):
|
||||
Optional second list of IDs for sequence pairs.
|
||||
|
||||
Returns:
|
||||
:obj:`List[int]`: list of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
|
||||
"""
|
||||
if token_ids_1 is None:
|
||||
return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
|
||||
cls = [self.cls_token_id]
|
||||
sep = [self.sep_token_id]
|
||||
return cls + token_ids_0 + sep + sep + token_ids_1 + sep
|
||||
|
||||
def get_special_tokens_mask(
|
||||
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
|
||||
) -> List[int]:
|
||||
"""
|
||||
Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
|
||||
special tokens using the tokenizer ``prepare_for_model`` methods.
|
||||
|
||||
Args:
|
||||
token_ids_0 (:obj:`List[int]`):
|
||||
List of ids.
|
||||
token_ids_1 (:obj:`List[int]`, `optional`):
|
||||
Optional second list of IDs for sequence pairs.
|
||||
already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
||||
Set to True if the token list is already formatted with special tokens for the model
|
||||
|
||||
Returns:
|
||||
:obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
|
||||
"""
|
||||
if already_has_special_tokens:
|
||||
if token_ids_1 is not None:
|
||||
raise ValueError(
|
||||
"You should not supply a second sequence if the provided sequence of "
|
||||
"ids is already formated with special tokens for the model."
|
||||
)
|
||||
return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0))
|
||||
|
||||
if token_ids_1 is None:
|
||||
return [1] + ([0] * len(token_ids_0)) + [1]
|
||||
return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
|
||||
|
||||
def create_token_type_ids_from_sequences(
|
||||
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
|
||||
) -> List[int]:
|
||||
"""
|
||||
Creates a mask from the two sequences passed to be used in a sequence-pair classification task. MPNet does not
|
||||
make use of token type ids, therefore a list of zeros is returned.
|
||||
|
||||
Args:
|
||||
token_ids_0 (:obj:`List[int]`):
|
||||
List of ids.
|
||||
token_ids_1 (:obj:`List[int]`, `optional`):
|
||||
Optional second list of IDs for sequence pairs.
|
||||
|
||||
Returns:
|
||||
:obj:`List[int]`: List of zeros.
|
||||
"""
|
||||
sep = [self.sep_token_id]
|
||||
cls = [self.cls_token_id]
|
||||
|
||||
if token_ids_1 is None:
|
||||
return len(cls + token_ids_0 + sep) * [0]
|
||||
return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]
|
||||
|
||||
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
|
||||
index = 0
|
||||
if os.path.isdir(save_directory):
|
||||
vocab_file = os.path.join(
|
||||
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
|
||||
)
|
||||
else:
|
||||
vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory
|
||||
with open(vocab_file, "w", encoding="utf-8") as writer:
|
||||
for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):
|
||||
if index != token_index:
|
||||
logger.warning(
|
||||
"Saving vocabulary to {}: vocabulary indices are not consecutive."
|
||||
" Please check that the vocabulary is not corrupted!".format(vocab_file)
|
||||
)
|
||||
index = token_index
|
||||
writer.write(token + "\n")
|
||||
index += 1
|
||||
return (vocab_file,)
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.tokenization_bert.BasicTokenizer
|
||||
class BasicTokenizer(object):
|
||||
"""
|
||||
Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).
|
||||
|
||||
Args:
|
||||
do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to lowercase the input when tokenizing.
|
||||
never_split (:obj:`Iterable`, `optional`):
|
||||
Collection of tokens which will never be split during tokenization. Only has an effect when
|
||||
:obj:`do_basic_tokenize=True`
|
||||
tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to tokenize Chinese characters.
|
||||
|
||||
This should likely be deactivated for Japanese (see this `issue
|
||||
<https://github.com/huggingface/transformers/issues/328>`__).
|
||||
strip_accents: (:obj:`bool`, `optional`):
|
||||
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
|
||||
value for :obj:`lowercase` (as in the original BERT).
|
||||
"""
|
||||
|
||||
def __init__(self, do_lower_case=True, never_split=None, tokenize_chinese_chars=True, strip_accents=None):
|
||||
if never_split is None:
|
||||
never_split = []
|
||||
self.do_lower_case = do_lower_case
|
||||
self.never_split = set(never_split)
|
||||
self.tokenize_chinese_chars = tokenize_chinese_chars
|
||||
self.strip_accents = strip_accents
|
||||
|
||||
def tokenize(self, text, never_split=None):
|
||||
"""
|
||||
Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see
|
||||
WordPieceTokenizer.
|
||||
|
||||
Args:
|
||||
**never_split**: (`optional`) list of str
|
||||
Kept for backward compatibility purposes. Now implemented directly at the base class level (see
|
||||
:func:`PreTrainedTokenizer.tokenize`) List of token not to split.
|
||||
"""
|
||||
# union() returns a new set by concatenating the two sets.
|
||||
never_split = self.never_split.union(set(never_split)) if never_split else self.never_split
|
||||
text = self._clean_text(text)
|
||||
|
||||
# This was added on November 1st, 2018 for the multilingual and Chinese
|
||||
# models. This is also applied to the English models now, but it doesn't
|
||||
# matter since the English models were not trained on any Chinese data
|
||||
# and generally don't have any Chinese data in them (there are Chinese
|
||||
# characters in the vocabulary because Wikipedia does have some Chinese
|
||||
# words in the English Wikipedia.).
|
||||
if self.tokenize_chinese_chars:
|
||||
text = self._tokenize_chinese_chars(text)
|
||||
orig_tokens = whitespace_tokenize(text)
|
||||
split_tokens = []
|
||||
for token in orig_tokens:
|
||||
if token not in never_split:
|
||||
if self.do_lower_case:
|
||||
token = token.lower()
|
||||
if self.strip_accents is not False:
|
||||
token = self._run_strip_accents(token)
|
||||
elif self.strip_accents:
|
||||
token = self._run_strip_accents(token)
|
||||
split_tokens.extend(self._run_split_on_punc(token, never_split))
|
||||
|
||||
output_tokens = whitespace_tokenize(" ".join(split_tokens))
|
||||
return output_tokens
|
||||
|
||||
def _run_strip_accents(self, text):
|
||||
"""Strips accents from a piece of text."""
|
||||
text = unicodedata.normalize("NFD", text)
|
||||
output = []
|
||||
for char in text:
|
||||
cat = unicodedata.category(char)
|
||||
if cat == "Mn":
|
||||
continue
|
||||
output.append(char)
|
||||
return "".join(output)
|
||||
|
||||
def _run_split_on_punc(self, text, never_split=None):
|
||||
"""Splits punctuation on a piece of text."""
|
||||
if never_split is not None and text in never_split:
|
||||
return [text]
|
||||
chars = list(text)
|
||||
i = 0
|
||||
start_new_word = True
|
||||
output = []
|
||||
while i < len(chars):
|
||||
char = chars[i]
|
||||
if _is_punctuation(char):
|
||||
output.append([char])
|
||||
start_new_word = True
|
||||
else:
|
||||
if start_new_word:
|
||||
output.append([])
|
||||
start_new_word = False
|
||||
output[-1].append(char)
|
||||
i += 1
|
||||
|
||||
return ["".join(x) for x in output]
|
||||
|
||||
def _tokenize_chinese_chars(self, text):
|
||||
"""Adds whitespace around any CJK character."""
|
||||
output = []
|
||||
for char in text:
|
||||
cp = ord(char)
|
||||
if self._is_chinese_char(cp):
|
||||
output.append(" ")
|
||||
output.append(char)
|
||||
output.append(" ")
|
||||
else:
|
||||
output.append(char)
|
||||
return "".join(output)
|
||||
|
||||
def _is_chinese_char(self, cp):
|
||||
"""Checks whether CP is the codepoint of a CJK character."""
|
||||
# This defines a "chinese character" as anything in the CJK Unicode block:
|
||||
# https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
|
||||
#
|
||||
# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
|
||||
# despite its name. The modern Korean Hangul alphabet is a different block,
|
||||
# as is Japanese Hiragana and Katakana. Those alphabets are used to write
|
||||
# space-separated words, so they are not treated specially and handled
|
||||
# like the all of the other languages.
|
||||
if (
|
||||
(cp >= 0x4E00 and cp <= 0x9FFF)
|
||||
or (cp >= 0x3400 and cp <= 0x4DBF) #
|
||||
or (cp >= 0x20000 and cp <= 0x2A6DF) #
|
||||
or (cp >= 0x2A700 and cp <= 0x2B73F) #
|
||||
or (cp >= 0x2B740 and cp <= 0x2B81F) #
|
||||
or (cp >= 0x2B820 and cp <= 0x2CEAF) #
|
||||
or (cp >= 0xF900 and cp <= 0xFAFF)
|
||||
or (cp >= 0x2F800 and cp <= 0x2FA1F) #
|
||||
): #
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _clean_text(self, text):
|
||||
"""Performs invalid character removal and whitespace cleanup on text."""
|
||||
output = []
|
||||
for char in text:
|
||||
cp = ord(char)
|
||||
if cp == 0 or cp == 0xFFFD or _is_control(char):
|
||||
continue
|
||||
if _is_whitespace(char):
|
||||
output.append(" ")
|
||||
else:
|
||||
output.append(char)
|
||||
return "".join(output)
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.tokenization_bert.WordpieceTokenizer
|
||||
class WordpieceTokenizer(object):
|
||||
"""Runs WordPiece tokenization."""
|
||||
|
||||
def __init__(self, vocab, unk_token, max_input_chars_per_word=100):
|
||||
self.vocab = vocab
|
||||
self.unk_token = unk_token
|
||||
self.max_input_chars_per_word = max_input_chars_per_word
|
||||
|
||||
def tokenize(self, text):
|
||||
"""
|
||||
Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform
|
||||
tokenization using the given vocabulary.
|
||||
|
||||
For example, :obj:`input = "unaffable"` wil return as output :obj:`["un", "##aff", "##able"]`.
|
||||
|
||||
Args:
|
||||
text: A single token or whitespace separated tokens. This should have
|
||||
already been passed through `BasicTokenizer`.
|
||||
|
||||
Returns:
|
||||
A list of wordpiece tokens.
|
||||
"""
|
||||
|
||||
output_tokens = []
|
||||
for token in whitespace_tokenize(text):
|
||||
chars = list(token)
|
||||
if len(chars) > self.max_input_chars_per_word:
|
||||
output_tokens.append(self.unk_token)
|
||||
continue
|
||||
|
||||
is_bad = False
|
||||
start = 0
|
||||
sub_tokens = []
|
||||
while start < len(chars):
|
||||
end = len(chars)
|
||||
cur_substr = None
|
||||
while start < end:
|
||||
substr = "".join(chars[start:end])
|
||||
if start > 0:
|
||||
substr = "##" + substr
|
||||
if substr in self.vocab:
|
||||
cur_substr = substr
|
||||
break
|
||||
end -= 1
|
||||
if cur_substr is None:
|
||||
is_bad = True
|
||||
break
|
||||
sub_tokens.append(cur_substr)
|
||||
start = end
|
||||
|
||||
if is_bad:
|
||||
output_tokens.append(self.unk_token)
|
||||
else:
|
||||
output_tokens.extend(sub_tokens)
|
||||
return output_tokens
|
||||
@@ -1,208 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The HuggingFace Inc. team, Microsoft Corporation.
|
||||
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Fast Tokenization classes for MPNet."""
|
||||
|
||||
import json
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from tokenizers import normalizers
|
||||
|
||||
from ...tokenization_utils import AddedToken
|
||||
from ...tokenization_utils_fast import PreTrainedTokenizerFast
|
||||
from ...utils import logging
|
||||
from .tokenization_mpnet import MPNetTokenizer
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}
|
||||
|
||||
PRETRAINED_VOCAB_FILES_MAP = {
|
||||
"vocab_file": {
|
||||
"microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt",
|
||||
},
|
||||
"tokenizer_file": {
|
||||
"microsoft/mpnet-base": "https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json",
|
||||
},
|
||||
}
|
||||
|
||||
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
|
||||
"microsoft/mpnet-base": 512,
|
||||
}
|
||||
|
||||
PRETRAINED_INIT_CONFIGURATION = {
|
||||
"microsoft/mpnet-base": {"do_lower_case": True},
|
||||
}
|
||||
|
||||
|
||||
class MPNetTokenizerFast(PreTrainedTokenizerFast):
|
||||
r"""
|
||||
Construct a "fast" MPNet tokenizer (backed by HuggingFace's `tokenizers` library). Based on WordPiece.
|
||||
|
||||
This tokenizer inherits from :class:`~transformers.PreTrainedTokenizerFast` which contains most of the main
|
||||
methods. Users should refer to this superclass for more information regarding those methods.
|
||||
|
||||
Args:
|
||||
vocab_file (:obj:`str`):
|
||||
File containing the vocabulary.
|
||||
do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to lowercase the input when tokenizing.
|
||||
bos_token (:obj:`str`, `optional`, defaults to :obj:`"<s>"`):
|
||||
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
|
||||
|
||||
.. note::
|
||||
|
||||
When building a sequence using special tokens, this is not the token that is used for the beginning of
|
||||
sequence. The token used is the :obj:`cls_token`.
|
||||
eos_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
|
||||
The end of sequence token.
|
||||
|
||||
.. note::
|
||||
|
||||
When building a sequence using special tokens, this is not the token that is used for the end of
|
||||
sequence. The token used is the :obj:`sep_token`.
|
||||
sep_token (:obj:`str`, `optional`, defaults to :obj:`"</s>"`):
|
||||
The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
|
||||
sequence classification or for a text and a question for question answering. It is also used as the last
|
||||
token of a sequence built with special tokens.
|
||||
cls_token (:obj:`str`, `optional`, defaults to :obj:`"<s>"`):
|
||||
The classifier token which is used when doing sequence classification (classification of the whole sequence
|
||||
instead of per-token classification). It is the first token of the sequence when built with special tokens.
|
||||
unk_token (:obj:`str`, `optional`, defaults to :obj:`"[UNK]"`):
|
||||
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
|
||||
token instead.
|
||||
pad_token (:obj:`str`, `optional`, defaults to :obj:`"<pad>"`):
|
||||
The token used for padding, for example when batching sequences of different lengths.
|
||||
mask_token (:obj:`str`, `optional`, defaults to :obj:`"<mask>"`):
|
||||
The token used for masking values. This is the token used when training this model with masked language
|
||||
modeling. This is the token which the model will try to predict.
|
||||
tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see `this
|
||||
issue <https://github.com/huggingface/transformers/issues/328>`__).
|
||||
strip_accents: (:obj:`bool`, `optional`):
|
||||
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
|
||||
value for :obj:`lowercase` (as in the original BERT).
|
||||
"""
|
||||
|
||||
vocab_files_names = VOCAB_FILES_NAMES
|
||||
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
|
||||
pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
|
||||
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
|
||||
slow_tokenizer_class = MPNetTokenizer
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_file,
|
||||
tokenizer_file=None,
|
||||
do_lower_case=True,
|
||||
bos_token="<s>",
|
||||
eos_token="</s>",
|
||||
sep_token="</s>",
|
||||
cls_token="<s>",
|
||||
unk_token="[UNK]",
|
||||
pad_token="<pad>",
|
||||
mask_token="<mask>",
|
||||
tokenize_chinese_chars=True,
|
||||
strip_accents=None,
|
||||
**kwargs
|
||||
):
|
||||
super().__init__(
|
||||
vocab_file,
|
||||
tokenizer_file=tokenizer_file,
|
||||
do_lower_case=do_lower_case,
|
||||
bos_token=bos_token,
|
||||
eos_token=eos_token,
|
||||
sep_token=sep_token,
|
||||
cls_token=cls_token,
|
||||
unk_token=unk_token,
|
||||
pad_token=pad_token,
|
||||
mask_token=mask_token,
|
||||
tokenize_chinese_chars=tokenize_chinese_chars,
|
||||
strip_accents=strip_accents,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
pre_tok_state = json.loads(self.backend_tokenizer.normalizer.__getstate__())
|
||||
if (
|
||||
pre_tok_state.get("do_lower_case", do_lower_case) != do_lower_case
|
||||
or pre_tok_state.get("strip_accents", strip_accents) != strip_accents
|
||||
):
|
||||
pre_tok_class = getattr(normalizers, pre_tok_state.pop("type"))
|
||||
pre_tok_state["do_lower_case"] = do_lower_case
|
||||
pre_tok_state["strip_accents"] = strip_accents
|
||||
self.backend_tokenizer.normalizer = pre_tok_class(**pre_tok_state)
|
||||
|
||||
self.do_lower_case = do_lower_case
|
||||
|
||||
@property
|
||||
def mask_token(self) -> str:
|
||||
"""
|
||||
:obj:`str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while
|
||||
not having been set.
|
||||
|
||||
MPNet tokenizer has a special mask token to be usble in the fill-mask pipeline. The mask token will greedily
|
||||
comprise the space before the `<mask>`.
|
||||
"""
|
||||
if self._mask_token is None and self.verbose:
|
||||
logger.error("Using mask_token, but it is not set yet.")
|
||||
return None
|
||||
return str(self._mask_token)
|
||||
|
||||
@mask_token.setter
|
||||
def mask_token(self, value):
|
||||
"""
|
||||
Overriding the default behavior of the mask token to have it eat the space before it.
|
||||
|
||||
This is needed to preserve backward compatibility with all the previously used models based on MPNet.
|
||||
"""
|
||||
# Mask token behave like a normal word, i.e. include the space before it
|
||||
# So we set lstrip to True
|
||||
value = AddedToken(value, lstrip=True, rstrip=False) if isinstance(value, str) else value
|
||||
self._mask_token = value
|
||||
|
||||
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
|
||||
output = [self.bos_token_id] + token_ids_0 + [self.eos_token_id]
|
||||
if token_ids_1 is None:
|
||||
return output
|
||||
|
||||
return output + [self.eos_token_id] + token_ids_1 + [self.eos_token_id]
|
||||
|
||||
def create_token_type_ids_from_sequences(
|
||||
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
|
||||
) -> List[int]:
|
||||
"""
|
||||
Creates a mask from the two sequences passed to be used in a sequence-pair classification task. MPNet does not
|
||||
make use of token type ids, therefore a list of zeros is returned
|
||||
|
||||
Args:
|
||||
token_ids_0 (:obj:`List[int]`):
|
||||
List of ids.
|
||||
token_ids_1 (:obj:`List[int]`, `optional`):
|
||||
Optional second list of IDs for sequence pairs
|
||||
|
||||
Returns:
|
||||
:obj:`List[int]`: List of zeros.
|
||||
"""
|
||||
sep = [self.sep_token_id]
|
||||
cls = [self.cls_token_id]
|
||||
|
||||
if token_ids_1 is None:
|
||||
return len(cls + token_ids_0 + sep) * [0]
|
||||
return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]
|
||||
|
||||
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
|
||||
files = self._tokenizer.model.save(save_directory, name=filename_prefix)
|
||||
return tuple(files)
|
||||
@@ -1226,8 +1226,6 @@ class RagTokenForGeneration(RagPreTrainedModel):
|
||||
early_stopping=None,
|
||||
use_cache=None,
|
||||
num_beams=None,
|
||||
num_beam_groups=None,
|
||||
diversity_penalty=None,
|
||||
bos_token_id=None,
|
||||
pad_token_id=None,
|
||||
eos_token_id=None,
|
||||
@@ -1304,13 +1302,6 @@ class RagTokenForGeneration(RagPreTrainedModel):
|
||||
should not appear in the generated text, use :obj:`tokenizer.encode(bad_word, add_prefix_space=True)`.
|
||||
num_beams (:obj:`int`, `optional`, defaults to 1):
|
||||
Number of beams for beam search. 1 means no beam search.
|
||||
num_beam_groups (:obj:`int`, `optional`, defaults to 1):
|
||||
Number of groups to divide :obj:`num_beams` into in order to ensure diversity among different groups of
|
||||
beams. `this paper <https://arxiv.org/pdf/1610.02424.pdf>`__ for more details.
|
||||
diversity_penalty (:obj:`float`, `optional`, defaults to 0.0):
|
||||
This value is subtracted from a beam's score if it generates a token same as any beam from other group
|
||||
at a particular time. Note that :obj:`diversity_penalty` is only effective if ``group beam search`` is
|
||||
enabled.
|
||||
num_return_sequences(:obj:`int`, `optional`, defaults to 1):
|
||||
The number of independently computed returned sequences for each element in the batch. Note that this
|
||||
is not the value we pass to the ``generator``'s `:func:`~transformers.PreTrainedModel.generate`
|
||||
@@ -1335,7 +1326,6 @@ class RagTokenForGeneration(RagPreTrainedModel):
|
||||
# set default parameters
|
||||
n_docs = n_docs if n_docs is not None else self.config.n_docs
|
||||
num_beams = num_beams if num_beams is not None else self.config.num_beams
|
||||
num_beam_groups = num_beam_groups if num_beam_groups is not None else self.config.num_beam_groups
|
||||
max_length = max_length if max_length is not None else self.config.max_length
|
||||
num_return_sequences = (
|
||||
num_return_sequences if num_return_sequences is not None else self.config.num_return_sequences
|
||||
@@ -1422,8 +1412,6 @@ class RagTokenForGeneration(RagPreTrainedModel):
|
||||
eos_token_id=eos_token_id,
|
||||
prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
|
||||
num_beams=num_beams,
|
||||
num_beam_groups=num_beam_groups,
|
||||
diversity_penalty=diversity_penalty,
|
||||
)
|
||||
|
||||
if num_beams == 1:
|
||||
|
||||
@@ -12,14 +12,13 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from typing import Callable, Dict, Tuple
|
||||
from typing import Callable, Dict
|
||||
|
||||
import numpy as np
|
||||
|
||||
import flax.linen as nn
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
from jax.random import PRNGKey
|
||||
|
||||
from ...file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
|
||||
from ...modeling_flax_utils import FlaxPreTrainedModel, gelu
|
||||
@@ -102,8 +101,8 @@ class FlaxRobertaLayerNorm(nn.Module):
|
||||
scale: bool = True # If True, multiply by scale (gamma). When the next layer is linear
|
||||
# (also e.g. nn.relu), this can be disabled since the scaling will be
|
||||
# done by the next layer.
|
||||
scale_init: Callable[..., np.ndarray] = jax.nn.initializers.ones
|
||||
bias_init: Callable[..., np.ndarray] = jax.nn.initializers.zeros
|
||||
bias_init: jnp.ndarray = nn.initializers.zeros
|
||||
scale_init: jnp.ndarray = nn.initializers.ones
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, x):
|
||||
@@ -123,13 +122,11 @@ class FlaxRobertaLayerNorm(nn.Module):
|
||||
mean2 = jnp.mean(jax.lax.square(x), axis=-1, keepdims=True)
|
||||
var = mean2 - jax.lax.square(mean)
|
||||
mul = jax.lax.rsqrt(var + self.epsilon)
|
||||
|
||||
if self.scale:
|
||||
mul = mul * jnp.asarray(self.param("gamma", self.scale_init, (features,)))
|
||||
mul = mul * jnp.asarray(self.param("gamma", self.scale_init, (features,)), self.dtype)
|
||||
y = (x - mean) * mul
|
||||
|
||||
if self.bias:
|
||||
y = y + jnp.asarray(self.param("beta", self.bias_init, (features,)))
|
||||
y = y + jnp.asarray(self.param("beta", self.bias_init, (features,)), self.dtype)
|
||||
return y
|
||||
|
||||
|
||||
@@ -142,9 +139,7 @@ class FlaxRobertaEmbedding(nn.Module):
|
||||
|
||||
vocab_size: int
|
||||
hidden_size: int
|
||||
kernel_init_scale: float = 0.2
|
||||
emb_init: Callable[..., np.ndarray] = jax.nn.initializers.normal(stddev=kernel_init_scale)
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
emb_init: Callable[..., np.ndarray] = nn.initializers.normal(stddev=0.1)
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, inputs):
|
||||
@@ -160,108 +155,66 @@ class FlaxRobertaEmbeddings(nn.Module):
|
||||
hidden_size: int
|
||||
type_vocab_size: int
|
||||
max_length: int
|
||||
kernel_init_scale: float = 0.2
|
||||
dropout_rate: float = 0.0
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, input_ids, token_type_ids, position_ids, attention_mask, deterministic: bool = True):
|
||||
def __call__(self, input_ids, token_type_ids, position_ids, attention_mask):
|
||||
|
||||
# Embed
|
||||
w_emb = FlaxRobertaEmbedding(
|
||||
self.vocab_size,
|
||||
self.hidden_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
name="word_embeddings",
|
||||
dtype=self.dtype,
|
||||
)(jnp.atleast_2d(input_ids.astype("i4")))
|
||||
p_emb = FlaxRobertaEmbedding(
|
||||
self.max_length,
|
||||
self.hidden_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
name="position_embeddings",
|
||||
dtype=self.dtype,
|
||||
)(jnp.atleast_2d(position_ids.astype("i4")))
|
||||
t_emb = FlaxRobertaEmbedding(
|
||||
self.type_vocab_size,
|
||||
self.hidden_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
name="token_type_embeddings",
|
||||
dtype=self.dtype,
|
||||
)(jnp.atleast_2d(token_type_ids.astype("i4")))
|
||||
w_emb = FlaxRobertaEmbedding(self.vocab_size, self.hidden_size, name="word_embeddings")(
|
||||
jnp.atleast_2d(input_ids.astype("i4"))
|
||||
)
|
||||
p_emb = FlaxRobertaEmbedding(self.max_length, self.hidden_size, name="position_embeddings")(
|
||||
jnp.atleast_2d(position_ids.astype("i4"))
|
||||
)
|
||||
t_emb = FlaxRobertaEmbedding(self.type_vocab_size, self.hidden_size, name="token_type_embeddings")(
|
||||
jnp.atleast_2d(token_type_ids.astype("i4"))
|
||||
)
|
||||
|
||||
# Sum all embeddings
|
||||
summed_emb = w_emb + jnp.broadcast_to(p_emb, w_emb.shape) + t_emb
|
||||
|
||||
# Layer Norm
|
||||
layer_norm = FlaxRobertaLayerNorm(name="layer_norm", dtype=self.dtype)(summed_emb)
|
||||
embeddings = nn.Dropout(rate=self.dropout_rate)(layer_norm, deterministic=deterministic)
|
||||
return embeddings
|
||||
layer_norm = FlaxRobertaLayerNorm(name="layer_norm")(summed_emb)
|
||||
|
||||
return layer_norm
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_flax_bert.FlaxBertAttention with Bert->Roberta
|
||||
class FlaxRobertaAttention(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state, attention_mask, deterministic: bool = True):
|
||||
def __call__(self, hidden_state, attention_mask):
|
||||
# Attention mask comes in as attention_mask.shape == (*batch_sizes, kv_length)
|
||||
# FLAX expects: attention_mask.shape == (*batch_sizes, 1, 1, kv_length) such that it is broadcastable
|
||||
# with attn_weights.shape == (*batch_sizes, num_heads, q_length, kv_length)
|
||||
attention_mask = jnp.expand_dims(attention_mask, axis=(-3, -2))
|
||||
self_att = nn.attention.SelfAttention(
|
||||
num_heads=self.num_heads,
|
||||
qkv_features=self.head_size,
|
||||
dropout_rate=self.dropout_rate,
|
||||
deterministic=deterministic,
|
||||
kernel_init=jax.nn.initializers.normal(self.kernel_init_scale, self.dtype),
|
||||
bias_init=jax.nn.initializers.zeros,
|
||||
name="self",
|
||||
dtype=self.dtype,
|
||||
)(hidden_state, attention_mask)
|
||||
self_att = nn.attention.SelfAttention(num_heads=self.num_heads, qkv_features=self.head_size, name="self")(
|
||||
hidden_state, attention_mask
|
||||
)
|
||||
|
||||
layer_norm = FlaxRobertaLayerNorm(name="layer_norm", dtype=self.dtype)(self_att + hidden_state)
|
||||
layer_norm = FlaxRobertaLayerNorm(name="layer_norm")(self_att + hidden_state)
|
||||
return layer_norm
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_flax_bert.FlaxBertIntermediate with Bert->Roberta
|
||||
class FlaxRobertaIntermediate(nn.Module):
|
||||
output_size: int
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state):
|
||||
# TODO: Add ACT2FN reference to change activation function
|
||||
dense = nn.Dense(
|
||||
features=self.output_size,
|
||||
kernel_init=jax.nn.initializers.normal(self.kernel_init_scale, self.dtype),
|
||||
name="dense",
|
||||
dtype=self.dtype,
|
||||
)(hidden_state)
|
||||
dense = nn.Dense(features=self.output_size, name="dense")(hidden_state)
|
||||
return gelu(dense)
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_flax_bert.FlaxBertOutput with Bert->Roberta
|
||||
class FlaxRobertaOutput(nn.Module):
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, intermediate_output, attention_output, deterministic: bool = True):
|
||||
hidden_state = nn.Dense(
|
||||
attention_output.shape[-1],
|
||||
kernel_init=jax.nn.initializers.normal(self.kernel_init_scale, self.dtype),
|
||||
name="dense",
|
||||
dtype=self.dtype,
|
||||
)(intermediate_output)
|
||||
hidden_state = nn.Dropout(rate=self.dropout_rate)(hidden_state, deterministic=deterministic)
|
||||
hidden_state = FlaxRobertaLayerNorm(name="layer_norm", dtype=self.dtype)(hidden_state + attention_output)
|
||||
def __call__(self, intermediate_output, attention_output):
|
||||
hidden_state = nn.Dense(attention_output.shape[-1], name="dense")(intermediate_output)
|
||||
hidden_state = FlaxRobertaLayerNorm(name="layer_norm")(hidden_state + attention_output)
|
||||
return hidden_state
|
||||
|
||||
|
||||
@@ -269,29 +222,14 @@ class FlaxRobertaLayer(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
intermediate_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state, attention_mask, deterministic: bool = True):
|
||||
attention = FlaxRobertaAttention(
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name="attention",
|
||||
dtype=self.dtype,
|
||||
)(hidden_state, attention_mask, deterministic=deterministic)
|
||||
intermediate = FlaxRobertaIntermediate(
|
||||
self.intermediate_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
name="intermediate",
|
||||
dtype=self.dtype,
|
||||
)(attention)
|
||||
output = FlaxRobertaOutput(
|
||||
kernel_init_scale=self.kernel_init_scale, dropout_rate=self.dropout_rate, name="output", dtype=self.dtype
|
||||
)(intermediate, attention, deterministic=deterministic)
|
||||
def __call__(self, hidden_state, attention_mask):
|
||||
attention = FlaxRobertaAttention(self.num_heads, self.head_size, name="attention")(
|
||||
hidden_state, attention_mask
|
||||
)
|
||||
intermediate = FlaxRobertaIntermediate(self.intermediate_size, name="intermediate")(attention)
|
||||
output = FlaxRobertaOutput(name="output")(intermediate, attention)
|
||||
|
||||
return output
|
||||
|
||||
@@ -306,12 +244,9 @@ class FlaxRobertaLayerCollection(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
intermediate_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, inputs, attention_mask, deterministic: bool = True):
|
||||
def __call__(self, inputs, attention_mask):
|
||||
assert self.num_layers > 0, f"num_layers should be >= 1, got ({self.num_layers})"
|
||||
|
||||
# Initialize input / output
|
||||
@@ -319,16 +254,8 @@ class FlaxRobertaLayerCollection(nn.Module):
|
||||
|
||||
# Forward over all encoders
|
||||
for i in range(self.num_layers):
|
||||
layer = FlaxRobertaLayer(
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
self.intermediate_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name=f"{i}",
|
||||
dtype=self.dtype,
|
||||
)
|
||||
input_i = layer(input_i, attention_mask, deterministic=deterministic)
|
||||
layer = FlaxRobertaLayer(self.num_heads, self.head_size, self.intermediate_size, name=f"{i}")
|
||||
input_i = layer(input_i, attention_mask)
|
||||
return input_i
|
||||
|
||||
|
||||
@@ -338,40 +265,22 @@ class FlaxRobertaEncoder(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
intermediate_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state, attention_mask, deterministic: bool = True):
|
||||
def __call__(self, hidden_state, attention_mask):
|
||||
layer = FlaxRobertaLayerCollection(
|
||||
self.num_layers,
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
self.intermediate_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name="layer",
|
||||
dtype=self.dtype,
|
||||
)(hidden_state, attention_mask, deterministic=deterministic)
|
||||
self.num_layers, self.num_heads, self.head_size, self.intermediate_size, name="layer"
|
||||
)(hidden_state, attention_mask)
|
||||
return layer
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_flax_bert.FlaxBertPooler with Bert->Roberta
|
||||
class FlaxRobertaPooler(nn.Module):
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, hidden_state):
|
||||
cls_token = hidden_state[:, 0]
|
||||
out = nn.Dense(
|
||||
hidden_state.shape[-1],
|
||||
kernel_init=jax.nn.initializers.normal(self.kernel_init_scale, self.dtype),
|
||||
name="dense",
|
||||
dtype=self.dtype,
|
||||
)(cls_token)
|
||||
return nn.tanh(out)
|
||||
out = nn.Dense(hidden_state.shape[-1], name="dense")(cls_token)
|
||||
return jax.lax.tanh(out)
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_flax_bert.FlaxBertModule with Bert->Roberta
|
||||
@@ -384,38 +293,21 @@ class FlaxRobertaModule(nn.Module):
|
||||
num_heads: int
|
||||
head_size: int
|
||||
intermediate_size: int
|
||||
dropout_rate: float = 0.0
|
||||
kernel_init_scale: float = 0.2
|
||||
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
|
||||
|
||||
@nn.compact
|
||||
def __call__(self, input_ids, attention_mask, token_type_ids, position_ids, deterministic: bool = True):
|
||||
def __call__(self, input_ids, attention_mask, token_type_ids, position_ids):
|
||||
|
||||
# Embedding
|
||||
embeddings = FlaxRobertaEmbeddings(
|
||||
self.vocab_size,
|
||||
self.hidden_size,
|
||||
self.type_vocab_size,
|
||||
self.max_length,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name="embeddings",
|
||||
dtype=self.dtype,
|
||||
)(input_ids, token_type_ids, position_ids, attention_mask, deterministic=deterministic)
|
||||
self.vocab_size, self.hidden_size, self.type_vocab_size, self.max_length, name="embeddings"
|
||||
)(input_ids, token_type_ids, position_ids, attention_mask)
|
||||
|
||||
# N stacked encoding layers
|
||||
encoder = FlaxRobertaEncoder(
|
||||
self.num_encoder_layers,
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
self.intermediate_size,
|
||||
kernel_init_scale=self.kernel_init_scale,
|
||||
dropout_rate=self.dropout_rate,
|
||||
name="encoder",
|
||||
dtype=self.dtype,
|
||||
)(embeddings, attention_mask, deterministic=deterministic)
|
||||
self.num_encoder_layers, self.num_heads, self.head_size, self.intermediate_size, name="encoder"
|
||||
)(embeddings, attention_mask)
|
||||
|
||||
pooled = FlaxRobertaPooler(kernel_init_scale=self.kernel_init_scale, name="pooler", dtype=self.dtype)(encoder)
|
||||
pooled = FlaxRobertaPooler(name="pooler")(encoder)
|
||||
return encoder, pooled
|
||||
|
||||
|
||||
@@ -504,8 +396,8 @@ class FlaxRobertaModel(FlaxPreTrainedModel):
|
||||
|
||||
return jax_state
|
||||
|
||||
def __init__(self, config: RobertaConfig, state: dict, seed: int = 0, dtype: jnp.dtype = jnp.float32):
|
||||
module = FlaxRobertaModule(
|
||||
def __init__(self, config: RobertaConfig, state: dict, seed: int = 0, **kwargs):
|
||||
model = FlaxRobertaModule(
|
||||
vocab_size=config.vocab_size,
|
||||
hidden_size=config.hidden_size,
|
||||
type_vocab_size=config.type_vocab_size,
|
||||
@@ -514,78 +406,31 @@ class FlaxRobertaModel(FlaxPreTrainedModel):
|
||||
num_heads=config.num_attention_heads,
|
||||
head_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
dropout_rate=config.hidden_dropout_prob,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
super().__init__(config, module, state, seed)
|
||||
super().__init__(config, model, state, seed)
|
||||
|
||||
@property
|
||||
def module(self) -> nn.Module:
|
||||
return self._module
|
||||
|
||||
@add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
|
||||
def __call__(
|
||||
self,
|
||||
input_ids,
|
||||
token_type_ids=None,
|
||||
attention_mask=None,
|
||||
position_ids=None,
|
||||
params: dict = None,
|
||||
dropout_rng: PRNGKey = None,
|
||||
train: bool = False,
|
||||
):
|
||||
input_ids, attention_mask, token_type_ids, position_ids = self._check_inputs(
|
||||
input_ids, attention_mask, token_type_ids, position_ids
|
||||
)
|
||||
|
||||
# Handle any PRNG if needed
|
||||
rngs = {}
|
||||
if dropout_rng is not None:
|
||||
rngs["dropout"] = dropout_rng
|
||||
|
||||
return self.module.apply(
|
||||
{"params": params or self.params},
|
||||
jnp.array(input_ids, dtype="i4"),
|
||||
jnp.array(attention_mask, dtype="i4"),
|
||||
jnp.array(token_type_ids, dtype="i4"),
|
||||
jnp.array(position_ids, dtype="i4"),
|
||||
not train,
|
||||
rngs=rngs,
|
||||
)
|
||||
|
||||
def init(self, rng: jax.random.PRNGKey, input_shape: Tuple):
|
||||
input_ids, attention_mask, token_type_ids, position_ids = self._check_inputs(
|
||||
jnp.zeros(input_shape, dtype="i4"), None, None, None
|
||||
)
|
||||
|
||||
params_rng, dropout_rng = jax.random.split(rng)
|
||||
rngs = {"params": params_rng, "dropout": dropout_rng}
|
||||
|
||||
self.params = self.module.init(rngs, input_ids, attention_mask, token_type_ids, position_ids)["params"]
|
||||
|
||||
def _check_inputs(self, input_ids, attention_mask, token_type_ids, position_ids):
|
||||
|
||||
def __call__(self, input_ids, token_type_ids=None, attention_mask=None, position_ids=None):
|
||||
if token_type_ids is None:
|
||||
token_type_ids = jnp.ones_like(input_ids)
|
||||
|
||||
if position_ids is None:
|
||||
position_ids = create_position_ids_from_input_ids(input_ids, self.config.pad_token_id)
|
||||
position_ids = jnp.arange(
|
||||
self.config.pad_token_id + 1, jnp.atleast_2d(input_ids).shape[-1] + self.config.pad_token_id + 1
|
||||
)
|
||||
|
||||
if attention_mask is None:
|
||||
attention_mask = jnp.ones_like(input_ids)
|
||||
|
||||
return input_ids, attention_mask, token_type_ids, position_ids
|
||||
|
||||
|
||||
def create_position_ids_from_input_ids(input_ids, padding_idx):
|
||||
"""
|
||||
Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
|
||||
are ignored. This is modified from fairseq's `utils.make_positions`.
|
||||
|
||||
Args:
|
||||
input_ids: jnp.ndarray
|
||||
padding_idx: int
|
||||
|
||||
Returns: jnp.ndarray
|
||||
"""
|
||||
# The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA.
|
||||
mask = (input_ids != padding_idx).astype("i4")
|
||||
incremental_indices = jnp.cumsum(mask, axis=1).astype("i4") * mask
|
||||
return incremental_indices.astype("i4") + padding_idx
|
||||
return self.model.apply(
|
||||
{"params": self.params},
|
||||
jnp.array(input_ids, dtype="i4"),
|
||||
jnp.array(attention_mask, dtype="i4"),
|
||||
jnp.array(token_type_ids, dtype="i4"),
|
||||
jnp.array(position_ids, dtype="i4"),
|
||||
)
|
||||
@@ -813,6 +813,9 @@ class T5Stack(T5PreTrainedModel):
|
||||
def get_input_embeddings(self):
|
||||
return self.embed_tokens
|
||||
|
||||
def get_output_embeddings(self):
|
||||
return self.embed_tokens
|
||||
|
||||
def set_input_embeddings(self, new_embeddings):
|
||||
self.embed_tokens = new_embeddings
|
||||
|
||||
|
||||
@@ -6,12 +6,11 @@ from ...file_utils import is_torch_available
|
||||
from .configuration_tapas import TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP, TapasConfig
|
||||
from .tokenization_tapas import TapasTokenizer
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
from .modeling_tapas import (
|
||||
TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
TapasModel,
|
||||
TapasForMaskedLM,
|
||||
TapasForQuestionAnswering,
|
||||
TapasForSequenceClassification,
|
||||
TapasModel,
|
||||
)
|
||||
)
|
||||
@@ -18,12 +18,10 @@
|
||||
from ...configuration_utils import PretrainedConfig
|
||||
|
||||
|
||||
TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP = {
|
||||
"nielsr/tapas-base-finetuned-sqa": "https://huggingface.co/nielsr/tapas-base-finetuned-sqa/resolve/main/config.json",
|
||||
"nielsr/tapas-base-finetuned-wtq": "https://huggingface.co/nielsr/tapas-base-finetuned-wtq/resolve/main/config.json",
|
||||
"nielsr/tapas-base-finetuned-wikisql-supervised": "https://huggingface.co/nielsr/tapas-base-finetuned-wikisql-supervised/resolve/main/config.json",
|
||||
"nielsr/tapas-base-finetuned-tabfact": "https://huggingface.co/nielsr/tapas-base-finetuned-tabfact/resolve/main/config.json",
|
||||
}
|
||||
TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP = {"nielsr/tapas-base-finetuned-sqa": "https://huggingface.co/nielsr/tapas-base-finetuned-sqa/resolve/main/config.json",
|
||||
"nielsr/tapas-base-finetuned-wtq": "https://huggingface.co/nielsr/tapas-base-finetuned-wtq/resolve/main/config.json",
|
||||
"nielsr/tapas-base-finetuned-wikisql-supervised": "https://huggingface.co/nielsr/tapas-base-finetuned-wikisql-supervised/resolve/main/config.json",
|
||||
"nielsr/tapas-base-finetuned-tabfact": "https://huggingface.co/nielsr/tapas-base-finetuned-tabfact/resolve/main/config.json"}
|
||||
|
||||
|
||||
class TapasConfig(PretrainedConfig):
|
||||
@@ -90,7 +88,7 @@ class TapasConfig(PretrainedConfig):
|
||||
use_gumbel_for_aggregation: (:obj:`bool`, `optional`, defaults to :obj:`False`):
|
||||
Whether to apply Gumbel-Softmax to aggregation selection.
|
||||
average_approximation_function: (:obj:`string`, `optional`, defaults to :obj:`"ratio"`):
|
||||
Method to calculate the expected average of cells in the weak supervision case. One of :obj:`"ratio"`,
|
||||
Method to calculate the expected average of cells in the weak supervision case. One of :obj:`"ratio"`,
|
||||
:obj:`"first_order"` or :obj:`"second_order"`.
|
||||
cell_selection_preference: (:obj:`float`, `optional`):
|
||||
Preference for cell selection in ambiguous cases. Only applicable in case of weak supervision for
|
||||
@@ -208,4 +206,4 @@ class TapasConfig(PretrainedConfig):
|
||||
self.allow_empty_column_selection = allow_empty_column_selection
|
||||
self.init_cell_selection_weights_to_zero = init_cell_selection_weights_to_zero
|
||||
self.reset_position_index_per_cell = reset_position_index_per_cell
|
||||
self.disable_per_token_loss = disable_per_token_loss
|
||||
self.disable_per_token_loss = disable_per_token_loss
|
||||
@@ -19,9 +19,13 @@ import argparse
|
||||
|
||||
import torch
|
||||
|
||||
from transformers import TapasForQuestionAnswering # noqa F401
|
||||
from transformers import TapasForSequenceClassification # noqa F401
|
||||
from transformers import TapasConfig, TapasModel, load_tf_weights_in_tapas
|
||||
from transformers import (
|
||||
TapasConfig,
|
||||
TapasModel,
|
||||
TapasForQuestionAnswering,
|
||||
TapasForSequenceClassification,
|
||||
load_tf_weights_in_tapas,
|
||||
)
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
@@ -84,7 +88,7 @@ def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, tapas_config_file, pyto
|
||||
|
||||
print("Building PyTorch model from configuration: {}".format(str(config)))
|
||||
model = TapasModel(config)
|
||||
# model = TapasForQuestionAnswering(config)
|
||||
#model = TapasForQuestionAnswering(config)
|
||||
# model = TapasForSequenceClassification(config)
|
||||
|
||||
# Load weights from tf checkpoint
|
||||
@@ -113,4 +117,4 @@ if __name__ == "__main__":
|
||||
"--pytorch_dump_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.tapas_config_file, args.pytorch_dump_path)
|
||||
convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.tapas_config_file, args.pytorch_dump_path)
|
||||
@@ -26,6 +26,7 @@ import torch.nn as nn
|
||||
from torch.nn import CrossEntropyLoss, MSELoss
|
||||
|
||||
from ...activations import ACT2FN
|
||||
from .configuration_tapas import TapasConfig
|
||||
from ...file_utils import (
|
||||
ModelOutput,
|
||||
add_start_docstrings,
|
||||
@@ -42,8 +43,6 @@ from ...modeling_utils import (
|
||||
prune_linear_layer,
|
||||
)
|
||||
from ...utils import logging
|
||||
from .configuration_tapas import TapasConfig
|
||||
|
||||
|
||||
# soft dependency
|
||||
if is_scatter_available():
|
||||
@@ -55,36 +54,9 @@ _CONFIG_FOR_DOC = "TapasConfig"
|
||||
_TOKENIZER_FOR_DOC = "TapasTokenizer"
|
||||
|
||||
TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST = [
|
||||
# large models
|
||||
"nielsr/tapas-large",
|
||||
"nielsr/tapas-large-finetuned-sqa",
|
||||
"nielsr/tapas-large-finetuned-wtq",
|
||||
"nielsr/tapas-large-finetuned-wikisql-supervised",
|
||||
"nielsr/tapas-large-finetuned-tabfact",
|
||||
# base models
|
||||
"nielsr/tapas-base",
|
||||
"nielsr/tapas-base-finetuned-sqa",
|
||||
"nielsr/tapas-base-finetuned-wtq",
|
||||
"nielsr/tapas-base-finetuned-wikisql-supervised",
|
||||
"nielsr/tapas-base-finetuned-tabfact",
|
||||
# small models
|
||||
"nielsr/tapas-small",
|
||||
"nielsr/tapas-small-finetuned-sqa",
|
||||
"nielsr/tapas-small-finetuned-wtq",
|
||||
"nielsr/tapas-small-finetuned-wikisql-supervised",
|
||||
"nielsr/tapas-small-finetuned-tabfact",
|
||||
# mini models
|
||||
"nielsr/tapas-mini",
|
||||
"nielsr/tapas-mini-finetuned-sqa",
|
||||
"nielsr/tapas-mini-finetuned-wtq",
|
||||
"nielsr/tapas-mini-finetuned-wikisql-supervised",
|
||||
"nielsr/tapas-mini-finetuned-tabfact",
|
||||
# tiny models
|
||||
"nielsr/tapas-tiny",
|
||||
"nielsr/tapas-tiny-finetuned-sqa",
|
||||
"nielsr/tapas-tiny-finetuned-wtq",
|
||||
"nielsr/tapas-tiny-finetuned-wikisql-supervised",
|
||||
"nielsr/tapas-tiny-finetuned-tabfact",
|
||||
# See all TAPAS models at https://huggingface.co/models?filter=tapas
|
||||
]
|
||||
|
||||
@@ -184,7 +156,7 @@ def load_tf_weights_in_tapas(model, config, tf_checkpoint_path):
|
||||
logger.info("Skipping {}".format("/".join(name)))
|
||||
continue
|
||||
# in case the model is TapasModel, we skip output_bias, output_weights, output_bias_cls and output_weights_cls
|
||||
# since this model does not have MLM and NSP heads
|
||||
# since this model does not have MLM and NSP heads
|
||||
if isinstance(model, TapasModel):
|
||||
if any(
|
||||
n
|
||||
@@ -358,6 +330,7 @@ class TapasEmbeddings(nn.Module):
|
||||
return embeddings
|
||||
|
||||
|
||||
# Copied from transformers.modeling_bert.BertSelfAttention with Bert->Tapas
|
||||
class TapasSelfAttention(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
@@ -436,7 +409,7 @@ class TapasSelfAttention(nn.Module):
|
||||
return outputs
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_bert.BertSelfOutput
|
||||
# Copied from transformers.modeling_bert.BertSelfOutput
|
||||
class TapasSelfOutput(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
@@ -451,7 +424,7 @@ class TapasSelfOutput(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_bert.BertAttention with Bert->Tapas
|
||||
# Copied from transformers.modeling_bert.BertAttention with Bert->Tapas
|
||||
class TapasAttention(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
@@ -499,7 +472,7 @@ class TapasAttention(nn.Module):
|
||||
return outputs
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_bert.BertIntermediate
|
||||
# Copied from transformers.modeling_bert.BertIntermediate
|
||||
class TapasIntermediate(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
@@ -515,7 +488,7 @@ class TapasIntermediate(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_bert.BertOutput
|
||||
# Copied from transformers.modeling_bert.BertOutput
|
||||
class TapasOutput(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
@@ -530,7 +503,7 @@ class TapasOutput(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_bert.BertLayer with Bert->Tapas
|
||||
# Copied from transformers.modeling_bert.BertLayer with Bert->Tapas
|
||||
class TapasLayer(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
@@ -590,6 +563,7 @@ class TapasLayer(nn.Module):
|
||||
return layer_output
|
||||
|
||||
|
||||
# Copied from transformers.modeling_bert.BertEncoder with Bert->Tapas
|
||||
class TapasEncoder(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
@@ -654,7 +628,7 @@ class TapasEncoder(nn.Module):
|
||||
)
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_bert.BertPooler
|
||||
# Copied from transformers.modeling_bert.BertPooler
|
||||
class TapasPooler(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
@@ -679,7 +653,7 @@ class TapasPreTrainedModel(PreTrainedModel):
|
||||
config_class = TapasConfig
|
||||
base_model_prefix = "tapas"
|
||||
|
||||
# Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights
|
||||
# Copied from transformers.modeling_bert.BertPreTrainedModel._init_weights
|
||||
def _init_weights(self, module):
|
||||
""" Initialize the weights """
|
||||
if isinstance(module, (nn.Linear, nn.Embedding)):
|
||||
@@ -712,10 +686,11 @@ TAPAS_START_DOCSTRING = r"""
|
||||
TAPAS_INPUTS_DOCSTRING = r"""
|
||||
Args:
|
||||
input_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`):
|
||||
Indices of input sequence tokens in the vocabulary. Indices can be obtained using
|
||||
:class:`~transformers.TapasTokenizer`. See :meth:`transformers.PreTrainedTokenizer.encode` and
|
||||
:meth:`transformers.PreTrainedTokenizer.__call__` for details.
|
||||
|
||||
Indices of input sequence tokens in the vocabulary.
|
||||
Indices can be obtained using :class:`~transformers.TapasTokenizer`. See
|
||||
:meth:`transformers.PreTrainedTokenizer.encode` and :meth:`transformers.PreTrainedTokenizer.__call__` for
|
||||
details.
|
||||
|
||||
`What are input IDs? <../glossary.html#input-ids>`__
|
||||
attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`):
|
||||
Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``:
|
||||
@@ -725,19 +700,20 @@ TAPAS_INPUTS_DOCSTRING = r"""
|
||||
|
||||
`What are attention masks? <../glossary.html#attention-mask>`__
|
||||
token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0}, 7)`, `optional`):
|
||||
Token indices that encode tabular structure. Indices can be obtained using
|
||||
:class:`~transformers.TapasTokenizer`. See this class for more info.
|
||||
|
||||
Token indices that encode tabular structure. Indices can be obtained using :class:`~transformers.TapasTokenizer`.
|
||||
See this class for more info.
|
||||
|
||||
`What are token type IDs? <../glossary.html#token-type-ids>`_
|
||||
position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`):
|
||||
Indices of positions of each input sequence tokens in the position embeddings. If
|
||||
``reset_position_index_per_cell`` of :class:`~transformers.TapasConfig` is set to ``True``, relative
|
||||
position embeddings will be used. Selected in the range ``[0, config.max_position_embeddings - 1]``.
|
||||
|
||||
Indices of positions of each input sequence tokens in the position embeddings. If ``reset_position_index_per_cell``
|
||||
of :class:`~transformers.TapasConfig` is set to ``True``, relative position embeddings will be used. Selected in the
|
||||
range ``[0, config.max_position_embeddings - 1]``.
|
||||
|
||||
`What are position IDs? <../glossary.html#position-ids>`_
|
||||
head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`):
|
||||
Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: - 1
|
||||
indicates the head is **not masked**, - 0 indicates the head is **masked**.
|
||||
Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``:
|
||||
- 1 indicates the head is **not masked**,
|
||||
- 0 indicates the head is **masked**.
|
||||
inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`):
|
||||
Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation.
|
||||
This is useful if you want more control over how to convert :obj:`input_ids` indices into associated
|
||||
@@ -824,8 +800,8 @@ class TapasModel(TapasPreTrainedModel):
|
||||
>>> tokenizer = TapasTokenizer.from_pretrained('google/tapas-base-uncased')
|
||||
>>> model = TapasModel.from_pretrained('google/tapas-base-uncased')
|
||||
|
||||
>>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
|
||||
... 'Age': ["56", "45", "59"],
|
||||
>>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
|
||||
... 'Age': ["56", "45", "59"],
|
||||
... 'Number of movies': ["87", "53", "69"]
|
||||
... }
|
||||
>>> table = pd.DataFrame.from_dict(data)
|
||||
@@ -925,9 +901,6 @@ class TapasForMaskedLM(TapasPreTrainedModel):
|
||||
def get_output_embeddings(self):
|
||||
return self.lm_head
|
||||
|
||||
def set_output_embeddings(self, word_embeddings):
|
||||
self.lm_head = word_embeddings
|
||||
|
||||
@add_start_docstrings_to_model_forward(TAPAS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
|
||||
@replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC)
|
||||
def forward(
|
||||
@@ -962,8 +935,8 @@ class TapasForMaskedLM(TapasPreTrainedModel):
|
||||
>>> tokenizer = TapasTokenizer.from_pretrained('google/tapas-base-uncased')
|
||||
>>> model = TapasForMaskedLM.from_pretrained('google/tapas-base-uncased')
|
||||
|
||||
>>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
|
||||
... 'Age': ["56", "45", "59"],
|
||||
>>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
|
||||
... 'Age': ["56", "45", "59"],
|
||||
... 'Number of movies': ["87", "53", "69"]
|
||||
... }
|
||||
>>> table = pd.DataFrame.from_dict(data)
|
||||
@@ -1010,11 +983,37 @@ class TapasForMaskedLM(TapasPreTrainedModel):
|
||||
)
|
||||
|
||||
|
||||
# Copied from transformers.modeling_roberta.RobertaLMHead with Roberta->Tapas
|
||||
class TapasLMHead(nn.Module):
|
||||
"""Tapas Head for masked language modeling."""
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
|
||||
|
||||
self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
||||
self.bias = nn.Parameter(torch.zeros(config.vocab_size))
|
||||
|
||||
# Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
|
||||
self.decoder.bias = self.bias
|
||||
|
||||
def forward(self, features, **kwargs):
|
||||
x = self.dense(features)
|
||||
x = gelu(x)
|
||||
x = self.layer_norm(x)
|
||||
|
||||
# project back to size of vocabulary with bias
|
||||
x = self.decoder(x)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
@add_start_docstrings(
|
||||
"""
|
||||
Tapas Model with a cell selection head and optional aggregation head on top for question-answering tasks on tables
|
||||
(linear layers on top of the hidden-states output to compute `logits` and optional `logits_aggregation`), e.g. for
|
||||
SQA, WTQ or WikiSQL-supervised tasks.
|
||||
Tapas Model with a cell selection head and optional aggregation head on top for question-answering tasks on
|
||||
tables (linear layers on top of the hidden-states output to compute `logits` and optional `logits_aggregation`),
|
||||
e.g. for SQA, WTQ or WikiSQL-supervised tasks.
|
||||
""",
|
||||
TAPAS_START_DOCSTRING,
|
||||
)
|
||||
@@ -1078,9 +1077,9 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
padding are 0.
|
||||
labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, seq_length)`, `optional`):
|
||||
Labels per token for computing the hierarchical cell selection loss. This encodes the positions of the
|
||||
answer appearing in the table. Can be obtained using :class:`~transformers.TapasTokenizer`.
|
||||
answer appearing in the table. Can be obtained using :class:`~transformers.TapasTokenizer`.
|
||||
|
||||
- 1 for tokens that are **part of the answer**,
|
||||
- 1 for tokens that are **part of the answer**,
|
||||
- 0 for tokens that are **not part of the answer**.
|
||||
|
||||
aggregation_labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, )`, `optional`):
|
||||
@@ -1088,12 +1087,12 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
should be in :obj:`[0, ..., config.num_aggregation_labels - 1]`. Only required in case of strong
|
||||
supervision for aggregation (WikiSQL-supervised).
|
||||
float_answer (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, )`, `optional`):
|
||||
Float answer for every example in the batch. Set to `float('nan')` for cell selection questions. Only
|
||||
required in case of weak supervision (WTQ) to calculate the aggregate mask and regression loss.
|
||||
Float answer for every example in the batch. Set to `float('nan')` for cell selection questions.
|
||||
Only required in case of weak supervision (WTQ) to calculate the aggregate mask and regression loss.
|
||||
numeric_values (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`, `optional`):
|
||||
Numeric values of every token, NaN for tokens which are not numeric values. Can be obtained using
|
||||
:class:`~transformers.TapasTokenizer`. Only required in case of weak supervision for aggregation (WTQ) to
|
||||
calculate the regression loss.
|
||||
:class:`~transformers.TapasTokenizer`. Only required in case of weak supervision for aggregation (WTQ)
|
||||
to calculate the regression loss.
|
||||
numeric_values_scale (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`, `optional`):
|
||||
Scale of the numeric values of every token. Can be obtained using :class:`~transformers.TapasTokenizer`.
|
||||
Only required in case of weak supervision for aggregation (WTQ) to calculate the regression loss.
|
||||
@@ -1108,8 +1107,8 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
>>> tokenizer = TapasTokenizer.from_pretrained('google/tapas-base-uncased-finetuned-wtq')
|
||||
>>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base-uncased-finetuned-wtq')
|
||||
|
||||
>>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
|
||||
... 'Age': ["56", "45", "59"],
|
||||
>>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
|
||||
... 'Age': ["56", "45", "59"],
|
||||
... 'Number of movies': ["87", "53", "69"]
|
||||
... }
|
||||
>>> table = pd.DataFrame.from_dict(data)
|
||||
@@ -1186,7 +1185,7 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
# Table cells only, without question tokens and table headers.
|
||||
if table_mask is None:
|
||||
table_mask = torch.where(row_ids > 0, torch.ones_like(row_ids), torch.zeros_like(row_ids))
|
||||
# torch.FloatTensor[batch_size, seq_length]
|
||||
# torch.FloatTensor[batch_size, seq_length]
|
||||
input_mask_float = attention_mask.float().to(device)
|
||||
table_mask_float = table_mask.float().to(device)
|
||||
# Mask for cells that exist in the table (i.e. that are not padding).
|
||||
@@ -1207,7 +1206,7 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
self.config.allow_empty_column_selection,
|
||||
)
|
||||
|
||||
# Aggregation logits
|
||||
########## Aggregation logits ##############
|
||||
logits_aggregation = None
|
||||
if self.config.num_aggregation_labels > 0:
|
||||
logits_aggregation = self.aggregation_classifier(pooled_output)
|
||||
@@ -1219,7 +1218,9 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
calculate_loss = True
|
||||
is_supervised = not self.config.num_aggregation_labels > 0 or not self.config.use_answer_as_supervision
|
||||
|
||||
# Semi-supervised cell selection in case of no aggregation:
|
||||
### Semi-supervised cell selection in case of no aggregation
|
||||
#############################################################
|
||||
|
||||
# If the answer (the denotation) appears directly in the table we might
|
||||
# select the answer without applying any aggregation function. There are
|
||||
# some ambiguous cases, see utils._calculate_aggregate_mask for more info.
|
||||
@@ -1230,9 +1231,7 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
aggregate_mask = None
|
||||
else:
|
||||
if float_answer is not None:
|
||||
assert (
|
||||
labels.shape[0] == float_answer.shape[0]
|
||||
), "Make sure the answers are a FloatTensor of shape (batch_size,)"
|
||||
assert labels.shape[0] == float_answer.shape[0], "Make sure the answers are a FloatTensor of shape (batch_size,)"
|
||||
# <float32>[batch_size]
|
||||
aggregate_mask = _calculate_aggregate_mask(
|
||||
float_answer,
|
||||
@@ -1244,7 +1243,9 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
else:
|
||||
raise ValueError("You have to specify float answers in order to calculate the aggregate mask")
|
||||
|
||||
# Cell selection log-likelihood
|
||||
### Cell selection log-likelihood
|
||||
#################################
|
||||
|
||||
if self.config.average_logits_per_cell:
|
||||
logits_per_cell, _ = reduce_mean(logits, cell_index)
|
||||
logits = gather(logits_per_cell, cell_index)
|
||||
@@ -1268,7 +1269,8 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
)
|
||||
dist_per_token = torch.distributions.Bernoulli(logits=logits)
|
||||
|
||||
# Supervised cell selection
|
||||
### Supervised cell selection
|
||||
#############################
|
||||
if self.config.disable_per_token_loss:
|
||||
pass
|
||||
elif is_supervised:
|
||||
@@ -1277,21 +1279,17 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
# For the not supervised case, do not assign loss for cell selection
|
||||
total_loss += torch.mean(selection_loss_per_example * (1.0 - aggregate_mask))
|
||||
|
||||
# Semi-supervised regression loss and supervised loss for aggregations
|
||||
### Semi-supervised regression loss and supervised loss for aggregations
|
||||
######################f###################################################
|
||||
if self.config.num_aggregation_labels > 0:
|
||||
if is_supervised:
|
||||
# Note that `aggregate_mask` is None if the setting is supervised.
|
||||
if aggregation_labels is not None:
|
||||
assert (
|
||||
labels.shape[0] == aggregation_labels.shape[0]
|
||||
), "Make sure the aggregation labels are a LongTensor of shape (batch_size,)"
|
||||
assert labels.shape[0] == aggregation_labels.shape[0], "Make sure the aggregation labels are a LongTensor of shape (batch_size,)"
|
||||
per_example_additional_loss = _calculate_aggregation_loss(
|
||||
logits_aggregation,
|
||||
aggregate_mask,
|
||||
aggregation_labels,
|
||||
self.config.use_answer_as_supervision,
|
||||
self.config.num_aggregation_labels,
|
||||
self.config.aggregation_loss_weight,
|
||||
logits_aggregation, aggregate_mask, aggregation_labels,
|
||||
self.config.use_answer_as_supervision, self.config.num_aggregation_labels,
|
||||
self.config.aggregation_loss_weight
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
@@ -1301,12 +1299,9 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
# Set aggregation labels to zeros
|
||||
aggregation_labels = torch.zeros(labels.shape[0], dtype=torch.long, device=labels.device)
|
||||
per_example_additional_loss = _calculate_aggregation_loss(
|
||||
logits_aggregation,
|
||||
aggregate_mask,
|
||||
aggregation_labels,
|
||||
self.config.use_answer_as_supervision,
|
||||
self.config.num_aggregation_labels,
|
||||
self.config.aggregation_loss_weight,
|
||||
logits_aggregation, aggregate_mask, aggregation_labels,
|
||||
self.config.use_answer_as_supervision, self.config.num_aggregation_labels,
|
||||
self.config.aggregation_loss_weight
|
||||
)
|
||||
|
||||
if self.config.use_answer_as_supervision:
|
||||
@@ -1354,8 +1349,8 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
|
||||
|
||||
@add_start_docstrings(
|
||||
"""
|
||||
Tapas Model with a sequence classification head on top (a linear layer on top of the pooled output), e.g. for table
|
||||
entailment tasks, such as TabFact (Chen et al., 2020).
|
||||
Tapas Model with a sequence classification head on top (a linear layer on top of the pooled output), e.g. for
|
||||
table entailment tasks, such as TabFact (Chen et al., 2020).
|
||||
""",
|
||||
TAPAS_START_DOCSTRING,
|
||||
)
|
||||
@@ -1403,8 +1398,8 @@ class TapasForSequenceClassification(TapasPreTrainedModel):
|
||||
>>> tokenizer = TapasTokenizer.from_pretrained('google/tapas-base-uncased-finetuned-tabfact')
|
||||
>>> model = TapasForSequenceClassification.from_pretrained('google/tapas-base-uncased-finetuned-tabfact')
|
||||
|
||||
>>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
|
||||
... 'Age': ["56", "45", "59"],
|
||||
>>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
|
||||
... 'Age': ["56", "45", "59"],
|
||||
... 'Number of movies': ["87", "53", "69"]
|
||||
... }
|
||||
>>> table = pd.DataFrame.from_dict(data)
|
||||
@@ -1467,7 +1462,7 @@ class AverageApproximationFunction(str, enum.Enum):
|
||||
SECOND_ORDER = "second_order"
|
||||
|
||||
|
||||
# Beginning of everything related to segmented tensors
|
||||
### Beginning of everything related to segmented tensors ###
|
||||
|
||||
|
||||
class IndexMap(object):
|
||||
@@ -1700,14 +1695,13 @@ def _segment_reduce(values, index, segment_reduce_fn, name):
|
||||
|
||||
def reduce_sum(values, index, name="segmented_reduce_sum"):
|
||||
"""
|
||||
Sums a tensor over its segments.
|
||||
|
||||
Outputs 0 for empty segments.
|
||||
|
||||
Sums a tensor over its segments.
|
||||
|
||||
Outputs 0 for empty segments.
|
||||
|
||||
This operations computes the sum over segments, with support for:
|
||||
|
||||
- Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices.
|
||||
- Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be a sum of
|
||||
- Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices.
|
||||
- Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be a sum of
|
||||
vectors rather than scalars. Only the middle dimensions [I1, ..., Ik] are reduced by the operation.
|
||||
|
||||
Args:
|
||||
@@ -1727,16 +1721,15 @@ def reduce_sum(values, index, name="segmented_reduce_sum"):
|
||||
|
||||
def reduce_mean(values, index, name="segmented_reduce_mean"):
|
||||
"""
|
||||
Averages a tensor over its segments.
|
||||
|
||||
Outputs 0 for empty segments.
|
||||
|
||||
Averages a tensor over its segments.
|
||||
|
||||
Outputs 0 for empty segments.
|
||||
|
||||
This operations computes the mean over segments, with support for:
|
||||
|
||||
- Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices.
|
||||
- Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be a mean of
|
||||
vectors rather than scalars.
|
||||
|
||||
- Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices.
|
||||
- Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be a mean of
|
||||
vectors rather than scalars.
|
||||
|
||||
Only the middle dimensions [I1, ..., Ik] are reduced by the operation.
|
||||
|
||||
Args:
|
||||
@@ -1756,16 +1749,15 @@ def reduce_mean(values, index, name="segmented_reduce_mean"):
|
||||
|
||||
def reduce_max(values, index, name="segmented_reduce_max"):
|
||||
"""
|
||||
Computes the maximum over segments.
|
||||
|
||||
Computes the maximum over segments.
|
||||
|
||||
This operation computes the maximum over segments, with support for:
|
||||
|
||||
- Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices.
|
||||
- Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be an element-wise
|
||||
maximum of vectors rather than scalars.
|
||||
- Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices.
|
||||
- Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be an element-wise
|
||||
maximum of vectors rather than scalars.
|
||||
|
||||
Only the middle dimensions [I1, ..., Ik] are reduced by the operation.
|
||||
|
||||
|
||||
Args:
|
||||
values (:obj:`torch.Tensor` of shape [B1, B2, ..., Bn, I1, .., Ik, V1, V2, ..]):
|
||||
Tensor containing the values of which the max must be taken segment-wise.
|
||||
@@ -1783,14 +1775,13 @@ def reduce_max(values, index, name="segmented_reduce_max"):
|
||||
|
||||
def reduce_min(values, index, name="segmented_reduce_min"):
|
||||
"""
|
||||
Computes the minimum over segments.
|
||||
|
||||
Computes the minimum over segments.
|
||||
|
||||
This operations computes the minimum over segments, with support for:
|
||||
|
||||
- Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices.
|
||||
- Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be an element-wise
|
||||
minimum of vectors rather than scalars.
|
||||
|
||||
- Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices.
|
||||
- Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be an element-wise minimum
|
||||
of vectors rather than scalars.
|
||||
|
||||
Only the middle dimensions [I1, ..., Ik] are reduced by the operation.
|
||||
|
||||
Args:
|
||||
@@ -1808,7 +1799,7 @@ def reduce_min(values, index, name="segmented_reduce_min"):
|
||||
return _segment_reduce(values, index, "min", name)
|
||||
|
||||
|
||||
# End of everything related to segmented tensors
|
||||
### End of everything related to segmented tensors ###
|
||||
|
||||
|
||||
def compute_column_logits(
|
||||
@@ -1889,10 +1880,13 @@ def _single_column_cell_selection_loss(token_logits, column_logits, labels, cell
|
||||
allowed to select cells in a single column. Logits outside of the most likely column according to
|
||||
`column_logits` will be set to a very low value (such that the probabilities are 0).
|
||||
"""
|
||||
# Part 1: column loss
|
||||
## Part 1: column loss
|
||||
|
||||
# First find the column we should select. We use the column with maximum number of selected cells.
|
||||
labels_per_column, _ = reduce_sum(torch.as_tensor(labels, dtype=torch.float32, device=labels.device), col_index)
|
||||
# First find the column we should select. We use the column with maximum
|
||||
# number of selected cells.
|
||||
labels_per_column, _ = reduce_sum(
|
||||
torch.as_tensor(labels, dtype=torch.float32, device=labels.device), col_index
|
||||
)
|
||||
# shape of labels_per_column is (batch_size, max_num_cols). It contains the number of label ids for every column, for every example
|
||||
column_label = torch.argmax(labels_per_column, dim=-1) # shape (batch_size,)
|
||||
# Check if there are no selected cells in the column. In that case the model
|
||||
@@ -1908,7 +1902,7 @@ def _single_column_cell_selection_loss(token_logits, column_logits, labels, cell
|
||||
column_dist = torch.distributions.Categorical(logits=column_logits) # shape (batch_size, max_num_cols)
|
||||
column_loss_per_example = -column_dist.log_prob(column_label)
|
||||
|
||||
# Part 2: cell loss
|
||||
## Part 2: cell loss
|
||||
|
||||
# Reduce the labels and logits to per-cell from per-token.
|
||||
# logits_per_cell: shape (batch_size, max_num_rows*max_num_cols) i.e. (batch_size, 64*32)
|
||||
@@ -2116,12 +2110,8 @@ def _calculate_aggregation_loss_unknown(logits_aggregation, aggregate_mask):
|
||||
|
||||
|
||||
def _calculate_aggregation_loss(
|
||||
logits_aggregation,
|
||||
aggregate_mask,
|
||||
aggregation_labels,
|
||||
use_answer_as_supervision,
|
||||
num_aggregation_labels,
|
||||
aggregation_loss_weight,
|
||||
logits_aggregation, aggregate_mask, aggregation_labels, use_answer_as_supervision, num_aggregation_labels,
|
||||
aggregation_loss_weight
|
||||
):
|
||||
"""
|
||||
Calculates the aggregation loss per example.
|
||||
@@ -2279,12 +2269,12 @@ def _calculate_regression_loss(
|
||||
each example in the batch. large_answer_loss_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size,)`): A
|
||||
mask which is 1 for examples for which their answer loss is larger than the answer_loss_cutoff.
|
||||
"""
|
||||
# float32 (batch_size,)
|
||||
# <float32>[batch_size]
|
||||
expected_result = _calculate_expected_result(
|
||||
dist_per_cell, numeric_values, numeric_values_scale, input_mask_float, logits_aggregation, config
|
||||
)
|
||||
|
||||
# float32 (batch_size,)
|
||||
# <float32>[batch_size]
|
||||
answer_masked = torch.where(torch.isnan(answer), torch.zeros_like(answer), answer)
|
||||
|
||||
if config.use_normalized_answer_loss:
|
||||
@@ -2311,4 +2301,4 @@ def _calculate_regression_loss(
|
||||
)
|
||||
per_example_answer_loss_scaled = config.answer_loss_importance * (per_example_answer_loss * aggregate_mask)
|
||||
|
||||
return per_example_answer_loss_scaled, large_answer_loss_mask
|
||||
return per_example_answer_loss_scaled, large_answer_loss_mask
|
||||
@@ -15,6 +15,7 @@
|
||||
""" Tokenization class for TAPAS model."""
|
||||
|
||||
|
||||
import ast
|
||||
import collections
|
||||
import datetime
|
||||
import enum
|
||||
@@ -26,28 +27,23 @@ import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Dict, Generator, List, Optional, Text, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
import pandas as pd
|
||||
import torch
|
||||
from transformers import add_end_docstrings
|
||||
|
||||
from ...file_utils import is_pandas_available
|
||||
from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
|
||||
from ...tokenization_utils_base import (
|
||||
ENCODE_KWARGS_DOCSTRING,
|
||||
BatchEncoding,
|
||||
EncodedInput,
|
||||
ExplicitEnum,
|
||||
PaddingStrategy,
|
||||
PreTokenizedInput,
|
||||
TensorType,
|
||||
TextInput,
|
||||
ExplicitEnum, ENCODE_KWARGS_DOCSTRING,
|
||||
)
|
||||
from ...utils import logging
|
||||
|
||||
|
||||
if is_pandas_available():
|
||||
import pandas as pd
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
@@ -55,52 +51,18 @@ VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
|
||||
|
||||
PRETRAINED_VOCAB_FILES_MAP = {
|
||||
"vocab_file": {
|
||||
"nielsr/tapas-base-finetuned-sqa": "https://huggingface.co/nielsr/tapas-base-finetuned-sqa/resolve/main/vocab.txt",
|
||||
"nielsr/tapas-base-finetuned-wtq": "https://huggingface.co/nielsr/tapas-base-finetuned-wtq/resolve/main/vocab.txt",
|
||||
"nielsr/tapas-base-finetuned-wikisql-supervised": "https://huggingface.co/nielsr/tapas-base-finetuned-wikisql-supervised/resolve/main/vocab.txt",
|
||||
"nielsr/tapas-base-finetuned-tabfact": "https://huggingface.co/nielsr/tapas-base-finetuned-tabfact/resolve/main/vocab.txt",
|
||||
"nielsr/tapas-base-finetuned-sqa": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
|
||||
"nielsr/tapas-base-finetuned-wtq": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
|
||||
"nielsr/tapas-base-finetuned-wikisql-supervised": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
|
||||
}
|
||||
}
|
||||
|
||||
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
|
||||
# large models
|
||||
"nielsr/tapas-large-finetuned-sqa": 512,
|
||||
"nielsr/tapas-large-finetuned-wtq": 512,
|
||||
"nielsr/tapas-large-finetuned-wikisql-supervised": 512,
|
||||
"nielsr/tapas-large-finetuned-tabfact": 512,
|
||||
# base models
|
||||
"nielsr/tapas-base-finetuned-sqa": 512,
|
||||
"nielsr/tapas-base-finetuned-wtq": 512,
|
||||
"nielsr/tapas-base-finetuned-wikisql-supervised": 512,
|
||||
"nielsr/tapas-base-finetuned-tabfact": 512,
|
||||
# medium models
|
||||
"nielsr/tapas-medium-finetuned-sqa": 512,
|
||||
"nielsr/tapas-medium-finetuned-wtq": 512,
|
||||
"nielsr/tapas-medium-finetuned-wikisql-supervised": 512,
|
||||
"nielsr/tapas-medium-finetuned-tabfact": 512,
|
||||
# small models
|
||||
"nielsr/tapas-base-finetuned-sqa": 512,
|
||||
"nielsr/tapas-base-finetuned-wtq": 512,
|
||||
"nielsr/tapas-base-finetuned-wikisql-supervised": 512,
|
||||
"nielsr/tapas-base-finetuned-tabfact": 512,
|
||||
# tiny models
|
||||
"nielsr/tapas-tiny-finetuned-sqa": 512,
|
||||
"nielsr/tapas-tiny-finetuned-wtq": 512,
|
||||
"nielsr/tapas-tiny-finetuned-wikisql-supervised": 512,
|
||||
"nielsr/tapas-tiny-finetuned-tabfact": 512,
|
||||
# mini models
|
||||
"nielsr/tapas-mini-finetuned-sqa": 512,
|
||||
"nielsr/tapas-mini-finetuned-wtq": 512,
|
||||
"nielsr/tapas-mini-finetuned-wikisql-supervised": 512,
|
||||
"nielsr/tapas-mini-finetuned-tabfact": 512,
|
||||
"nielsr/tapas-base-finetuned-sqa": 1024,
|
||||
"nielsr/tapas-base-finetuned-wtq": 1024,
|
||||
"nielsr/tapas-base-finetuned-wikisql-supervised": 1024,
|
||||
}
|
||||
|
||||
PRETRAINED_INIT_CONFIGURATION = {
|
||||
"nielsr/tapas-base-finetuned-sqa": {"do_lower_case": True},
|
||||
"nielsr/tapas-base-finetuned-wtq": {"do_lower_case": True},
|
||||
"nielsr/tapas-base-finetuned-wikisql-supervised": {"do_lower_case": True},
|
||||
"nielsr/tapas-base-finetuned-tabfact": {"do_lower_case": True},
|
||||
}
|
||||
|
||||
|
||||
class TapasTruncationStrategy(ExplicitEnum):
|
||||
@@ -160,7 +122,6 @@ def whitespace_tokenize(text):
|
||||
tokens = text.split()
|
||||
return tokens
|
||||
|
||||
|
||||
TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
|
||||
add_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to encode the sequences with the special tokens relative to their model.
|
||||
@@ -219,14 +180,14 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
tokens, special tokens and padding.
|
||||
- row_ids: indicate to which row of the table a token belongs (starting from 1). Is 0 for all question tokens,
|
||||
special tokens and padding. Tokens of column headers are also 0.
|
||||
- prev_labels: indicate whether a token was (part of) an answer to the previous question (1) or not (0). Useful in
|
||||
a conversational setup (such as SQA).
|
||||
- prev_labels: indicate whether a token was (part of) an answer to the previous question (1) or not (0). Useful
|
||||
in a conversational setup (such as SQA).
|
||||
- column_ranks: indicate the rank of a table token relative to a column, if applicable. For example, if you have a
|
||||
column "number of movies" with values 87, 53 and 69, then the column ranks of these tokens are 3, 1 and 2
|
||||
respectively. 0 for all question tokens, special tokens and padding.
|
||||
column "number of movies" with values 87, 53 and 69, then the column ranks of these tokens are 3, 1 and 2 respectively.
|
||||
0 for all question tokens, special tokens and padding.
|
||||
- inv_column_ranks: indicate the inverse rank of a table token relative to a column, if applicable. For example, if
|
||||
you have a column "number of movies" with values 87, 53 and 69, then the inverse column ranks of these tokens are
|
||||
1, 3 and 2 respectively. 0 for all question tokens, special tokens and padding.
|
||||
you have a column "number of movies" with values 87, 53 and 69, then the inverse column ranks of these tokens are 1, 3 and
|
||||
2 respectively. 0 for all question tokens, special tokens and padding.
|
||||
- numeric_relations: indicate numeric relations between the question and the tokens of the table. 0 for all
|
||||
question tokens, special tokens and padding.
|
||||
|
||||
@@ -310,9 +271,6 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
additional_special_tokens: Optional[List[str]] = None,
|
||||
**kwargs
|
||||
):
|
||||
if not is_pandas_available():
|
||||
raise ImportError("Pandas is required for the TAPAS tokenizer.")
|
||||
|
||||
if additional_special_tokens is not None:
|
||||
if empty_token not in additional_special_tokens:
|
||||
additional_special_tokens.append(empty_token)
|
||||
@@ -496,8 +454,8 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
|
||||
) -> List[int]:
|
||||
"""
|
||||
Build model inputs from a question and flattened table for question answering or sequence classification tasks
|
||||
by concatenating and adding special tokens.
|
||||
Build model inputs from a question and flattened table for question answering or sequence classification tasks by concatenating and
|
||||
adding special tokens.
|
||||
|
||||
Args:
|
||||
token_ids_0 (:obj:`List[int]`): The ids of the question.
|
||||
@@ -545,7 +503,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
@add_end_docstrings(TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
|
||||
def __call__(
|
||||
self,
|
||||
table: "pd.DataFrame",
|
||||
table: pd.DataFrame,
|
||||
queries: Optional[
|
||||
Union[
|
||||
TextInput,
|
||||
@@ -556,8 +514,18 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
List[EncodedInput],
|
||||
]
|
||||
] = None,
|
||||
answer_coordinates: Optional[Union[List[Tuple], List[List[Tuple]]]] = None,
|
||||
answer_text: Optional[Union[List[TextInput], List[List[TextInput]]]] = None,
|
||||
answer_coordinates: Optional[
|
||||
Union[
|
||||
List[Tuple],
|
||||
List[List[Tuple]]
|
||||
]
|
||||
] = None,
|
||||
answer_text: Optional[
|
||||
Union[
|
||||
List[TextInput],
|
||||
List[List[TextInput]]
|
||||
]
|
||||
] = None,
|
||||
add_special_tokens: bool = True,
|
||||
padding: Union[bool, str, PaddingStrategy] = False,
|
||||
truncation: Union[bool, str, TapasTruncationStrategy] = False,
|
||||
@@ -578,24 +546,22 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
|
||||
Args:
|
||||
table (:obj:`pd.DataFrame`):
|
||||
Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas
|
||||
dataframe to convert it to string.
|
||||
Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas dataframe to
|
||||
convert it to string.
|
||||
queries (:obj:`str` or :obj:`List[str]`):
|
||||
Question or batch of questions related to a table to be encoded. Note that in case of a batch, all
|
||||
questions must refer to the **same** table.
|
||||
Question or batch of questions related to a table to be encoded. Note that
|
||||
in case of a batch, all questions must refer to the **same** table.
|
||||
answer_coordinates (:obj:`List[Tuple]` or :obj:`List[List[Tuple]]`, `optional`):
|
||||
Answer coordinates of each table-question pair in the batch. In case only a single table-question pair
|
||||
is provided, then the answer_coordinates must be a single list of one or more tuples. Each tuple must
|
||||
be a (row_index, column_index) pair. The first data row (not the column header row) has index 0. The
|
||||
first column has index 0. In case a batch of table-question pairs is provided, then the
|
||||
answer_coordinates must be a list of lists of tuples (each list corresponding to a single
|
||||
table-question pair).
|
||||
is provided, then the answer_coordinates must be a single list of one or more tuples. Each tuple must be
|
||||
a (row_index, column_index) pair. The first data row (not the column header row) has index 0. The first column
|
||||
has index 0. In case a batch of table-question pairs is provided, then the answer_coordinates must be a
|
||||
list of lists of tuples (each list corresponding to a single table-question pair).
|
||||
answer_text (:obj:`List[str]` or :obj:`List[List[str]]`, `optional`):
|
||||
Answer text of each table-question pair in the batch. In case only a single table-question pair is
|
||||
provided, then the answer_text must be a single list of one or more strings. Each string must be the
|
||||
answer text of a corresponding answer coordinate. In case a batch of table-question pairs is provided,
|
||||
then the answer_coordinates must be a list of lists of strings (each list corresponding to a single
|
||||
table-question pair).
|
||||
Answer text of each table-question pair in the batch. In case only a single table-question pair
|
||||
is provided, then the answer_text must be a single list of one or more strings. Each string must be
|
||||
the answer text of a corresponding answer coordinate. In case a batch of table-question pairs is provided, then
|
||||
the answer_coordinates must be a list of lists of strings (each list corresponding to a single table-question pair).
|
||||
"""
|
||||
assert isinstance(table, pd.DataFrame), "Table must be of type pd.DataFrame"
|
||||
|
||||
@@ -669,7 +635,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
@add_end_docstrings(ENCODE_KWARGS_DOCSTRING, TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
|
||||
def batch_encode_plus(
|
||||
self,
|
||||
table: "pd.DataFrame",
|
||||
table: pd.DataFrame,
|
||||
queries: Optional[
|
||||
Union[
|
||||
List[TextInput],
|
||||
@@ -702,20 +668,20 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
|
||||
Args:
|
||||
table (:obj:`pd.DataFrame`):
|
||||
Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas
|
||||
dataframe to convert it to string.
|
||||
Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas dataframe to
|
||||
convert it to string.
|
||||
queries (:obj:`List[str]`):
|
||||
Batch of questions related to a table to be encoded. Note that all questions must refer to the **same**
|
||||
table.
|
||||
Batch of questions related to a table to be encoded. Note that all questions must refer to
|
||||
the **same** table.
|
||||
answer_coordinates (:obj:`List[Tuple]` or :obj:`List[List[Tuple]]`, `optional`):
|
||||
Answer coordinates of each table-question pair in the batch. Each tuple must be a (row_index,
|
||||
column_index) pair. The first data row (not the column header row) has index 0. The first column has
|
||||
index 0. The answer_coordinates must be a list of lists of tuples (each list corresponding to a single
|
||||
table-question pair).
|
||||
Answer coordinates of each table-question pair in the batch. Each tuple must be
|
||||
a (row_index, column_index) pair. The first data row (not the column header row) has index 0. The first column
|
||||
has index 0. The answer_coordinates must be a
|
||||
list of lists of tuples (each list corresponding to a single table-question pair).
|
||||
answer_text (:obj:`List[str]` or :obj:`List[List[str]]`, `optional`):
|
||||
Answer text of each table-question pair in the batch. In case a batch of table-question pairs is
|
||||
provided, then the answer_coordinates must be a list of lists of strings (each list corresponding to a
|
||||
single table-question pair). Each string must be the answer text of a corresponding answer coordinate.
|
||||
Answer text of each table-question pair in the batch. In case a batch of table-question pairs is provided, then
|
||||
the answer_coordinates must be a list of lists of strings (each list corresponding to a single table-question pair). Each string must be
|
||||
the answer text of a corresponding answer coordinate.
|
||||
"""
|
||||
if return_token_type_ids is not None and not add_special_tokens:
|
||||
raise ValueError(
|
||||
@@ -818,7 +784,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
|
||||
def _batch_prepare_for_model(
|
||||
self,
|
||||
raw_table: "pd.DataFrame",
|
||||
raw_table: pd.DataFrame,
|
||||
raw_queries: Union[
|
||||
List[TextInput],
|
||||
List[PreTokenizedInput],
|
||||
@@ -866,8 +832,8 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
return_tensors=None, # We convert the whole batch to tensors at the end
|
||||
prepend_batch_axis=False,
|
||||
verbose=verbose,
|
||||
prev_answer_coordinates=answer_coordinates[index - 1] if index != 0 else None,
|
||||
prev_answer_text=answer_text[index - 1] if index != 0 else None,
|
||||
prev_answer_coordinates=answer_coordinates[index-1] if index != 0 else None,
|
||||
prev_answer_text=answer_text[index-1] if index != 0 else None,
|
||||
)
|
||||
|
||||
for key, value in outputs.items():
|
||||
@@ -890,7 +856,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
@add_end_docstrings(ENCODE_KWARGS_DOCSTRING)
|
||||
def encode(
|
||||
self,
|
||||
table: "pd.DataFrame",
|
||||
table: pd.DataFrame,
|
||||
query: Optional[
|
||||
Union[
|
||||
TextInput,
|
||||
@@ -907,13 +873,13 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
) -> List[int]:
|
||||
"""
|
||||
Prepare a table and a string for the model. This method does not return token type IDs, attention masks, etc.
|
||||
which are necessary for the model to work correctly. Use that method if you want to build your processing on
|
||||
your own, otherwise refer to ``__call__``.
|
||||
which are necessary for the model to work correctly. Use that method if you want to build your processing
|
||||
on your own, otherwise refer to ``__call__``.
|
||||
|
||||
Args:
|
||||
table (:obj:`pd.DataFrame`):
|
||||
Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas
|
||||
dataframe to convert it to string.
|
||||
Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas dataframe to
|
||||
convert it to string.
|
||||
query (:obj:`str` or :obj:`List[str]`):
|
||||
Question related to a table to be encoded.
|
||||
"""
|
||||
@@ -933,7 +899,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
@add_end_docstrings(ENCODE_KWARGS_DOCSTRING, TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
|
||||
def encode_plus(
|
||||
self,
|
||||
table: "pd.DataFrame",
|
||||
table: pd.DataFrame,
|
||||
query: Optional[
|
||||
Union[
|
||||
TextInput,
|
||||
@@ -962,17 +928,19 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
|
||||
Args:
|
||||
table (:obj:`pd.DataFrame`):
|
||||
Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas
|
||||
dataframe to convert it to string.
|
||||
Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas dataframe to
|
||||
convert it to string.
|
||||
query (:obj:`str` or :obj:`List[str]`):
|
||||
Question related to a table to be encoded.
|
||||
answer_coordinates (:obj:`List[Tuple]` or :obj:`List[List[Tuple]]`, `optional`):
|
||||
Answer coordinates of each table-question pair in the batch. The answer_coordinates must be a single
|
||||
list of one or more tuples. Each tuple must be a (row_index, column_index) pair. The first data row
|
||||
(not the column header row) has index 0. The first column has index 0.
|
||||
list of one or more tuples. Each tuple must be
|
||||
a (row_index, column_index) pair. The first data row (not the column header row) has index 0. The first column
|
||||
has index 0.
|
||||
answer_text (:obj:`List[str]` or :obj:`List[List[str]]`, `optional`):
|
||||
Answer text of each table-question pair in the batch. The answer_text must be a single list of one or
|
||||
more strings. Each string must be the answer text of a corresponding answer coordinate.
|
||||
Answer text of each table-question pair in the batch. The answer_text must be a single list of one
|
||||
or more strings. Each string must be
|
||||
the answer text of a corresponding answer coordinate.
|
||||
"""
|
||||
if return_token_type_ids is not None and not add_special_tokens:
|
||||
raise ValueError(
|
||||
@@ -1016,7 +984,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
|
||||
def _encode_plus(
|
||||
self,
|
||||
table: "pd.DataFrame",
|
||||
table: pd.DataFrame,
|
||||
query: Union[
|
||||
TextInput,
|
||||
PreTokenizedInput,
|
||||
@@ -1072,7 +1040,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
@add_end_docstrings(ENCODE_KWARGS_DOCSTRING, TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
|
||||
def prepare_for_model(
|
||||
self,
|
||||
raw_table: "pd.DataFrame",
|
||||
raw_table: pd.DataFrame,
|
||||
raw_query: Union[
|
||||
TextInput,
|
||||
PreTokenizedInput,
|
||||
@@ -1098,8 +1066,8 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
**kwargs
|
||||
) -> BatchEncoding:
|
||||
"""
|
||||
Prepares a sequence of input id so that it can be used by the model. It adds special tokens, truncates
|
||||
sequences if overflowing while taking into account the special tokens.
|
||||
Prepares a sequence of input id so that it can be used by the model. It
|
||||
adds special tokens, truncates sequences if overflowing while taking into account the special tokens.
|
||||
|
||||
Args:
|
||||
raw_table (:obj:`pd.DataFrame`):
|
||||
@@ -1112,11 +1080,13 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
The query after tokenization.
|
||||
answer_coordinates (:obj:`List[Tuple]` or :obj:`List[List[Tuple]]`, `optional`):
|
||||
Answer coordinates of each table-question pair in the batch. The answer_coordinates must be a single
|
||||
list of one or more tuples. Each tuple must be a (row_index, column_index) pair. The first data row
|
||||
(not the column header row) has index 0. The first column has index 0.
|
||||
list of one or more tuples. Each tuple must be
|
||||
a (row_index, column_index) pair. The first data row (not the column header row) has index 0. The first column
|
||||
has index 0.
|
||||
answer_text (:obj:`List[str]` or :obj:`List[List[str]]`, `optional`):
|
||||
Answer text of each table-question pair in the batch. The answer_text must be a single list of one or
|
||||
more strings. Each string must be the answer text of a corresponding answer coordinate.
|
||||
Answer text of each table-question pair in the batch. The answer_text must be a single list of one
|
||||
or more strings. Each string must be
|
||||
the answer text of a corresponding answer coordinate.
|
||||
"""
|
||||
if isinstance(padding, bool):
|
||||
if padding and (max_length is not None or pad_to_multiple_of is not None):
|
||||
@@ -1147,10 +1117,9 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
num_columns = self._get_num_columns(raw_table)
|
||||
_, _, num_tokens = self._get_table_boundaries(tokenized_table)
|
||||
|
||||
if truncation != TapasTruncationStrategy.DO_NOT_TRUNCATE:
|
||||
num_rows, num_tokens = self._get_truncated_table_rows(
|
||||
query_tokens, tokenized_table, num_rows, num_columns, max_length, truncation_strategy=truncation
|
||||
)
|
||||
if truncation != TapasTruncationStrategy.DO_NOT_TRUNCATE and max_length:
|
||||
num_rows, num_tokens = self._get_truncated_table_rows(query_tokens, tokenized_table, num_rows, num_columns,
|
||||
max_length, truncation_strategy=truncation)
|
||||
table_data = list(self._get_table_values(tokenized_table, num_columns, num_rows, num_tokens))
|
||||
|
||||
query_ids = self.convert_tokens_to_ids(query_tokens)
|
||||
@@ -1184,15 +1153,19 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
column_ids, row_ids, table_data, prev_answer_text, prev_answer_coordinates
|
||||
)
|
||||
|
||||
# FIRST: parse both the table and question in terms of numeric values
|
||||
|
||||
### FIRST: parse both the table and question in terms of numeric values
|
||||
|
||||
raw_table = add_numeric_table_values(raw_table)
|
||||
raw_query = add_numeric_values_to_question(raw_query)
|
||||
|
||||
# SECOND: add numeric-related features (and not parse them in these functions):
|
||||
|
||||
column_ranks, inv_column_ranks = self._get_numeric_column_ranks(column_ids, row_ids, raw_table)
|
||||
numeric_relations = self._get_numeric_relations(raw_query, column_ids, row_ids, raw_table)
|
||||
|
||||
### SECOND: add numeric-related features (and not parse them in these functions):
|
||||
|
||||
column_ranks, inv_column_ranks = self._get_numeric_column_ranks(
|
||||
column_ids, row_ids, raw_table
|
||||
)
|
||||
numeric_relations = self._get_numeric_relations(
|
||||
raw_query, column_ids, row_ids, raw_table
|
||||
)
|
||||
|
||||
# Load from model defaults
|
||||
if return_token_type_ids is None:
|
||||
@@ -1205,7 +1178,9 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
encoded_inputs["attention_mask"] = attention_mask
|
||||
|
||||
if answer_coordinates is not None and answer_text is not None:
|
||||
labels = self.get_answer_ids(column_ids, row_ids, table_data, answer_text, answer_coordinates)
|
||||
labels = self.get_answer_ids(
|
||||
column_ids, row_ids, table_data, answer_text, answer_coordinates
|
||||
)
|
||||
numeric_values = self._get_numeric_values(raw_table, column_ids, row_ids)
|
||||
numeric_values_scale = self._get_numeric_values_scale(raw_table, column_ids, row_ids)
|
||||
|
||||
@@ -1263,13 +1238,13 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
return batch_outputs
|
||||
|
||||
def _get_truncated_table_rows(
|
||||
self,
|
||||
query_tokens: List[str],
|
||||
tokenized_table: TokenizedTable,
|
||||
num_rows: int,
|
||||
num_columns: int,
|
||||
max_length: int,
|
||||
truncation_strategy: Union[str, TapasTruncationStrategy],
|
||||
self,
|
||||
query_tokens: List[str],
|
||||
tokenized_table: TokenizedTable,
|
||||
num_rows: int,
|
||||
num_columns: int,
|
||||
max_length: int,
|
||||
truncation_strategy: Union[str, TapasTruncationStrategy],
|
||||
) -> Tuple[int, int]:
|
||||
"""
|
||||
Truncates a sequence pair in-place following the strategy.
|
||||
@@ -1291,18 +1266,19 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
|
||||
Returns:
|
||||
:obj:`Tuple(int, int)`: tuple containing the number of rows after truncation, and the number of tokens
|
||||
available for each table element.
|
||||
available for each table element.
|
||||
"""
|
||||
if not isinstance(truncation_strategy, TapasTruncationStrategy):
|
||||
truncation_strategy = TapasTruncationStrategy(truncation_strategy)
|
||||
|
||||
if max_length is None:
|
||||
max_length = self.model_max_length
|
||||
|
||||
if truncation_strategy == TapasTruncationStrategy.DROP_ROWS_TO_FIT:
|
||||
while True:
|
||||
num_tokens = self._get_max_num_tokens(
|
||||
query_tokens, tokenized_table, num_rows=num_rows, num_columns=num_columns, max_length=max_length
|
||||
query_tokens,
|
||||
tokenized_table,
|
||||
num_rows=num_rows,
|
||||
num_columns=num_columns,
|
||||
max_length=max_length
|
||||
)
|
||||
|
||||
if num_tokens is not None:
|
||||
@@ -1378,9 +1354,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
List of question tokens. Returns: :obj:`int`: the number of tokens left for the table, given the model
|
||||
max length.
|
||||
"""
|
||||
return (max_length if max_length is not None else self.model_max_length) - self._question_encoding_cost(
|
||||
question_tokens
|
||||
)
|
||||
return (max_length if max_length is not None else self.model_max_length) - self._question_encoding_cost(question_tokens)
|
||||
|
||||
def _get_table_values(self, table, num_columns, num_rows, num_tokens) -> Generator[TableValue, None, None]:
|
||||
"""Iterates over partial table and returns token, column and row indexes."""
|
||||
@@ -1417,7 +1391,14 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
def _get_table_cost(self, table, num_columns, num_rows, num_tokens):
|
||||
return sum(1 for _ in self._get_table_values(table, num_columns, num_rows, num_tokens))
|
||||
|
||||
def _get_max_num_tokens(self, question_tokens, tokenized_table, num_columns, num_rows, max_length):
|
||||
def _get_max_num_tokens(
|
||||
self,
|
||||
question_tokens,
|
||||
tokenized_table,
|
||||
num_columns,
|
||||
num_rows,
|
||||
max_length
|
||||
):
|
||||
"""Computes max number of tokens that can be squeezed into the budget."""
|
||||
token_budget = self._get_token_budget(question_tokens, max_length)
|
||||
_, _, max_num_tokens = self._get_table_boundaries(tokenized_table)
|
||||
@@ -1684,12 +1665,12 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
|
||||
def _get_all_answer_ids(self, column_ids, row_ids, answer_coordinates):
|
||||
"""
|
||||
Maps answer coordinates of a question to token indexes.
|
||||
|
||||
In the SQA format (TSV), the coordinates are given as (row, column) tuples. Here, we first swap them to
|
||||
(column, row) format before calling _get_all_answer_ids_from_coordinates.
|
||||
Maps answer coordinates of a question to token indexes.
|
||||
|
||||
In the SQA format (TSV), the coordinates are given as (row, column) tuples. Here, we first
|
||||
swap them to (column, row) format before calling _get_all_answer_ids_from_coordinates.
|
||||
"""
|
||||
|
||||
|
||||
def _to_coordinates(answer_coordinates_question):
|
||||
return [(coords[1], coords[0]) for coords in answer_coordinates_question]
|
||||
|
||||
@@ -1774,13 +1755,18 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
raise ValueError("Couldn't find all answers")
|
||||
return answer_ids
|
||||
|
||||
def get_answer_ids(self, column_ids, row_ids, tokenized_table, answer_texts_question, answer_coordinates_question):
|
||||
def get_answer_ids(
|
||||
self, column_ids, row_ids, tokenized_table, answer_texts_question, answer_coordinates_question
|
||||
):
|
||||
if self.update_answer_coordinates:
|
||||
return self._find_answer_ids_from_answer_texts(
|
||||
column_ids,
|
||||
row_ids,
|
||||
tokenized_table,
|
||||
answer_texts=[self.tokenize(at) for at in answer_texts_question],
|
||||
answer_texts=[
|
||||
self.tokenize(at)
|
||||
for at in answer_texts_question
|
||||
],
|
||||
)
|
||||
return self._get_answer_ids(column_ids, row_ids, answer_coordinates_question)
|
||||
|
||||
@@ -1841,9 +1827,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
if "numeric_values" in encoded_inputs:
|
||||
encoded_inputs["numeric_values"] = encoded_inputs["numeric_values"] + [float("nan")] * difference
|
||||
if "numeric_values_scale" in encoded_inputs:
|
||||
encoded_inputs["numeric_values_scale"] = (
|
||||
encoded_inputs["numeric_values_scale"] + [1.0] * difference
|
||||
)
|
||||
encoded_inputs["numeric_values_scale"] = encoded_inputs["numeric_values_scale"] + [1.0] * difference
|
||||
if "special_tokens_mask" in encoded_inputs:
|
||||
encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference
|
||||
encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [self.pad_token_id] * difference
|
||||
@@ -1857,11 +1841,9 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
if "labels" in encoded_inputs:
|
||||
encoded_inputs["labels"] = [0] * difference + encoded_inputs["labels"]
|
||||
if "numeric_values" in encoded_inputs:
|
||||
encoded_inputs["numeric_values"] = [float("nan")] * difference + encoded_inputs["numeric_values"]
|
||||
encoded_inputs["numeric_values"] = [float("nan")] * difference + encoded_inputs["numeric_values"]
|
||||
if "numeric_values_scale" in encoded_inputs:
|
||||
encoded_inputs["numeric_values_scale"] = [1.0] * difference + encoded_inputs[
|
||||
"numeric_values_scale"
|
||||
]
|
||||
encoded_inputs["numeric_values_scale"] = [1.0] * difference + encoded_inputs["numeric_values_scale"]
|
||||
if "special_tokens_mask" in encoded_inputs:
|
||||
encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
|
||||
encoded_inputs["input_ids"] = [self.pad_token_id] * difference + encoded_inputs["input_ids"]
|
||||
@@ -1873,7 +1855,7 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
|
||||
return encoded_inputs
|
||||
|
||||
# Everything related to converting logits to predictions
|
||||
#### Everything related to converting logits to predictions ####
|
||||
|
||||
def _get_cell_token_probs(self, probabilities, segment_ids, row_ids, column_ids):
|
||||
for i, p in enumerate(probabilities):
|
||||
@@ -1890,41 +1872,41 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
col = column_ids[i] - 1
|
||||
row = row_ids[i] - 1
|
||||
coords_to_probs[(col, row)].append(prob)
|
||||
return {coords: np.array(cell_probs).mean() for coords, cell_probs in coords_to_probs.items()}
|
||||
return {coords: torch.as_tensor(cell_probs).mean() for coords, cell_probs in coords_to_probs.items()}
|
||||
|
||||
def convert_logits_to_predictions(self, data, logits, logits_agg=None, cell_classification_threshold=0.5):
|
||||
def convert_logits_to_predictions(
|
||||
self, data, logits, logits_agg=None, cell_classification_threshold=0.5
|
||||
):
|
||||
"""
|
||||
Converts logits of :class:`~transformers.TapasForQuestionAnswering` to actual predicted answer coordinates and
|
||||
optional aggregation indices.
|
||||
Converts logits of :class:`~transformers.TapasForQuestionAnswering` to actual predicted answer coordinates
|
||||
and optional aggregation indices.
|
||||
|
||||
Args:
|
||||
data (:obj:`dict`):
|
||||
Dictionary mapping features to actual values. Should be created using
|
||||
:class:`~transformers.TapasTokenizer`.
|
||||
logits (:obj:`np.ndarray` of shape ``(batch_size, sequence_length)``):
|
||||
logits (:obj:`torch.FloatTensor` of shape ``(batch_size, sequence_length)``):
|
||||
Tensor containing the logits at the token level.
|
||||
logits_agg (:obj:`np.ndarray` of shape ``(batch_size, num_aggregation_labels)``, `optional`):
|
||||
logits_agg (:obj:`torch.FloatTensor` of shape ``(batch_size, num_aggregation_labels)``, `optional`):
|
||||
Tensor containing the aggregation logits.
|
||||
cell_classification_threshold (:obj:`float`, `optional`, defaults to 0.5):
|
||||
Threshold to be used for cell selection. All table cells for which their probability is larger than
|
||||
this threshold will be selected.
|
||||
|
||||
Returns:
|
||||
:obj:`tuple` comprising various elements depending on the inputs: predicted_answer_coordinates
|
||||
(``List[List[[tuple]]`` of length ``batch_size``): Predicted answer coordinates as a list of lists of
|
||||
tuples. Each element in the list contains the predicted answer coordinates of a single example in the
|
||||
batch, as a list of tuples. Each tuple is a cell, i.e. (row index, column index).
|
||||
predicted_aggregation_indices (`optional`, returned when ``logits_aggregation`` is provided) ``List[int]``
|
||||
of length ``batch_size``: Predicted aggregation operator indices of the aggregation head.
|
||||
:obj:`tuple` comprising various elements depending on the inputs:
|
||||
predicted_answer_coordinates (``List[List[[tuple]]`` of length ``batch_size``):
|
||||
Predicted answer coordinates as a list of lists of tuples. Each element in the list contains the predicted answer coordinates
|
||||
of a single example in the batch, as a list of tuples. Each tuple is a cell, i.e. (row index, column index).
|
||||
predicted_aggregation_indices (`optional`, returned when ``logits_aggregation`` is provided) ``List[int]`` of length ``batch_size``:
|
||||
Predicted aggregation operator indices of the aggregation head.
|
||||
"""
|
||||
# input data is of type float32
|
||||
# np.log(np.finfo(np.float32).max) = 88.72284
|
||||
# Any value over 88.72284 will overflow when passed through the exponential, sending a warning
|
||||
# We disable this warning by truncating the logits.
|
||||
logits[logits < -88.7] = -88.7
|
||||
# compute probabilities from token logits
|
||||
dist_per_token = torch.distributions.Bernoulli(logits=logits)
|
||||
probabilities = dist_per_token.probs * data["attention_mask"].type(torch.float32).to(
|
||||
dist_per_token.probs.device
|
||||
)
|
||||
|
||||
# Compute probabilities from token logits
|
||||
probabilities = 1 / (1 + np.exp(-logits)) * data["attention_mask"]
|
||||
token_types = [
|
||||
"segment_ids",
|
||||
"column_ids",
|
||||
@@ -1982,13 +1964,13 @@ class TapasTokenizer(PreTrainedTokenizer):
|
||||
|
||||
return output
|
||||
|
||||
# End of everything related to converting logits to predictions
|
||||
#### End of everything related to converting logits to predictions ####
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.tokenization_bert.BasicTokenizer
|
||||
class BasicTokenizer(object):
|
||||
"""
|
||||
Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).
|
||||
Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.)
|
||||
|
||||
Args:
|
||||
do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
@@ -1997,10 +1979,8 @@ class BasicTokenizer(object):
|
||||
Collection of tokens which will never be split during tokenization. Only has an effect when
|
||||
:obj:`do_basic_tokenize=True`
|
||||
tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||||
Whether or not to tokenize Chinese characters.
|
||||
|
||||
This should likely be deactivated for Japanese (see this `issue
|
||||
<https://github.com/huggingface/transformers/issues/328>`__).
|
||||
Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see this
|
||||
`issue <https://github.com/huggingface/transformers/issues/328>`__).
|
||||
strip_accents: (:obj:`bool`, `optional`):
|
||||
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
|
||||
value for :obj:`lowercase` (as in the original BERT).
|
||||
@@ -2017,7 +1997,7 @@ class BasicTokenizer(object):
|
||||
def tokenize(self, text, never_split=None):
|
||||
"""
|
||||
Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see
|
||||
WordPieceTokenizer.
|
||||
WordPieceTokenizer
|
||||
|
||||
Args:
|
||||
**never_split**: (`optional`) list of str
|
||||
@@ -2147,13 +2127,12 @@ class WordpieceTokenizer(object):
|
||||
def tokenize(self, text):
|
||||
"""
|
||||
Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform
|
||||
tokenization using the given vocabulary.
|
||||
|
||||
For example, :obj:`input = "unaffable"` wil return as output :obj:`["un", "##aff", "##able"]`.
|
||||
tokenization using the given vocabulary. For example, :obj:`input = "unaffable"` wil return as output
|
||||
:obj:`["un", "##aff", "##able"]`
|
||||
|
||||
Args:
|
||||
text: A single token or whitespace separated tokens. This should have
|
||||
already been passed through `BasicTokenizer`.
|
||||
already been passed through `BasicTokenizer`
|
||||
|
||||
Returns:
|
||||
A list of wordpiece tokens.
|
||||
@@ -2193,10 +2172,11 @@ class WordpieceTokenizer(object):
|
||||
return output_tokens
|
||||
|
||||
|
||||
|
||||
# Below: utilities for TAPAS tokenizer (independent from PyTorch/Tensorflow).
|
||||
# This includes functions to parse numeric values (dates and numbers) from both the table and questions in order
|
||||
# to create the column_ranks, inv_column_ranks, numeric_values, numeric values_scale and numeric_relations in
|
||||
# prepare_for_model of TapasTokenizer.
|
||||
# prepare_for_model of TapasTokenizer.
|
||||
# These are meant to be used in an academic setup, for production use cases Gold mine or Aqua should be used.
|
||||
|
||||
|
||||
@@ -2242,16 +2222,16 @@ class Cell:
|
||||
|
||||
@dataclass
|
||||
class Question:
|
||||
original_text: Text # The original raw question string.
|
||||
text: Text # The question string after normalization.
|
||||
original_text: Text # The original raw question string.
|
||||
text: Text # The question string after normalization.
|
||||
numeric_spans: Optional[List[NumericValueSpan]] = None
|
||||
|
||||
|
||||
|
||||
# Below: all functions from number_utils.py as well as 2 functions (namely get_all_spans and normalize_for_match)
|
||||
# from text_utils.py of the original implementation. URL's:
|
||||
# from text_utils.py of the original implementation. URL's:
|
||||
# - https://github.com/google-research/tapas/blob/master/tapas/utils/number_utils.py
|
||||
# - https://github.com/google-research/tapas/blob/master/tapas/utils/text_utils.py
|
||||
|
||||
|
||||
|
||||
# Constants for parsing date expressions.
|
||||
# Masks that specify (by a bool) which of (year, month, day) will be populated.
|
||||
@@ -2293,7 +2273,7 @@ _DATE_PATTERNS = (
|
||||
_FIELD_TO_REGEX = (
|
||||
("%A", r"\w+"), # Weekday as locale’s full name.
|
||||
("%B", r"\w+"), # Month as locale’s full name.
|
||||
("%Y", r"\d{4}"), # Year with century as a decimal number.
|
||||
("%Y", r"\d{4}"), # Year with century as a decimal number.
|
||||
("%b", r"\w{3}"), # Month as locale’s abbreviated name.
|
||||
("%d", r"\d{1,2}"), # Day of the month as a zero-padded decimal number.
|
||||
("%m", r"\d{1,2}"), # Month as a zero-padded decimal number.
|
||||
@@ -2526,9 +2506,9 @@ def parse_text(text):
|
||||
|
||||
|
||||
# Below: all functions from number_annotation_utils.py and 2 functions (namely filter_invalid_unicode
|
||||
# and filter_invalid_unicode_from_table) from text_utils.py of the original implementation. URL's:
|
||||
# and filter_invalid_unicode_from_table) from text_utils.py of the original implementation. URL's:
|
||||
# - https://github.com/google-research/tapas/blob/master/tapas/utils/number_annotation_utils.py
|
||||
# - https://github.com/google-research/tapas/blob/master/tapas/utils/text_utils.py
|
||||
# - https://github.com/google-research/tapas/blob/master/tapas/utils/text_utils.py
|
||||
|
||||
|
||||
_PrimitiveNumericValue = Union[float, Tuple[Optional[float], Optional[float], Optional[float]]]
|
||||
@@ -2536,7 +2516,7 @@ _SortKeyFn = Callable[[NumericValue], Tuple[float, Ellipsis]]
|
||||
|
||||
_DATE_TUPLE_SIZE = 3
|
||||
|
||||
EMPTY_TEXT = "EMPTY"
|
||||
EMPTY_TEXT = 'EMPTY'
|
||||
|
||||
NUMBER_TYPE = "number"
|
||||
DATE_TYPE = "date"
|
||||
@@ -2619,21 +2599,22 @@ def get_numeric_sort_key_fn(numeric_values):
|
||||
return _sort_key_fn
|
||||
|
||||
|
||||
def _consolidate_numeric_values(row_index_to_values, min_consolidation_fraction, debug_info):
|
||||
"""
|
||||
Finds the most common numeric values in a column and returns them
|
||||
|
||||
def _consolidate_numeric_values(
|
||||
row_index_to_values,
|
||||
min_consolidation_fraction,
|
||||
debug_info):
|
||||
"""Finds the most common numeric values in a column and returns them.
|
||||
Args:
|
||||
row_index_to_values:
|
||||
row_index_to_values:
|
||||
For each row index all the values in that cell.
|
||||
min_consolidation_fraction:
|
||||
min_consolidation_fraction:
|
||||
Fraction of cells that need to have consolidated value.
|
||||
debug_info:
|
||||
Additional information only used for logging
|
||||
|
||||
debug_info:
|
||||
Additional information only used for logging.
|
||||
Returns:
|
||||
For each row index the first value that matches the most common value. Rows that don't have a matching value
|
||||
are dropped. Empty list if values can't be consolidated.
|
||||
For each row index the first value that matches the most common value.
|
||||
Rows that don't have a matching value are dropped. Empty list if values can't
|
||||
be consolidated.
|
||||
"""
|
||||
type_counts = collections.Counter()
|
||||
for numeric_values in row_index_to_values.values():
|
||||
@@ -2655,7 +2636,7 @@ def _consolidate_numeric_values(row_index_to_values, min_consolidation_fraction,
|
||||
max_type = DATE_TYPE
|
||||
else:
|
||||
max_type = next(iter(valid_types))
|
||||
|
||||
|
||||
new_row_index_to_value = {}
|
||||
for index, values in row_index_to_values.items():
|
||||
# Extract the first matching value.
|
||||
@@ -2674,10 +2655,9 @@ def _get_numeric_values(text):
|
||||
|
||||
|
||||
def _get_column_values(table, col_index):
|
||||
"""
|
||||
Parses text in column and returns a dict mapping row_index to values. This is the _get_column_values function from
|
||||
number_annotation_utils.py of the original implementation
|
||||
|
||||
"""Parses text in column and returns a dict mapping row_index to values.
|
||||
This is the _get_column_values function from number_annotation_utils.py of the
|
||||
original implementation.
|
||||
Args:
|
||||
table: Pandas dataframe
|
||||
col_index: integer, indicating the index of the column to get the numeric values of
|
||||
@@ -2707,7 +2687,9 @@ def add_numeric_values_to_question(question):
|
||||
original_text = question
|
||||
question = normalize_for_match(question)
|
||||
numeric_spans = parse_text(question)
|
||||
return Question(original_text=original_text, text=question, numeric_spans=numeric_spans)
|
||||
return Question(original_text=original_text,
|
||||
text=question,
|
||||
numeric_spans=numeric_spans)
|
||||
|
||||
|
||||
def filter_invalid_unicode(text):
|
||||
@@ -2716,10 +2698,10 @@ def filter_invalid_unicode(text):
|
||||
|
||||
|
||||
def filter_invalid_unicode_from_table(table):
|
||||
"""
|
||||
Removes invalid unicode from table. Checks whether a table cell text contains an invalid unicode encoding. If yes,
|
||||
reset the table cell text to an empty str and log a warning for each invalid cell
|
||||
|
||||
"""Removes invalid unicode from table.
|
||||
Checks whether a table cell text contains an invalid unicode encoding. If yes,
|
||||
reset the table cell text to an empty str and log a warning for each invalid
|
||||
cell.
|
||||
Args:
|
||||
table: table to clean.
|
||||
"""
|
||||
@@ -2732,48 +2714,46 @@ def filter_invalid_unicode_from_table(table):
|
||||
cell, is_invalid = filter_invalid_unicode(cell)
|
||||
if is_invalid:
|
||||
logging.warning(
|
||||
"Scrub an invalid table body @ table_id: %s, row_index: %d, " "col_index: %d",
|
||||
table.table_id,
|
||||
row_index,
|
||||
col_index,
|
||||
)
|
||||
"Scrub an invalid table body @ table_id: %s, row_index: %d, "
|
||||
"col_index: %d", table.table_id, row_index, col_index)
|
||||
for col_index, column in enumerate(table.columns):
|
||||
column, is_invalid = filter_invalid_unicode(column)
|
||||
if is_invalid:
|
||||
logging.warning("Scrub an invalid table header @ table_id: %s, col_index: %d", table.table_id, col_index)
|
||||
logging.warning(
|
||||
"Scrub an invalid table header @ table_id: %s, col_index: %d",
|
||||
table.table_id, col_index)
|
||||
|
||||
|
||||
def add_numeric_table_values(table, min_consolidation_fraction=0.7, debug_info=None):
|
||||
"""
|
||||
Parses text in table column-wise and adds the consolidated values. Consolidation refers to finding values with a
|
||||
common types (date or number)
|
||||
|
||||
def add_numeric_table_values(table,
|
||||
min_consolidation_fraction=0.7,
|
||||
debug_info = None):
|
||||
"""Parses text in table column-wise and adds the consolidated values.
|
||||
Consolidation refers to finding values with a common types (date or number).
|
||||
Args:
|
||||
table:
|
||||
table:
|
||||
Table to annotate.
|
||||
min_consolidation_fraction:
|
||||
min_consolidation_fraction:
|
||||
Fraction of cells in a column that need to have consolidated value.
|
||||
debug_info:
|
||||
debug_info:
|
||||
Additional information used for logging.
|
||||
"""
|
||||
table = table.copy()
|
||||
# First, filter table on invalid unicode
|
||||
filter_invalid_unicode_from_table(table)
|
||||
|
||||
|
||||
# Second, replace cell values by Cell objects
|
||||
for row_index, row in table.iterrows():
|
||||
for col_index, cell in enumerate(row):
|
||||
table.iloc[row_index, col_index] = Cell(text=cell)
|
||||
|
||||
|
||||
# Third, add numeric_value attributes to these Cell objects
|
||||
for col_index, column in enumerate(table.columns):
|
||||
column_values = _consolidate_numeric_values(
|
||||
_get_column_values(table, col_index),
|
||||
min_consolidation_fraction=min_consolidation_fraction,
|
||||
debug_info=(debug_info, column),
|
||||
)
|
||||
debug_info=(debug_info, column))
|
||||
|
||||
for row_index, numeric_value in column_values.items():
|
||||
table.iloc[row_index, col_index].numeric_value = numeric_value
|
||||
|
||||
return table
|
||||
return table
|
||||
@@ -28,9 +28,8 @@ from .file_utils import (
|
||||
_datasets_available,
|
||||
_faiss_available,
|
||||
_flax_available,
|
||||
_pandas_available,
|
||||
_scatter_available,
|
||||
_sentencepiece_available,
|
||||
_scatter_available,
|
||||
_tf_available,
|
||||
_tokenizers_available,
|
||||
_torch_available,
|
||||
@@ -223,23 +222,10 @@ def require_tokenizers(test_case):
|
||||
return test_case
|
||||
|
||||
|
||||
def require_pandas(test_case):
|
||||
"""
|
||||
Decorator marking a test that requires pandas.
|
||||
|
||||
These tests are skipped when pandas isn't installed.
|
||||
|
||||
"""
|
||||
if not _pandas_available:
|
||||
return unittest.skip("test requires pandas")(test_case)
|
||||
else:
|
||||
return test_case
|
||||
|
||||
|
||||
def require_scatter(test_case):
|
||||
"""
|
||||
Decorator marking a test that requires PyTorch Scatter. These tests are skipped when PyTorch Scatter isn't
|
||||
installed.
|
||||
Decorator marking a test that requires PyTorch Scatter.
|
||||
These tests are skipped when PyTorch Scatter isn't installed.
|
||||
"""
|
||||
if not _scatter_available:
|
||||
return unittest.skip("test requires PyTorch Scatter")(test_case)
|
||||
|
||||
@@ -2814,7 +2814,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
|
||||
len_ids = len(ids)
|
||||
len_pair_ids = len(pair_ids) if pair else 0
|
||||
|
||||
if return_token_type_ids and not add_special_tokens:
|
||||
if return_token_type_ids is not None and not add_special_tokens:
|
||||
raise ValueError(
|
||||
"Asking to return token_type_ids while setting add_special_tokens to False "
|
||||
"results in an undefined behavior. Please set add_special_tokens to True or "
|
||||
|
||||
@@ -543,7 +543,7 @@ class Trainer:
|
||||
self.args.output_dir = checkpoint_dir
|
||||
output_dir = os.path.join(self.args.output_dir, f"{PREFIX_CHECKPOINT_DIR}-{self.state.global_step}")
|
||||
self.save_model(output_dir)
|
||||
if self.is_world_process_zero():
|
||||
if self.is_world_master():
|
||||
self.state.save_to_json(os.path.join(output_dir, "trainer_state.json"))
|
||||
torch.save(self.optimizer.state_dict(), os.path.join(output_dir, "optimizer.pt"))
|
||||
torch.save(self.lr_scheduler.state_dict(), os.path.join(output_dir, "scheduler.pt"))
|
||||
|
||||
@@ -14,15 +14,6 @@ class FlaxAutoModel:
|
||||
requires_flax(self)
|
||||
|
||||
|
||||
class FlaxBertForMaskedLM:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_flax(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_flax(self)
|
||||
|
||||
|
||||
class FlaxBertModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_flax(self)
|
||||
|
||||
@@ -118,11 +118,6 @@ class BeamSearchScorer:
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class HammingDiversityLogitsProcessor:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class LogitsProcessor:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
@@ -153,11 +148,6 @@ class NoRepeatNGramLogitsProcessor:
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class PrefixConstrainedLogitsProcessor:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class RepetitionPenaltyLogitsProcessor:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
@@ -450,15 +440,6 @@ class BartModel:
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class BartPretrainedModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class PretrainedBartModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
@@ -1389,77 +1370,6 @@ def load_tf_weights_in_mobilebert(*args, **kwargs):
|
||||
requires_pytorch(load_tf_weights_in_mobilebert)
|
||||
|
||||
|
||||
MPNET_PRETRAINED_MODEL_ARCHIVE_LIST = None
|
||||
|
||||
|
||||
class MPNetForMaskedLM:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class MPNetForMultipleChoice:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class MPNetForQuestionAnswering:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class MPNetForSequenceClassification:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class MPNetForTokenClassification:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class MPNetLayer:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class MPNetModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class MPNetPreTrainedModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class MT5EncoderModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
@@ -1867,45 +1777,6 @@ def load_tf_weights_in_t5(*args, **kwargs):
|
||||
requires_pytorch(load_tf_weights_in_t5)
|
||||
|
||||
|
||||
TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST = None
|
||||
|
||||
|
||||
class TapasForMaskedLM:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class TapasForQuestionAnswering:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class TapasForSequenceClassification:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
class TapasModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_pytorch(self)
|
||||
|
||||
|
||||
TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST = None
|
||||
|
||||
|
||||
|
||||
@@ -1006,77 +1006,6 @@ class TFMobileBertPreTrainedModel:
|
||||
requires_tf(self)
|
||||
|
||||
|
||||
TF_MPNET_PRETRAINED_MODEL_ARCHIVE_LIST = None
|
||||
|
||||
|
||||
class TFMPNetForMaskedLM:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
|
||||
class TFMPNetForMultipleChoice:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
|
||||
class TFMPNetForQuestionAnswering:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
|
||||
class TFMPNetForSequenceClassification:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
|
||||
class TFMPNetForTokenClassification:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
|
||||
class TFMPNetMainLayer:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
|
||||
class TFMPNetModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
|
||||
class TFMPNetPreTrainedModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
|
||||
class TFMT5EncoderModel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tf(self)
|
||||
|
||||
@@ -164,15 +164,6 @@ class MobileBertTokenizerFast:
|
||||
requires_tokenizers(self)
|
||||
|
||||
|
||||
class MPNetTokenizerFast:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tokenizers(self)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(self, *args, **kwargs):
|
||||
requires_tokenizers(self)
|
||||
|
||||
|
||||
class MT5TokenizerFast:
|
||||
def __init__(self, *args, **kwargs):
|
||||
requires_tokenizers(self)
|
||||
|
||||
+3
-8
@@ -17,24 +17,20 @@
|
||||
# limitations under the License.
|
||||
|
||||
{%- if cookiecutter.generate_tensorflow_and_pytorch == "PyTorch & TensorFlow" %}
|
||||
from ...file_utils import is_tf_available, is_torch_available, is_tokenizers_available
|
||||
from ...file_utils import is_tf_available, is_torch_available
|
||||
{%- elif cookiecutter.generate_tensorflow_and_pytorch == "PyTorch" %}
|
||||
from ...file_utils import is_torch_available, is_tokenizers_available
|
||||
from ...file_utils import is_torch_available
|
||||
{%- elif cookiecutter.generate_tensorflow_and_pytorch == "TensorFlow" %}
|
||||
from ...file_utils import is_tf_available, is_tokenizers_available
|
||||
from ...file_utils import is_tf_available
|
||||
{% endif %}
|
||||
from .configuration_{{cookiecutter.lowercase_modelname}} import {{cookiecutter.uppercase_modelname}}_PRETRAINED_CONFIG_ARCHIVE_MAP, {{cookiecutter.camelcase_modelname}}Config
|
||||
from .tokenization_{{cookiecutter.lowercase_modelname}} import {{cookiecutter.camelcase_modelname}}Tokenizer
|
||||
|
||||
if is_tokenizers_available():
|
||||
from .tokenization_{{cookiecutter.lowercase_modelname}}_fast import {{cookiecutter.camelcase_modelname}}TokenizerFast
|
||||
|
||||
{%- if (cookiecutter.generate_tensorflow_and_pytorch == "PyTorch & TensorFlow" or cookiecutter.generate_tensorflow_and_pytorch == "PyTorch") %}
|
||||
if is_torch_available():
|
||||
from .modeling_{{cookiecutter.lowercase_modelname}} import (
|
||||
{{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
{{cookiecutter.camelcase_modelname}}ForMaskedLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
|
||||
{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
@@ -50,7 +46,6 @@ if is_tf_available():
|
||||
from .modeling_tf_{{cookiecutter.lowercase_modelname}} import (
|
||||
TF_{{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForMaskedLM,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
|
||||
+26
-111
@@ -17,7 +17,6 @@
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
from transformers.modeling_tf_outputs import TFCausalLMOutput
|
||||
from ...activations_tf import get_tf_activation
|
||||
from ...file_utils import (
|
||||
MULTIPLE_CHOICE_DUMMY_INPUTS,
|
||||
@@ -41,7 +40,6 @@ from ...modeling_tf_utils import (
|
||||
TFQuestionAnsweringLoss,
|
||||
TFSequenceClassificationLoss,
|
||||
TFTokenClassificationLoss,
|
||||
TFCausalLanguageModelingLoss,
|
||||
TFSequenceSummary,
|
||||
get_initializer,
|
||||
input_processing,
|
||||
@@ -113,23 +111,19 @@ class TF{{cookiecutter.camelcase_modelname}}Embeddings(tf.keras.layers.Layer):
|
||||
mode="embedding",
|
||||
training=False,
|
||||
):
|
||||
"""
|
||||
Get token embeddings of inputs.
|
||||
|
||||
"""Get token embeddings of inputs.
|
||||
Args:
|
||||
inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids)
|
||||
mode: string, a valid value is one of "embedding" and "linear".
|
||||
|
||||
Returns:
|
||||
outputs: If mode == "embedding", output embedding tensor, float32 with shape [batch_size, length,
|
||||
embedding_size]; if mode == "linear", output linear tensor, float32 with shape [batch_size, length,
|
||||
vocab_size].
|
||||
|
||||
outputs: (1) If mode == "embedding", output embedding tensor, float32 with
|
||||
shape [batch_size, length, embedding_size]; (2) mode == "linear", output
|
||||
linear tensor, float32 with shape [batch_size, length, vocab_size].
|
||||
Raises:
|
||||
ValueError: if mode is not valid.
|
||||
|
||||
Shared weights logic adapted from
|
||||
https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
|
||||
https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24
|
||||
"""
|
||||
if mode == "embedding":
|
||||
return self._embedding(input_ids, position_ids, token_type_ids, inputs_embeds, training=training)
|
||||
@@ -167,12 +161,9 @@ class TF{{cookiecutter.camelcase_modelname}}Embeddings(tf.keras.layers.Layer):
|
||||
return embeddings
|
||||
|
||||
def _linear(self, inputs):
|
||||
"""
|
||||
Computes logits by running inputs through a linear layer.
|
||||
|
||||
"""Computes logits by running inputs through a linear layer.
|
||||
Args:
|
||||
inputs: A float32 tensor with shape [batch_size, length, hidden_size].
|
||||
|
||||
inputs: A float32 tensor with shape [batch_size, length, hidden_size]
|
||||
Returns:
|
||||
float32 tensor with shape [batch_size, length, vocab_size].
|
||||
"""
|
||||
@@ -336,6 +327,7 @@ class TF{{cookiecutter.camelcase_modelname}}Output(tf.keras.layers.Layer):
|
||||
return hidden_states
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_tf_bert.TFBertLayer with Bert->{{cookiecutter.camelcase_modelname}}
|
||||
class TF{{cookiecutter.camelcase_modelname}}Layer(tf.keras.layers.Layer):
|
||||
def __init__(self, config, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -344,7 +336,6 @@ class TF{{cookiecutter.camelcase_modelname}}Layer(tf.keras.layers.Layer):
|
||||
self.intermediate = TF{{cookiecutter.camelcase_modelname}}Intermediate(config, name="intermediate")
|
||||
self.{{cookiecutter.lowercase_modelname}}_output = TF{{cookiecutter.camelcase_modelname}}Output(config, name="output")
|
||||
|
||||
# Copied from transformers.models.bert.modeling_tf_bert.TFBertLayer.call with bert->{{cookiecutter.lowercase_modelname}}
|
||||
def call(self, hidden_states, attention_mask, head_mask, output_attentions, training=False):
|
||||
attention_outputs = self.attention(
|
||||
hidden_states, attention_mask, head_mask, output_attentions, training=training
|
||||
@@ -356,7 +347,7 @@ class TF{{cookiecutter.camelcase_modelname}}Layer(tf.keras.layers.Layer):
|
||||
|
||||
return outputs
|
||||
|
||||
# Copied from transformers.models.bert.modeling_tf_bert.TFBertEncoder with Bert->{{cookiecutter.camelcase_modelname}}
|
||||
|
||||
class TF{{cookiecutter.camelcase_modelname}}Encoder(tf.keras.layers.Layer):
|
||||
def __init__(self, config, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
@@ -462,6 +453,20 @@ class TF{{cookiecutter.camelcase_modelname}}MLMHead(tf.keras.layers.Layer):
|
||||
return prediction_scores
|
||||
|
||||
|
||||
class TF{{cookiecutter.camelcase_modelname}}NSPHead(tf.keras.layers.Layer):
|
||||
def __init__(self, config, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.seq_relationship = tf.keras.layers.Dense(
|
||||
2, kernel_initializer=get_initializer(config.initializer_range), name="seq_relationship"
|
||||
)
|
||||
|
||||
def call(self, pooled_output):
|
||||
seq_relationship_score = self.seq_relationship(pooled_output)
|
||||
|
||||
return seq_relationship_score
|
||||
|
||||
|
||||
@keras_serializable
|
||||
class TF{{cookiecutter.camelcase_modelname}}MainLayer(tf.keras.layers.Layer):
|
||||
config_class = {{cookiecutter.camelcase_modelname}}Config
|
||||
@@ -595,6 +600,7 @@ class TF{{cookiecutter.camelcase_modelname}}MainLayer(tf.keras.layers.Layer):
|
||||
)
|
||||
|
||||
|
||||
# Copied from transformers.models.bert.modeling_tf_bert.TFBertPreTrainedModel with Bert->{{cookiecutter.camelcase_modelname}}
|
||||
class TF{{cookiecutter.camelcase_modelname}}PreTrainedModel(TFPreTrainedModel):
|
||||
"""An abstract class to handle weights initialization and
|
||||
a simple interface for downloading and loading pretrained models.
|
||||
@@ -849,97 +855,6 @@ class TF{{cookiecutter.camelcase_modelname}}ForMaskedLM(TF{{cookiecutter.camelca
|
||||
attentions=outputs.attentions,
|
||||
)
|
||||
|
||||
@add_start_docstrings(
|
||||
"""{{cookiecutter.modelname}} Model with a `language modeling` head on top for CLM fine-tuning. """, {{cookiecutter.uppercase_modelname}}_START_DOCSTRING
|
||||
)
|
||||
class TF{{cookiecutter.camelcase_modelname}}ForCausalLM(TF{{cookiecutter.camelcase_modelname}}PreTrainedModel, TFCausalLanguageModelingLoss):
|
||||
|
||||
def __init__(self, config, *inputs, **kwargs):
|
||||
super().__init__(config, *inputs, **kwargs)
|
||||
|
||||
if not config.is_decoder:
|
||||
logger.warning("If you want to use `TF{{cookiecutter.camelcase_modelname}}ForCausalLM` as a standalone, add `is_decoder=True.`")
|
||||
|
||||
self.{{cookiecutter.lowercase_modelname}} = TF{{cookiecutter.camelcase_modelname}}MainLayer(config, name="{{cookiecutter.lowercase_modelname}}")
|
||||
self.mlm = TF{{cookiecutter.camelcase_modelname}}MLMHead(config, self.{{cookiecutter.lowercase_modelname}}.embeddings, name="mlm___cls")
|
||||
|
||||
def get_output_embeddings(self):
|
||||
return self.{{cookiecutter.lowercase_modelname}}.embeddings
|
||||
|
||||
@add_code_sample_docstrings(
|
||||
tokenizer_class=_TOKENIZER_FOR_DOC,
|
||||
checkpoint="{{cookiecutter.checkpoint_identifier}}",
|
||||
output_type=TFCausalLMOutput,
|
||||
config_class=_CONFIG_FOR_DOC,
|
||||
)
|
||||
def call(
|
||||
self,
|
||||
input_ids=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
labels=None,
|
||||
training=False,
|
||||
**kwargs,
|
||||
):
|
||||
r"""
|
||||
labels (:obj:`tf.Tensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
|
||||
Labels for computing the cross entropy classification loss. Indices should be in ``[0, ...,
|
||||
config.vocab_size - 1]``.
|
||||
"""
|
||||
inputs = input_processing(
|
||||
func=self.call,
|
||||
config=self.config,
|
||||
input_ids=input_ids,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
labels=labels,
|
||||
training=training,
|
||||
kwargs_call=kwargs,
|
||||
)
|
||||
outputs = self.{{cookiecutter.lowercase_modelname}}(
|
||||
inputs["input_ids"],
|
||||
attention_mask=inputs["attention_mask"],
|
||||
token_type_ids=inputs["token_type_ids"],
|
||||
position_ids=inputs["position_ids"],
|
||||
head_mask=inputs["head_mask"],
|
||||
inputs_embeds=inputs["inputs_embeds"],
|
||||
output_attentions=inputs["output_attentions"],
|
||||
output_hidden_states=inputs["output_hidden_states"],
|
||||
return_dict=inputs["return_dict"],
|
||||
training=inputs["training"],
|
||||
)
|
||||
sequence_output = outputs[0]
|
||||
logits = self.mlm(sequence_output, training=inputs["training"])
|
||||
loss = None
|
||||
|
||||
if inputs["labels"] is not None:
|
||||
# shift labels to the left and cut last logit token
|
||||
logits = logits[:, :-1]
|
||||
labels = inputs["labels"][:, 1:]
|
||||
loss = self.compute_loss(labels, logits)
|
||||
|
||||
if not inputs["return_dict"]:
|
||||
output = (logits,) + outputs[1:]
|
||||
return ((loss,) + output) if loss is not None else output
|
||||
|
||||
return TFCausalLMOutput(
|
||||
loss=loss,
|
||||
logits=logits,
|
||||
hidden_states=outputs.hidden_states,
|
||||
attentions=outputs.attentions,
|
||||
)
|
||||
|
||||
class TF{{cookiecutter.camelcase_modelname}}ClassificationHead(tf.keras.layers.Layer):
|
||||
"""Head for sentence-level classification tasks."""
|
||||
@@ -1236,7 +1151,7 @@ class TF{{cookiecutter.camelcase_modelname}}ForTokenClassification(TF{{cookiecut
|
||||
training=training,
|
||||
kwargs_call=kwargs,
|
||||
)
|
||||
outputs = self.{{cookiecutter.lowercase_modelname}}(
|
||||
outputs = self.{{cookiecutter.uppercase_modelname}}(
|
||||
inputs["input_ids"],
|
||||
attention_mask=inputs["attention_mask"],
|
||||
token_type_ids=inputs["token_type_ids"],
|
||||
@@ -1331,7 +1246,7 @@ class TF{{cookiecutter.camelcase_modelname}}ForQuestionAnswering(TF{{cookiecutte
|
||||
training=training,
|
||||
kwargs_call=kwargs,
|
||||
)
|
||||
outputs = self.{{cookiecutter.lowercase_modelname}}(
|
||||
outputs = self.{{cookiecutter.uppercase_modelname}}(
|
||||
inputs["input_ids"],
|
||||
attention_mask=inputs["attention_mask"],
|
||||
token_type_ids=inputs["token_type_ids"],
|
||||
|
||||
+13
-169
@@ -18,6 +18,7 @@
|
||||
|
||||
import math
|
||||
import os
|
||||
import warnings
|
||||
|
||||
import torch
|
||||
import torch.utils.checkpoint
|
||||
@@ -28,11 +29,10 @@ from ...file_utils import (
|
||||
add_code_sample_docstrings,
|
||||
add_start_docstrings,
|
||||
add_start_docstrings_to_model_forward,
|
||||
replace_return_docstrings,
|
||||
)
|
||||
from ...modeling_outputs import (
|
||||
BaseModelOutputWithCrossAttentions,
|
||||
CausalLMOutputWithCrossAttentions,
|
||||
BaseModelOutput,
|
||||
BaseModelOutputWithPooling,
|
||||
MaskedLMOutput,
|
||||
MultipleChoiceModelOutput,
|
||||
QuestionAnsweringModelOutput,
|
||||
@@ -157,7 +157,6 @@ class {{cookiecutter.camelcase_modelname}}Embeddings(nn.Module):
|
||||
|
||||
# position_ids (1, len position emb) is contiguous in memory and exported when serialized
|
||||
self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)))
|
||||
self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
|
||||
|
||||
def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None):
|
||||
if input_ids is not None:
|
||||
@@ -175,12 +174,10 @@ class {{cookiecutter.camelcase_modelname}}Embeddings(nn.Module):
|
||||
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.word_embeddings(input_ids)
|
||||
position_embeddings = self.position_embeddings(position_ids)
|
||||
token_type_embeddings = self.token_type_embeddings(token_type_ids)
|
||||
|
||||
embeddings = inputs_embeds + token_type_embeddings
|
||||
if self.position_embedding_type == "absolute":
|
||||
position_embeddings = self.position_embeddings(position_ids)
|
||||
embeddings += position_embeddings
|
||||
embeddings = inputs_embeds + position_embeddings + token_type_embeddings
|
||||
embeddings = self.LayerNorm(embeddings)
|
||||
embeddings = self.dropout(embeddings)
|
||||
return embeddings
|
||||
@@ -205,10 +202,6 @@ class {{cookiecutter.camelcase_modelname}}SelfAttention(nn.Module):
|
||||
self.value = nn.Linear(config.hidden_size, self.all_head_size)
|
||||
|
||||
self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
|
||||
self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
|
||||
if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
|
||||
self.max_position_embeddings = config.max_position_embeddings
|
||||
self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)
|
||||
|
||||
def transpose_for_scores(self, x):
|
||||
new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
|
||||
@@ -243,23 +236,6 @@ class {{cookiecutter.camelcase_modelname}}SelfAttention(nn.Module):
|
||||
|
||||
# Take the dot product between "query" and "key" to get the raw attention scores.
|
||||
attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
|
||||
|
||||
if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
|
||||
seq_length = hidden_states.size()[1]
|
||||
position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1)
|
||||
position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1)
|
||||
distance = position_ids_l - position_ids_r
|
||||
positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
|
||||
positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility
|
||||
|
||||
if self.position_embedding_type == "relative_key":
|
||||
relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
|
||||
attention_scores = attention_scores + relative_position_scores
|
||||
elif self.position_embedding_type == "relative_key_query":
|
||||
relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
|
||||
relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
|
||||
attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
|
||||
|
||||
attention_scores = attention_scores / math.sqrt(self.attention_head_size)
|
||||
if attention_mask is not None:
|
||||
# Apply the attention mask is (precomputed for all layers in {{cookiecutter.camelcase_modelname}}Model forward() function)
|
||||
@@ -456,11 +432,10 @@ class {{cookiecutter.camelcase_modelname}}Encoder(nn.Module):
|
||||
encoder_attention_mask=None,
|
||||
output_attentions=False,
|
||||
output_hidden_states=False,
|
||||
return_dict=True,
|
||||
return_dict=False,
|
||||
):
|
||||
all_hidden_states = () if output_hidden_states else None
|
||||
all_self_attentions = () if output_attentions else None
|
||||
all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
|
||||
all_attentions = () if output_attentions else None
|
||||
for i, layer_module in enumerate(self.layer):
|
||||
if output_hidden_states:
|
||||
all_hidden_states = all_hidden_states + (hidden_states,)
|
||||
@@ -494,24 +469,15 @@ class {{cookiecutter.camelcase_modelname}}Encoder(nn.Module):
|
||||
)
|
||||
hidden_states = layer_outputs[0]
|
||||
if output_attentions:
|
||||
all_self_attentions = all_self_attentions + (layer_outputs[1],)
|
||||
if self.config.add_cross_attention:
|
||||
all_cross_attentions = all_cross_attentions + (layer_outputs[2],)
|
||||
all_attentions = all_attentions + (layer_outputs[1],)
|
||||
|
||||
if output_hidden_states:
|
||||
all_hidden_states = all_hidden_states + (hidden_states,)
|
||||
|
||||
if not return_dict:
|
||||
return tuple(
|
||||
v
|
||||
for v in [hidden_states, all_hidden_states, all_self_attentions, all_cross_attentions]
|
||||
if v is not None
|
||||
)
|
||||
return BaseModelOutputWithCrossAttentions(
|
||||
last_hidden_state=hidden_states,
|
||||
hidden_states=all_hidden_states,
|
||||
attentions=all_self_attentions,
|
||||
cross_attentions=all_cross_attentions,
|
||||
return tuple(v for v in [hidden_states, all_hidden_states, all_attentions] if v is not None)
|
||||
return BaseModelOutput(
|
||||
last_hidden_state=hidden_states, hidden_states=all_hidden_states, attentions=all_attentions
|
||||
)
|
||||
|
||||
|
||||
@@ -698,7 +664,7 @@ class {{cookiecutter.camelcase_modelname}}Model({{cookiecutter.camelcase_modelna
|
||||
@add_code_sample_docstrings(
|
||||
tokenizer_class=_TOKENIZER_FOR_DOC,
|
||||
checkpoint="{{cookiecutter.checkpoint_identifier}}",
|
||||
output_type=BaseModelOutputWithCrossAttentions,
|
||||
output_type=BaseModelOutput,
|
||||
config_class=_CONFIG_FOR_DOC,
|
||||
)
|
||||
def forward(
|
||||
@@ -789,11 +755,10 @@ class {{cookiecutter.camelcase_modelname}}Model({{cookiecutter.camelcase_modelna
|
||||
if not return_dict:
|
||||
return (sequence_output,) + encoder_outputs[1:]
|
||||
|
||||
return BaseModelOutputWithCrossAttentions(
|
||||
return BaseModelOutput(
|
||||
last_hidden_state=sequence_output,
|
||||
hidden_states=encoder_outputs.hidden_states,
|
||||
attentions=encoder_outputs.attentions,
|
||||
cross_attentions=encoder_outputs.cross_attentions,
|
||||
)
|
||||
|
||||
|
||||
@@ -898,127 +863,6 @@ class {{cookiecutter.camelcase_modelname}}ForMaskedLM({{cookiecutter.camelcase_m
|
||||
return {"input_ids": input_ids, "attention_mask": attention_mask}
|
||||
|
||||
|
||||
@add_start_docstrings(
|
||||
"""{{cookiecutter.modelname}} Model with a `language modeling` head on top for CLM fine-tuning. """, {{cookiecutter.uppercase_modelname}}_START_DOCSTRING
|
||||
)
|
||||
class {{cookiecutter.camelcase_modelname}}ForCausalLM({{cookiecutter.camelcase_modelname}}PreTrainedModel):
|
||||
|
||||
_keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"]
|
||||
|
||||
def __init__(self, config):
|
||||
super().__init__(config)
|
||||
|
||||
if not config.is_decoder:
|
||||
logger.warning("If you want to use `{{cookiecutter.camelcase_modelname}}ForCausalLM` as a standalone, add `is_decoder=True.`")
|
||||
|
||||
self.{{cookiecutter.lowercase_modelname}} = {{cookiecutter.camelcase_modelname}}Model(config)
|
||||
self.cls = {{cookiecutter.camelcase_modelname}}OnlyMLMHead(config)
|
||||
|
||||
self.init_weights()
|
||||
|
||||
def get_output_embeddings(self):
|
||||
return self.cls.predictions.decoder
|
||||
|
||||
def set_output_embeddings(self, new_embeddings):
|
||||
self.cls.predictions.decoder = new_embeddings
|
||||
|
||||
@add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
|
||||
@replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC)
|
||||
def forward(
|
||||
self,
|
||||
input_ids=None,
|
||||
attention_mask=None,
|
||||
token_type_ids=None,
|
||||
position_ids=None,
|
||||
head_mask=None,
|
||||
inputs_embeds=None,
|
||||
encoder_hidden_states=None,
|
||||
encoder_attention_mask=None,
|
||||
labels=None,
|
||||
output_attentions=None,
|
||||
output_hidden_states=None,
|
||||
return_dict=None,
|
||||
):
|
||||
r"""
|
||||
encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
|
||||
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
|
||||
the model is configured as a decoder.
|
||||
encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
|
||||
Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
|
||||
the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
|
||||
|
||||
- 1 for tokens that are **not masked**,
|
||||
- 0 for tokens that are **masked**.
|
||||
labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
|
||||
Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
|
||||
``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are
|
||||
ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]``.
|
||||
|
||||
Returns:
|
||||
|
||||
Example::
|
||||
|
||||
>>> from transformers import {{cookiecutter.camelcase_modelname}}Tokenizer, {{cookiecutter.camelcase_modelname}}ForCausalLM, {{cookiecutter.camelcase_modelname}}Config
|
||||
>>> import torch
|
||||
|
||||
>>> tokenizer = {{cookiecutter.camelcase_modelname}}Tokenizer.from_pretrained('{{cookiecutter.checkpoint_identifier}}')
|
||||
>>> config = {{cookiecutter.camelcase_modelname}}Config.from_pretrained("{{cookiecutter.checkpoint_identifier}}")
|
||||
>>> config.is_decoder = True
|
||||
>>> model = {{cookiecutter.camelcase_modelname}}ForCausalLM.from_pretrained('{{cookiecutter.checkpoint_identifier}}', config=config)
|
||||
|
||||
>>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
|
||||
>>> outputs = model(**inputs)
|
||||
|
||||
>>> prediction_logits = outputs.logits
|
||||
"""
|
||||
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
||||
|
||||
outputs = self.{{cookiecutter.lowercase_modelname}}(
|
||||
input_ids,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
position_ids=position_ids,
|
||||
head_mask=head_mask,
|
||||
inputs_embeds=inputs_embeds,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
output_attentions=output_attentions,
|
||||
output_hidden_states=output_hidden_states,
|
||||
return_dict=return_dict,
|
||||
)
|
||||
|
||||
sequence_output = outputs[0]
|
||||
prediction_scores = self.cls(sequence_output)
|
||||
|
||||
lm_loss = None
|
||||
if labels is not None:
|
||||
# we are doing next-token prediction; shift prediction scores and input ids by one
|
||||
shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous()
|
||||
labels = labels[:, 1:].contiguous()
|
||||
loss_fct = CrossEntropyLoss()
|
||||
lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
|
||||
|
||||
if not return_dict:
|
||||
output = (prediction_scores,) + outputs[1:]
|
||||
return ((lm_loss,) + output) if lm_loss is not None else output
|
||||
|
||||
return CausalLMOutputWithCrossAttentions(
|
||||
loss=lm_loss,
|
||||
logits=prediction_scores,
|
||||
hidden_states=outputs.hidden_states,
|
||||
attentions=outputs.attentions,
|
||||
cross_attentions=outputs.cross_attentions,
|
||||
)
|
||||
|
||||
def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **model_kwargs):
|
||||
input_shape = input_ids.shape
|
||||
|
||||
# if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
|
||||
if attention_mask is None:
|
||||
attention_mask = input_ids.new_ones(input_shape)
|
||||
|
||||
return {"input_ids": input_ids, "attention_mask": attention_mask}
|
||||
|
||||
class {{cookiecutter.camelcase_modelname}}ClassificationHead(nn.Module):
|
||||
"""Head for sentence-level classification tasks."""
|
||||
|
||||
|
||||
-49
@@ -32,7 +32,6 @@ if is_tf_available():
|
||||
TF{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForTokenClassification,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
TF{{cookiecutter.camelcase_modelname}}Model,
|
||||
)
|
||||
|
||||
@@ -135,21 +134,6 @@ class TF{{cookiecutter.camelcase_modelname}}ModelTester:
|
||||
|
||||
self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
|
||||
|
||||
def create_and_check_lm_head(
|
||||
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
config.is_decoder = True
|
||||
model = TF{{cookiecutter.camelcase_modelname}}ForCausalLM(config=config)
|
||||
inputs = {
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": input_mask,
|
||||
"token_type_ids": token_type_ids,
|
||||
}
|
||||
prediction_scores = model(inputs)["logits"]
|
||||
self.parent.assertListEqual(
|
||||
list(prediction_scores.numpy().shape), [self.batch_size, self.seq_length, self.vocab_size]
|
||||
)
|
||||
|
||||
def create_and_check_for_masked_lm(
|
||||
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
@@ -240,7 +224,6 @@ class TF{{cookiecutter.camelcase_modelname}}ModelTest(TFModelTesterMixin, unitte
|
||||
all_model_classes = (
|
||||
(
|
||||
TF{{cookiecutter.camelcase_modelname}}Model,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForMaskedLM,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
@@ -266,10 +249,6 @@ class TF{{cookiecutter.camelcase_modelname}}ModelTest(TFModelTesterMixin, unitte
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_for_masked_lm(*config_and_inputs)
|
||||
|
||||
def test_for_causal_lm(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_lm_head(*config_and_inputs)
|
||||
|
||||
def test_for_multiple_choice(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs)
|
||||
@@ -290,31 +269,3 @@ class TF{{cookiecutter.camelcase_modelname}}ModelTest(TFModelTesterMixin, unitte
|
||||
def test_model_from_pretrained(self):
|
||||
model = TF{{cookiecutter.camelcase_modelname}}Model.from_pretrained("{{cookiecutter.checkpoint_identifier}}")
|
||||
self.assertIsNotNone(model)
|
||||
|
||||
@require_tf
|
||||
class TF{{cookiecutter.camelcase_modelname}}ModelIntegrationTest(unittest.TestCase):
|
||||
@slow
|
||||
def test_inference_masked_lm(self):
|
||||
model = TF{{cookiecutter.camelcase_modelname}}ForMaskedLM.from_pretrained("{{cookiecutter.checkpoint_identifier}}")
|
||||
input_ids = tf.constant([[0, 1, 2, 3, 4, 5]])
|
||||
output = model(input_ids)[0]
|
||||
|
||||
# TODO Replace vocab size
|
||||
vocab_size = 32000
|
||||
|
||||
expected_shape = [1, 6, vocab_size]
|
||||
self.assertEqual(output.shape, expected_shape)
|
||||
|
||||
print(output[:, :3, :3])
|
||||
|
||||
# TODO Replace values below with what was printed above.
|
||||
expected_slice = tf.constant(
|
||||
[
|
||||
[
|
||||
[-0.05243197, -0.04498899, 0.05512108],
|
||||
[-0.07444685, -0.01064632, 0.04352357],
|
||||
[-0.05020351, 0.05530146, 0.00700043],
|
||||
]
|
||||
]
|
||||
)
|
||||
tf.debugging.assert_near(output[:, :3, :3], expected_slice, atol=1e-4)
|
||||
-141
@@ -17,7 +17,6 @@
|
||||
|
||||
import unittest
|
||||
|
||||
from tests.test_modeling_common import floats_tensor
|
||||
from transformers import is_torch_available
|
||||
from transformers.testing_utils import require_torch, slow, torch_device
|
||||
from .test_configuration_common import ConfigTester
|
||||
@@ -26,12 +25,9 @@ from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
import torch
|
||||
|
||||
from transformers import (
|
||||
{{cookiecutter.camelcase_modelname}}Config,
|
||||
{{cookiecutter.camelcase_modelname}}ForMaskedLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
|
||||
{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
@@ -126,33 +122,6 @@ class {{cookiecutter.camelcase_modelname}}ModelTester:
|
||||
|
||||
return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
|
||||
def prepare_config_and_inputs_for_decoder(self):
|
||||
(
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
) = self.prepare_config_and_inputs()
|
||||
|
||||
config.is_decoder = True
|
||||
encoder_hidden_states = floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
|
||||
encoder_attention_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
|
||||
|
||||
return (
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
)
|
||||
|
||||
def create_and_check_model(
|
||||
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
@@ -164,56 +133,6 @@ class {{cookiecutter.camelcase_modelname}}ModelTester:
|
||||
result = model(input_ids)
|
||||
self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
|
||||
|
||||
def create_and_check_model_as_decoder(
|
||||
self,
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
):
|
||||
config.add_cross_attention = True
|
||||
model = {{cookiecutter.camelcase_modelname}}Model(config)
|
||||
model.to(torch_device)
|
||||
model.eval()
|
||||
result = model(
|
||||
input_ids,
|
||||
attention_mask=input_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
)
|
||||
result = model(
|
||||
input_ids,
|
||||
attention_mask=input_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
)
|
||||
result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
|
||||
self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
|
||||
|
||||
def create_and_check_for_causal_lm(
|
||||
self,
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
):
|
||||
model = {{cookiecutter.camelcase_modelname}}ForCausalLM(config=config)
|
||||
model.to(torch_device)
|
||||
model.eval()
|
||||
result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
|
||||
|
||||
def create_and_check_for_masked_lm(
|
||||
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
@@ -299,7 +218,6 @@ class {{cookiecutter.camelcase_modelname}}ModelTest(ModelTesterMixin, unittest.T
|
||||
(
|
||||
{{cookiecutter.camelcase_modelname}}Model,
|
||||
{{cookiecutter.camelcase_modelname}}ForMaskedLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
|
||||
{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
@@ -308,7 +226,6 @@ class {{cookiecutter.camelcase_modelname}}ModelTest(ModelTesterMixin, unittest.T
|
||||
if is_torch_available()
|
||||
else ()
|
||||
)
|
||||
all_generative_model_classes = ({{cookiecutter.camelcase_modelname}}ForCausalLM,) if is_torch_available() else ()
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = {{cookiecutter.camelcase_modelname}}ModelTester(self)
|
||||
@@ -321,12 +238,6 @@ class {{cookiecutter.camelcase_modelname}}ModelTest(ModelTesterMixin, unittest.T
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_model(*config_and_inputs)
|
||||
|
||||
def test_model_various_embeddings(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
for type in ["absolute", "relative_key", "relative_key_query"]:
|
||||
config_and_inputs[0].position_embedding_type = type
|
||||
self.model_tester.create_and_check_model(*config_and_inputs)
|
||||
|
||||
def test_for_masked_lm(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_for_masked_lm(*config_and_inputs)
|
||||
@@ -347,38 +258,6 @@ class {{cookiecutter.camelcase_modelname}}ModelTest(ModelTesterMixin, unittest.T
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_for_token_classification(*config_and_inputs)
|
||||
|
||||
def test_model_as_decoder(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
|
||||
self.model_tester.create_and_check_model_as_decoder(*config_and_inputs)
|
||||
|
||||
def test_model_as_decoder_with_default_input_mask(self):
|
||||
# This regression test was failing with PyTorch < 1.3
|
||||
(
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
) = self.model_tester.prepare_config_and_inputs_for_decoder()
|
||||
|
||||
input_mask = None
|
||||
|
||||
self.model_tester.create_and_check_model_as_decoder(
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
)
|
||||
|
||||
@slow
|
||||
def test_model_from_pretrained(self):
|
||||
for model_name in {{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
@@ -386,23 +265,3 @@ class {{cookiecutter.camelcase_modelname}}ModelTest(ModelTesterMixin, unittest.T
|
||||
self.assertIsNotNone(model)
|
||||
|
||||
|
||||
@require_torch
|
||||
class {{cookiecutter.camelcase_modelname}}ModelIntegrationTest(unittest.TestCase):
|
||||
@slow
|
||||
def test_inference_masked_lm(self):
|
||||
model = {{cookiecutter.camelcase_modelname}}ForMaskedLM.from_pretrained("{{cookiecutter.checkpoint_identifier}}")
|
||||
input_ids = torch.tensor([[0, 1, 2, 3, 4, 5]])
|
||||
output = model(input_ids)[0]
|
||||
|
||||
# TODO Replace vocab size
|
||||
vocab_size = 32000
|
||||
|
||||
expected_shape = torch.Size((1, 6, vocab_size))
|
||||
self.assertEqual(output.shape, expected_shape)
|
||||
|
||||
# TODO Replace values below with what was printed above.
|
||||
expected_slice = torch.tensor(
|
||||
[[[-0.0483, 0.1188, -0.0313], [-0.0606, 0.1435, 0.0199], [-0.0235, 0.1519, 0.0175]]]
|
||||
)
|
||||
|
||||
self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4))
|
||||
+3
-18
@@ -31,7 +31,6 @@
|
||||
from .models.{{cookiecutter.lowercase_modelname}} import (
|
||||
{{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
{{cookiecutter.camelcase_modelname}}ForMaskedLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
|
||||
{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
@@ -48,7 +47,6 @@
|
||||
from .models.{{cookiecutter.lowercase_modelname}} import (
|
||||
TF_{{cookiecutter.uppercase_modelname}}_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForMaskedLM,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
@@ -59,13 +57,10 @@
|
||||
)
|
||||
# End.
|
||||
|
||||
# Below: "if is_tokenizers_available():"
|
||||
# Replace with:
|
||||
from models.{{cookiecutter.lowercase_modelname}} import {{cookiecutter.camelcase_modelname}}TokenizerFast
|
||||
|
||||
# Below: "from .models.albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig"
|
||||
# Replace with:
|
||||
from .models.{{cookiecutter.lowercase_modelname}} import {{cookiecutter.uppercase_modelname}}_PRETRAINED_CONFIG_ARCHIVE_MAP, {{cookiecutter.camelcase_modelname}}Config, {{cookiecutter.camelcase_modelname}}Tokenizer
|
||||
from .models.{{cookiecutter.lowercase_modelname}} import {{cookiecutter.uppercase_modelname}}_PRETRAINED_CONFIG_ARCHIVE_MAP, {{cookiecutter.camelcase_modelname}}Config
|
||||
# End.
|
||||
|
||||
|
||||
@@ -101,9 +96,9 @@ from ..{{cookiecutter.lowercase_modelname}}.configuration_{{cookiecutter.lowerca
|
||||
|
||||
# Below: "# Add modeling imports here"
|
||||
# Replace with:
|
||||
|
||||
from ..{{cookiecutter.lowercase_modelname}}.modeling_{{cookiecutter.lowercase_modelname}} import (
|
||||
{{cookiecutter.camelcase_modelname}}ForMaskedLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
|
||||
{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
@@ -122,11 +117,6 @@ from ..{{cookiecutter.lowercase_modelname}}.modeling_{{cookiecutter.lowercase_mo
|
||||
({{cookiecutter.camelcase_modelname}}Config, {{cookiecutter.camelcase_modelname}}ForMaskedLM),
|
||||
# End.
|
||||
|
||||
# Below: "# Model for Causal LM mapping"
|
||||
# Replace with:
|
||||
({{cookiecutter.camelcase_modelname}}Config, {{cookiecutter.camelcase_modelname}}ForCausalLM),
|
||||
# End.
|
||||
|
||||
# Below: "# Model for Masked LM mapping"
|
||||
# Replace with:
|
||||
({{cookiecutter.camelcase_modelname}}Config, {{cookiecutter.camelcase_modelname}}ForMaskedLM),
|
||||
@@ -161,9 +151,9 @@ from ..{{cookiecutter.lowercase_modelname}}.modeling_{{cookiecutter.lowercase_mo
|
||||
|
||||
# Below: "# Add modeling imports here"
|
||||
# Replace with:
|
||||
|
||||
from ..{{cookiecutter.lowercase_modelname}}.modeling_tf_{{cookiecutter.lowercase_modelname}} import (
|
||||
TF{{cookiecutter.camelcase_modelname}}ForMaskedLM,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForCausalLM,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForMultipleChoice,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForQuestionAnswering,
|
||||
TF{{cookiecutter.camelcase_modelname}}ForSequenceClassification,
|
||||
@@ -182,11 +172,6 @@ from ..{{cookiecutter.lowercase_modelname}}.modeling_tf_{{cookiecutter.lowercase
|
||||
({{cookiecutter.camelcase_modelname}}Config, TF{{cookiecutter.camelcase_modelname}}ForMaskedLM),
|
||||
# End.
|
||||
|
||||
# Below: "# Model for Causal LM mapping"
|
||||
# Replace with:
|
||||
({{cookiecutter.camelcase_modelname}}Config, TF{{cookiecutter.camelcase_modelname}}ForCausalLM),
|
||||
# End.
|
||||
|
||||
# Below: "# Model for Masked LM mapping"
|
||||
# Replace with:
|
||||
({{cookiecutter.camelcase_modelname}}Config, TF{{cookiecutter.camelcase_modelname}}ForMaskedLM),
|
||||
|
||||
-150
@@ -1,150 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright {{cookiecutter.authors}} and The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Tokenization classes for {{cookiecutter.modelname}}."""
|
||||
|
||||
{%- if cookiecutter.tokenizer_type == "Based on BERT" %}
|
||||
from ...utils import logging
|
||||
from ..bert.tokenization_bert_fast import BertTokenizerFast
|
||||
from .tokenization_{{cookiecutter.lowercase_modelname}} import {{cookiecutter.camelcase_modelname}}Tokenizer
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
|
||||
|
||||
PRETRAINED_VOCAB_FILES_MAP = {
|
||||
"vocab_file": {
|
||||
"{{cookiecutter.checkpoint_identifier}}": "https://huggingface.co/{{cookiecutter.checkpoint_identifier}}/resolve/main/vocab.txt",
|
||||
}
|
||||
}
|
||||
|
||||
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
|
||||
"{{cookiecutter.checkpoint_identifier}}": 512,
|
||||
}
|
||||
|
||||
|
||||
PRETRAINED_INIT_CONFIGURATION = {
|
||||
"{{cookiecutter.checkpoint_identifier}}": {"do_lower_case": False},
|
||||
}
|
||||
|
||||
|
||||
class {{cookiecutter.camelcase_modelname}}TokenizerFast(BertTokenizerFast):
|
||||
r"""
|
||||
Construct a "fast" {{cookiecutter.modelname}} tokenizer (backed by HuggingFace's `tokenizers` library).
|
||||
|
||||
:class:`~transformers.{{cookiecutter.camelcase_modelname}}TokenizerFast` is identical to :class:`~transformers.BertTokenizerFast` and runs
|
||||
end-to-end tokenization: punctuation splitting and wordpiece.
|
||||
|
||||
Refer to superclass :class:`~transformers.BertTokenizerFast` for usage examples and documentation concerning
|
||||
parameters.
|
||||
"""
|
||||
|
||||
vocab_files_names = VOCAB_FILES_NAMES
|
||||
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
|
||||
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
|
||||
pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
|
||||
slow_tokenizer_class = {{cookiecutter.camelcase_modelname}}Tokenizer
|
||||
{%- elif cookiecutter.tokenizer_type == "Standalone" %}
|
||||
from typing import List, Optional
|
||||
|
||||
from tokenizers import ByteLevelBPETokenizer
|
||||
|
||||
from ...tokenization_utils_fast import PreTrainedTokenizerFast
|
||||
from ...utils import logging
|
||||
from .tokenization_{{cookiecutter.lowercase_modelname}} import {{cookiecutter.camelcase_modelname}}Tokenizer
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
VOCAB_FILES_NAMES = {}
|
||||
|
||||
PRETRAINED_VOCAB_FILES_MAP = {}
|
||||
|
||||
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
|
||||
"{{cookiecutter.checkpoint_identifier}}": 1024,
|
||||
}
|
||||
|
||||
class {{cookiecutter.camelcase_modelname}}TokenizerFast(PreTrainedTokenizerFast):
|
||||
"""
|
||||
Construct a "fast" {{cookiecutter.modelname}} tokenizer (backed by HuggingFace's `tokenizers` library).
|
||||
|
||||
Args:
|
||||
vocab_file (:obj:`str`):
|
||||
Path to the vocabulary file.
|
||||
"""
|
||||
|
||||
vocab_files_names = VOCAB_FILES_NAMES
|
||||
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
|
||||
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
|
||||
slow_tokenizer_class = {{cookiecutter.camelcase_modelname}}Tokenizer
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_file,
|
||||
merges_file,
|
||||
unk_token="<|endoftext|>",
|
||||
bos_token="<|endoftext|>",
|
||||
eos_token="<|endoftext|>",
|
||||
add_prefix_space=False,
|
||||
trim_offsets=True,
|
||||
**kwargs
|
||||
):
|
||||
super().__init__(
|
||||
ByteLevelBPETokenizer(
|
||||
vocab_file=vocab_file,
|
||||
merges_file=merges_file,
|
||||
add_prefix_space=add_prefix_space,
|
||||
trim_offsets=trim_offsets,
|
||||
),
|
||||
bos_token=bos_token,
|
||||
eos_token=eos_token,
|
||||
unk_token=unk_token,
|
||||
**kwargs,
|
||||
)
|
||||
self.add_prefix_space = add_prefix_space
|
||||
|
||||
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
|
||||
output = [self.bos_token_id] + token_ids_0 + [self.eos_token_id]
|
||||
if token_ids_1 is None:
|
||||
return output
|
||||
|
||||
return output + [self.eos_token_id] + token_ids_1 + [self.eos_token_id]
|
||||
|
||||
|
||||
def create_token_type_ids_from_sequences(
|
||||
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
|
||||
) -> List[int]:
|
||||
"""
|
||||
Create a mask from the two sequences passed to be used in a sequence-pair classification task.
|
||||
{{cookiecutter.modelname}} does not make use of token type ids, therefore a list of zeros is returned.
|
||||
|
||||
Args:
|
||||
token_ids_0 (:obj:`List[int]`):
|
||||
List of IDs.
|
||||
token_ids_1 (:obj:`List[int]`, `optional`):
|
||||
Optional second list of IDs for sequence pairs.
|
||||
|
||||
Returns:
|
||||
:obj:`List[int]`: List of zeros.
|
||||
"""
|
||||
sep = [self.sep_token_id]
|
||||
cls = [self.cls_token_id]
|
||||
|
||||
if token_ids_1 is None:
|
||||
return len(cls + token_ids_0 + sep) * [0]
|
||||
return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]
|
||||
|
||||
|
||||
{% endif %}
|
||||
+19
@@ -17,6 +17,7 @@
|
||||
{%- if cookiecutter.tokenizer_type == "Based on BERT" %}
|
||||
from ...utils import logging
|
||||
from ..bert.tokenization_bert import BertTokenizer
|
||||
from ..bert.tokenization_bert_fast import BertTokenizerFast
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
@@ -54,12 +55,30 @@ class {{cookiecutter.camelcase_modelname}}Tokenizer(BertTokenizer):
|
||||
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
|
||||
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
|
||||
pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
|
||||
|
||||
|
||||
class {{cookiecutter.camelcase_modelname}}TokenizerFast(BertTokenizerFast):
|
||||
r"""
|
||||
Construct a "fast" {{cookiecutter.modelname}} tokenizer (backed by HuggingFace's `tokenizers` library).
|
||||
|
||||
:class:`~transformers.{{cookiecutter.camelcase_modelname}}TokenizerFast` is identical to :class:`~transformers.BertTokenizerFast` and runs
|
||||
end-to-end tokenization: punctuation splitting and wordpiece.
|
||||
|
||||
Refer to superclass :class:`~transformers.BertTokenizerFast` for usage examples and documentation concerning
|
||||
parameters.
|
||||
"""
|
||||
|
||||
vocab_files_names = VOCAB_FILES_NAMES
|
||||
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
|
||||
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
|
||||
pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
|
||||
{%- elif cookiecutter.tokenizer_type == "Standalone" %}
|
||||
from typing import List, Optional
|
||||
|
||||
from tokenizers import ByteLevelBPETokenizer
|
||||
|
||||
from ...tokenization_utils import AddedToken, PreTrainedTokenizer
|
||||
from ...tokenization_utils_base import BatchEncoding
|
||||
from ...tokenization_utils_fast import PreTrainedTokenizerFast
|
||||
from ...utils import logging
|
||||
|
||||
|
||||
-14
@@ -58,13 +58,6 @@ Tips:
|
||||
:members: forward
|
||||
|
||||
|
||||
{{cookiecutter.camelcase_modelname}}ForCausalLM
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.{{cookiecutter.camelcase_modelname}}ForCausalLM
|
||||
:members: forward
|
||||
|
||||
|
||||
{{cookiecutter.camelcase_modelname}}ForMaskedLM
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
@@ -116,13 +109,6 @@ TF{{cookiecutter.camelcase_modelname}}ForMaskedLM
|
||||
:members: call
|
||||
|
||||
|
||||
TF{{cookiecutter.camelcase_modelname}}ForCausalLM
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: transformers.TF{{cookiecutter.camelcase_modelname}}ForCausalLM
|
||||
:members: forward
|
||||
|
||||
|
||||
TF{{cookiecutter.camelcase_modelname}}ForSequenceClassification
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"modelname": "Template",
|
||||
"uppercase_modelname": "TEMPLATE",
|
||||
"lowercase_modelname": "template",
|
||||
"camelcase_modelname": "Template",
|
||||
"modelname": "EncoderBERT",
|
||||
"uppercase_modelname": "ENCODER_BERT",
|
||||
"lowercase_modelname": "encoder_bert",
|
||||
"camelcase_modelname": "EncoderBert",
|
||||
"authors": "The HuggingFace Team",
|
||||
"checkpoint_identifier": "brand-new-bert-base-cased",
|
||||
"tokenizer_type": "Based on BERT",
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"modelname": "TemplatePT",
|
||||
"uppercase_modelname": "TEMPLATE_PT",
|
||||
"lowercase_modelname": "template_pt",
|
||||
"camelcase_modelname": "TemplatePt",
|
||||
"modelname": "PTEncoderBERT",
|
||||
"uppercase_modelname": "PT_ENCODER_BERT",
|
||||
"lowercase_modelname": "pt_encoder_bert",
|
||||
"camelcase_modelname": "PtEncoderBert",
|
||||
"authors": "The HuggingFace Team",
|
||||
"checkpoint_identifier": "brand-new-bert-base-cased",
|
||||
"tokenizer_type": "Based on BERT",
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"modelname": "TemplateBI",
|
||||
"uppercase_modelname": "TEMPLATE_BI",
|
||||
"lowercase_modelname": "template_bi",
|
||||
"camelcase_modelname": "TemplateBi",
|
||||
"modelname": "BIEncoderBERT",
|
||||
"uppercase_modelname": "BI_ENCODER_BERT",
|
||||
"lowercase_modelname": "bi_encoder_bert",
|
||||
"camelcase_modelname": "BiEncoderBert",
|
||||
"authors": "The HuggingFace Team",
|
||||
"checkpoint_identifier": "bi-brand-new-bert-base-cased",
|
||||
"tokenizer_type": "Standalone",
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"modelname": "TemplateTF",
|
||||
"uppercase_modelname": "TEMPLATE_TF",
|
||||
"lowercase_modelname": "template_tf",
|
||||
"camelcase_modelname": "TemplateTf",
|
||||
"modelname": "TFEncoderBERT",
|
||||
"uppercase_modelname": "TF_ENCODER_BERT",
|
||||
"lowercase_modelname": "tf_encoder_bert",
|
||||
"camelcase_modelname": "TfEncoderBert",
|
||||
"authors": "The HuggingFace Team",
|
||||
"checkpoint_identifier": "brand-new-bert-base-cased",
|
||||
"tokenizer_type": "Based on BERT",
|
||||
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"version": "v2.0",
|
||||
"data": [{
|
||||
"title": "Normans",
|
||||
"paragraphs": [{
|
||||
"qas": [{
|
||||
"question": "In what country is Normandy located?",
|
||||
"id": "56ddde6b9a695914005b9628",
|
||||
"answers": [{
|
||||
"text": "France",
|
||||
"answer_start": 159
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"question": "When were the Normans in Normandy?",
|
||||
"id": "56ddde6b9a695914005b9629",
|
||||
"answers": [{
|
||||
"text": "10th and 11th centuries",
|
||||
"answer_start": 94
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"question": "From which countries did the Norse originate?",
|
||||
"id": "56ddde6b9a695914005b962a",
|
||||
"answers": [{
|
||||
"text": "Denmark, Iceland and Norway",
|
||||
"answer_start": 256
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "Rollo",
|
||||
"answer_start": 308
|
||||
}],
|
||||
"question": "Who did King Charles III swear fealty to?",
|
||||
"id": "5ad39d53604f3c001a3fe8d3",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "10th century",
|
||||
"answer_start": 671
|
||||
}],
|
||||
"question": "When did the Frankish identity emerge?",
|
||||
"id": "5ad39d53604f3c001a3fe8d4",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}],
|
||||
"context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries."
|
||||
}, {
|
||||
"qas": [{
|
||||
"question": "Who was the duke in the battle of Hastings?",
|
||||
"id": "56dddf4066d3e219004dad5f",
|
||||
"answers": [{
|
||||
"text": "William the Conqueror",
|
||||
"answer_start": 1022
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "Antioch",
|
||||
"answer_start": 1295
|
||||
}],
|
||||
"question": "What principality did William the conquerer found?",
|
||||
"id": "5ad3a266604f3c001a3fea2b",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}],
|
||||
"context": "The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands."
|
||||
}]
|
||||
}, {
|
||||
"title": "Computational_complexity_theory",
|
||||
"paragraphs": [{
|
||||
"qas": [{
|
||||
"question": "What branch of theoretical computer science deals with broadly classifying computational problems by difficulty and class of relationship?",
|
||||
"id": "56e16182e3433e1400422e28",
|
||||
"answers": [{
|
||||
"text": "Computational complexity theory",
|
||||
"answer_start": 0
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "algorithm",
|
||||
"answer_start": 472
|
||||
}],
|
||||
"question": "What is a manual application of mathematical steps?",
|
||||
"id": "5ad5316b5b96ef001a10ab76",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}],
|
||||
"context": "Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm."
|
||||
}, {
|
||||
"qas": [{
|
||||
"question": "What measure of a computational problem broadly defines the inherent difficulty of the solution?",
|
||||
"id": "56e16839cd28a01900c67887",
|
||||
"answers": [{
|
||||
"text": "if its solution requires significant resources",
|
||||
"answer_start": 46
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"question": "What method is used to intuitively assess or quantify the amount of resources required to solve a computational problem?",
|
||||
"id": "56e16839cd28a01900c67888",
|
||||
"answers": [{
|
||||
"text": "mathematical models of computation",
|
||||
"answer_start": 176
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"question": "What are two basic primary resources used to guage complexity?",
|
||||
"id": "56e16839cd28a01900c67889",
|
||||
"answers": [{
|
||||
"text": "time and storage",
|
||||
"answer_start": 305
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "the number of gates in a circuit",
|
||||
"answer_start": 436
|
||||
}],
|
||||
"question": "What unit is measured to determine circuit simplicity?",
|
||||
"id": "5ad532575b96ef001a10ab7f",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "the number of processors",
|
||||
"answer_start": 502
|
||||
}],
|
||||
"question": "What number is used in perpendicular computing?",
|
||||
"id": "5ad532575b96ef001a10ab80",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}],
|
||||
"context": "A problem is regarded as inherently difficult if its solution requires significant resources, whatever the algorithm used. The theory formalizes this intuition, by introducing mathematical models of computation to study these problems and quantifying the amount of resources needed to solve them, such as time and storage. Other complexity measures are also used, such as the amount of communication (used in communication complexity), the number of gates in a circuit (used in circuit complexity) and the number of processors (used in parallel computing). One of the roles of computational complexity theory is to determine the practical limits on what computers can and cannot do."
|
||||
}]
|
||||
}]
|
||||
}
|
||||
-201
@@ -1,201 +0,0 @@
|
||||
{
|
||||
"version": 2.0,
|
||||
"data": [
|
||||
{
|
||||
"id": "56ddde6b9a695914005b9628",
|
||||
"question": "In what country is Normandy located?",
|
||||
"context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.",
|
||||
"answers": {
|
||||
"answer_start": [
|
||||
159,
|
||||
159,
|
||||
159,
|
||||
159
|
||||
],
|
||||
"text": [
|
||||
"France",
|
||||
"France",
|
||||
"France",
|
||||
"France"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "56ddde6b9a695914005b9629",
|
||||
"question": "When were the Normans in Normandy?",
|
||||
"context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.",
|
||||
"answers": {
|
||||
"answer_start": [
|
||||
94,
|
||||
87,
|
||||
94,
|
||||
94
|
||||
],
|
||||
"text": [
|
||||
"10th and 11th centuries",
|
||||
"in the 10th and 11th centuries",
|
||||
"10th and 11th centuries",
|
||||
"10th and 11th centuries"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "56ddde6b9a695914005b962a",
|
||||
"question": "From which countries did the Norse originate?",
|
||||
"context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.",
|
||||
"answers": {
|
||||
"answer_start": [
|
||||
256,
|
||||
256,
|
||||
256,
|
||||
256
|
||||
],
|
||||
"text": [
|
||||
"Denmark, Iceland and Norway",
|
||||
"Denmark, Iceland and Norway",
|
||||
"Denmark, Iceland and Norway",
|
||||
"Denmark, Iceland and Norway"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5ad39d53604f3c001a3fe8d3",
|
||||
"question": "Who did King Charles III swear fealty to?",
|
||||
"context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.",
|
||||
"answers": {
|
||||
"answer_start": [],
|
||||
"text": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5ad39d53604f3c001a3fe8d4",
|
||||
"question": "When did the Frankish identity emerge?",
|
||||
"context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.",
|
||||
"answers": {
|
||||
"answer_start": [],
|
||||
"text": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "56dddf4066d3e219004dad5f",
|
||||
"question": "Who was the duke in the battle of Hastings?",
|
||||
"context": "The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands.",
|
||||
"answers": {
|
||||
"answer_start": [
|
||||
1022,
|
||||
1022,
|
||||
1022
|
||||
],
|
||||
"text": [
|
||||
"William the Conqueror",
|
||||
"William the Conqueror",
|
||||
"William the Conqueror"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5ad3a266604f3c001a3fea2b",
|
||||
"question": "What principality did William the conquerer found?",
|
||||
"context": "The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands.",
|
||||
"answers": {
|
||||
"answer_start": [],
|
||||
"text": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "56e16182e3433e1400422e28",
|
||||
"question": "What branch of theoretical computer science deals with broadly classifying computational problems by difficulty and class of relationship?",
|
||||
"context": "Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm.",
|
||||
"answers": {
|
||||
"answer_start": [
|
||||
0,
|
||||
0,
|
||||
0
|
||||
],
|
||||
"text": [
|
||||
"Computational complexity theory",
|
||||
"Computational complexity theory",
|
||||
"Computational complexity theory"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5ad5316b5b96ef001a10ab76",
|
||||
"question": "What is a manual application of mathematical steps?",
|
||||
"context": "Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm.",
|
||||
"answers": {
|
||||
"answer_start": [],
|
||||
"text": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "56e16839cd28a01900c67887",
|
||||
"question": "What measure of a computational problem broadly defines the inherent difficulty of the solution?",
|
||||
"context": "A problem is regarded as inherently difficult if its solution requires significant resources, whatever the algorithm used. The theory formalizes this intuition, by introducing mathematical models of computation to study these problems and quantifying the amount of resources needed to solve them, such as time and storage. Other complexity measures are also used, such as the amount of communication (used in communication complexity), the number of gates in a circuit (used in circuit complexity) and the number of processors (used in parallel computing). One of the roles of computational complexity theory is to determine the practical limits on what computers can and cannot do.",
|
||||
"answers": {
|
||||
"answer_start": [
|
||||
46,
|
||||
49,
|
||||
46
|
||||
],
|
||||
"text": [
|
||||
"if its solution requires significant resources",
|
||||
"its solution requires significant resources",
|
||||
"if its solution requires significant resources"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "56e16839cd28a01900c67888",
|
||||
"question": "What method is used to intuitively assess or quantify the amount of resources required to solve a computational problem?",
|
||||
"context": "A problem is regarded as inherently difficult if its solution requires significant resources, whatever the algorithm used. The theory formalizes this intuition, by introducing mathematical models of computation to study these problems and quantifying the amount of resources needed to solve them, such as time and storage. Other complexity measures are also used, such as the amount of communication (used in communication complexity), the number of gates in a circuit (used in circuit complexity) and the number of processors (used in parallel computing). One of the roles of computational complexity theory is to determine the practical limits on what computers can and cannot do.",
|
||||
"answers": {
|
||||
"answer_start": [
|
||||
176,
|
||||
176,
|
||||
176
|
||||
],
|
||||
"text": [
|
||||
"mathematical models of computation",
|
||||
"mathematical models of computation",
|
||||
"mathematical models of computation"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "56e16839cd28a01900c67889",
|
||||
"question": "What are two basic primary resources used to guage complexity?",
|
||||
"context": "A problem is regarded as inherently difficult if its solution requires significant resources, whatever the algorithm used. The theory formalizes this intuition, by introducing mathematical models of computation to study these problems and quantifying the amount of resources needed to solve them, such as time and storage. Other complexity measures are also used, such as the amount of communication (used in communication complexity), the number of gates in a circuit (used in circuit complexity) and the number of processors (used in parallel computing). One of the roles of computational complexity theory is to determine the practical limits on what computers can and cannot do.",
|
||||
"answers": {
|
||||
"answer_start": [
|
||||
305,
|
||||
305,
|
||||
305
|
||||
],
|
||||
"text": [
|
||||
"time and storage",
|
||||
"time and storage",
|
||||
"time and storage"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5ad532575b96ef001a10ab7f",
|
||||
"question": "What unit is measured to determine circuit simplicity?",
|
||||
"context": "A problem is regarded as inherently difficult if its solution requires significant resources, whatever the algorithm used. The theory formalizes this intuition, by introducing mathematical models of computation to study these problems and quantifying the amount of resources needed to solve them, such as time and storage. Other complexity measures are also used, such as the amount of communication (used in communication complexity), the number of gates in a circuit (used in circuit complexity) and the number of processors (used in parallel computing). One of the roles of computational complexity theory is to determine the practical limits on what computers can and cannot do.",
|
||||
"answers": {
|
||||
"answer_start": [],
|
||||
"text": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5ad532575b96ef001a10ab80",
|
||||
"question": "What number is used in perpendicular computing?",
|
||||
"context": "A problem is regarded as inherently difficult if its solution requires significant resources, whatever the algorithm used. The theory formalizes this intuition, by introducing mathematical models of computation to study these problems and quantifying the amount of resources needed to solve them, such as time and storage. Other complexity measures are also used, such as the amount of communication (used in communication complexity), the number of gates in a circuit (used in circuit complexity) and the number of processors (used in parallel computing). One of the roles of computational complexity theory is to determine the practical limits on what computers can and cannot do.",
|
||||
"answers": {
|
||||
"answer_start": [],
|
||||
"text": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"version": "v2.0",
|
||||
"data": [{
|
||||
"title": "Normans",
|
||||
"paragraphs": [{
|
||||
"qas": [{
|
||||
"question": "In what country is Normandy located?",
|
||||
"id": "56ddde6b9a695914005b9628",
|
||||
"answers": [{
|
||||
"text": "France",
|
||||
"answer_start": 159
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"question": "When were the Normans in Normandy?",
|
||||
"id": "56ddde6b9a695914005b9629",
|
||||
"answers": [{
|
||||
"text": "10th and 11th centuries",
|
||||
"answer_start": 94
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"question": "From which countries did the Norse originate?",
|
||||
"id": "56ddde6b9a695914005b962a",
|
||||
"answers": [{
|
||||
"text": "Denmark, Iceland and Norway",
|
||||
"answer_start": 256
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "Rollo",
|
||||
"answer_start": 308
|
||||
}],
|
||||
"question": "Who did King Charles III swear fealty to?",
|
||||
"id": "5ad39d53604f3c001a3fe8d3",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "10th century",
|
||||
"answer_start": 671
|
||||
}],
|
||||
"question": "When did the Frankish identity emerge?",
|
||||
"id": "5ad39d53604f3c001a3fe8d4",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}],
|
||||
"context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries."
|
||||
}, {
|
||||
"qas": [{
|
||||
"question": "Who was the duke in the battle of Hastings?",
|
||||
"id": "56dddf4066d3e219004dad5f",
|
||||
"answers": [{
|
||||
"text": "William the Conqueror",
|
||||
"answer_start": 1022
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "Antioch",
|
||||
"answer_start": 1295
|
||||
}],
|
||||
"question": "What principality did William the conquerer found?",
|
||||
"id": "5ad3a266604f3c001a3fea2b",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}],
|
||||
"context": "The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands."
|
||||
}]
|
||||
}, {
|
||||
"title": "Computational_complexity_theory",
|
||||
"paragraphs": [{
|
||||
"qas": [{
|
||||
"question": "What branch of theoretical computer science deals with broadly classifying computational problems by difficulty and class of relationship?",
|
||||
"id": "56e16182e3433e1400422e28",
|
||||
"answers": [{
|
||||
"text": "Computational complexity theory",
|
||||
"answer_start": 0
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "algorithm",
|
||||
"answer_start": 472
|
||||
}],
|
||||
"question": "What is a manual application of mathematical steps?",
|
||||
"id": "5ad5316b5b96ef001a10ab76",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}],
|
||||
"context": "Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm."
|
||||
}, {
|
||||
"qas": [{
|
||||
"question": "What measure of a computational problem broadly defines the inherent difficulty of the solution?",
|
||||
"id": "56e16839cd28a01900c67887",
|
||||
"answers": [{
|
||||
"text": "if its solution requires significant resources",
|
||||
"answer_start": 46
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"question": "What method is used to intuitively assess or quantify the amount of resources required to solve a computational problem?",
|
||||
"id": "56e16839cd28a01900c67888",
|
||||
"answers": [{
|
||||
"text": "mathematical models of computation",
|
||||
"answer_start": 176
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"question": "What are two basic primary resources used to guage complexity?",
|
||||
"id": "56e16839cd28a01900c67889",
|
||||
"answers": [{
|
||||
"text": "time and storage",
|
||||
"answer_start": 305
|
||||
}],
|
||||
"is_impossible": false
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "the number of gates in a circuit",
|
||||
"answer_start": 436
|
||||
}],
|
||||
"question": "What unit is measured to determine circuit simplicity?",
|
||||
"id": "5ad532575b96ef001a10ab7f",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}, {
|
||||
"plausible_answers": [{
|
||||
"text": "the number of processors",
|
||||
"answer_start": 502
|
||||
}],
|
||||
"question": "What number is used in perpendicular computing?",
|
||||
"id": "5ad532575b96ef001a10ab80",
|
||||
"answers": [],
|
||||
"is_impossible": true
|
||||
}],
|
||||
"context": "A problem is regarded as inherently difficult if its solution requires significant resources, whatever the algorithm used. The theory formalizes this intuition, by introducing mathematical models of computation to study these problems and quantifying the amount of resources needed to solve them, such as time and storage. Other complexity measures are also used, such as the amount of communication (used in communication complexity), the number of gates in a circuit (used in circuit complexity) and the number of processors (used in parallel computing). One of the roles of computational complexity theory is to determine the practical limits on what computers can and cannot do."
|
||||
}]
|
||||
}]
|
||||
}
|
||||
@@ -27,7 +27,6 @@ if is_torch_available():
|
||||
import torch.nn.functional as F
|
||||
|
||||
from transformers.generation_logits_process import (
|
||||
HammingDiversityLogitsProcessor,
|
||||
LogitsProcessorList,
|
||||
MinLengthLogitsProcessor,
|
||||
NoBadWordsLogitsProcessor,
|
||||
@@ -303,30 +302,3 @@ class LogitsProcessorTest(unittest.TestCase):
|
||||
self.assertListEqual(
|
||||
torch.isinf(filtered_scores).tolist(), [[False, False, True, True, True], [True, True, False, False, True]]
|
||||
)
|
||||
|
||||
def test_hamming_diversity(self):
|
||||
vocab_size = 4
|
||||
num_beams = 2
|
||||
num_beam_groups = 2
|
||||
|
||||
scores = self._get_uniform_logits(num_beams, vocab_size)
|
||||
# batch_idx = 0 -> index batch_idx * num_beam_groups -> idx = 0 * 2 = 0 -> penalises tokens 1
|
||||
# batch_idx = 1 -> index batch_idx * num_beam_groups -> idx = 1 * 2 = 2 -> penalises tokens 1
|
||||
current_tokens = torch.tensor([0, 3, 1, 2], device=torch_device, dtype=torch.long)
|
||||
|
||||
diversity_logits_processor = HammingDiversityLogitsProcessor(
|
||||
diversity_penalty=1.0, num_beams=num_beams, num_beam_groups=num_beam_groups
|
||||
)
|
||||
|
||||
processed_scores = diversity_logits_processor(None, scores, current_tokens, 1)
|
||||
|
||||
self.assertTrue(
|
||||
torch.allclose(
|
||||
processed_scores[0], torch.tensor([-0.7500, 0.2500, 0.2500, 0.2500], device=torch_device), atol=1e-3
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
torch.allclose(
|
||||
processed_scores[1], torch.tensor([0.2500, -0.7500, 0.2500, 0.2500], device=torch_device), atol=1e-3
|
||||
)
|
||||
)
|
||||
@@ -17,16 +17,15 @@
|
||||
import unittest
|
||||
|
||||
from transformers import is_torch_available
|
||||
from transformers.testing_utils import require_torch, slow, torch_device
|
||||
from transformers.testing_utils import require_torch, torch_device
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
import torch
|
||||
|
||||
from transformers import BartForConditionalGeneration, BartTokenizer, top_k_top_p_filtering
|
||||
from transformers import top_k_top_p_filtering
|
||||
from transformers.generation_beam_search import BeamSearchScorer
|
||||
from transformers.generation_logits_process import (
|
||||
HammingDiversityLogitsProcessor,
|
||||
LogitsProcessorList,
|
||||
MinLengthLogitsProcessor,
|
||||
NoBadWordsLogitsProcessor,
|
||||
@@ -62,7 +61,7 @@ class GenerationTesterMixin:
|
||||
return config, input_ids, attention_mask, max_length
|
||||
|
||||
@staticmethod
|
||||
def _get_logits_processor_and_kwargs(input_length, eos_token_id, diversity_penalty=None):
|
||||
def _get_logits_processor_and_kwargs(input_length, eos_token_id):
|
||||
process_kwargs = {
|
||||
"min_length": input_length + 1,
|
||||
"bad_words_ids": [[1, 0]],
|
||||
@@ -71,13 +70,6 @@ class GenerationTesterMixin:
|
||||
}
|
||||
logits_processor = LogitsProcessorList(
|
||||
(
|
||||
[
|
||||
HammingDiversityLogitsProcessor(diversity_penalty, num_beams=2, num_beam_groups=2),
|
||||
]
|
||||
if diversity_penalty is not None
|
||||
else []
|
||||
)
|
||||
+ (
|
||||
[
|
||||
MinLengthLogitsProcessor(process_kwargs["min_length"], eos_token_id),
|
||||
]
|
||||
@@ -123,28 +115,6 @@ class GenerationTesterMixin:
|
||||
)
|
||||
return beam_kwargs, beam_scorer
|
||||
|
||||
@staticmethod
|
||||
def _get_diverse_beam_scorer_and_kwargs(batch_size, max_length, num_return_sequences=1):
|
||||
beam_kwargs = {
|
||||
"early_stopping": False,
|
||||
"length_penalty": 2.0,
|
||||
"num_beams": 2,
|
||||
"num_return_sequences": num_return_sequences,
|
||||
"num_beam_groups": 2, # one beam per group
|
||||
"diversity_penalty": 2.0,
|
||||
}
|
||||
beam_scorer = BeamSearchScorer(
|
||||
batch_size=batch_size,
|
||||
max_length=max_length,
|
||||
num_beams=beam_kwargs["num_beams"],
|
||||
device=torch_device,
|
||||
length_penalty=beam_kwargs["length_penalty"],
|
||||
do_early_stopping=beam_kwargs["early_stopping"],
|
||||
num_beam_hyps_to_keep=num_return_sequences,
|
||||
num_beam_groups=beam_kwargs["num_beam_groups"],
|
||||
)
|
||||
return beam_kwargs, beam_scorer
|
||||
|
||||
@staticmethod
|
||||
def _get_encoder_outputs(model, input_ids, attention_mask, num_interleave=1):
|
||||
encoder = model.get_encoder()
|
||||
@@ -438,92 +408,6 @@ class GenerationTesterMixin:
|
||||
|
||||
self.assertIsNotNone(output_ids_generate)
|
||||
|
||||
def test_group_beam_search_generate(self):
|
||||
for model_class in self.all_generative_model_classes:
|
||||
config, input_ids, attention_mask, max_length = self._get_input_ids_and_config()
|
||||
|
||||
logits_process_kwargs, logits_processor = self._get_logits_processor_and_kwargs(
|
||||
input_ids.shape[-1], config.eos_token_id, diversity_penalty=2.0
|
||||
)
|
||||
|
||||
model = model_class(config).to(torch_device)
|
||||
model.eval()
|
||||
|
||||
# check `generate()` and `group_beam_search()` are equal
|
||||
if model.config.is_encoder_decoder:
|
||||
max_length = 4
|
||||
beam_kwargs, beam_scorer = self._get_diverse_beam_scorer_and_kwargs(input_ids.shape[0], max_length)
|
||||
output_ids_generate = model.generate(
|
||||
input_ids,
|
||||
attention_mask=attention_mask,
|
||||
do_sample=False,
|
||||
max_length=max_length,
|
||||
**beam_kwargs,
|
||||
**logits_process_kwargs,
|
||||
)
|
||||
|
||||
# group_beam_search does not automatically interleave `batch_size` dim for `num_beams`
|
||||
kwargs = {}
|
||||
if model.config.is_encoder_decoder:
|
||||
encoder_outputs, input_ids_clone, attention_mask_clone = self._get_encoder_outputs(
|
||||
model, input_ids, attention_mask, num_interleave=beam_scorer.num_beams
|
||||
)
|
||||
kwargs["encoder_outputs"] = encoder_outputs
|
||||
input_ids_clone = input_ids_clone.repeat_interleave(beam_scorer.num_beams, dim=0)
|
||||
else:
|
||||
attention_mask_clone = attention_mask.repeat_interleave(beam_scorer.num_beams, dim=0)
|
||||
input_ids_clone = input_ids.repeat_interleave(beam_scorer.num_beams, dim=0)
|
||||
|
||||
with torch.no_grad():
|
||||
output_ids_group_beam_search = model.group_beam_search(
|
||||
input_ids_clone,
|
||||
beam_scorer,
|
||||
max_length=max_length,
|
||||
attention_mask=attention_mask_clone,
|
||||
logits_processor=logits_processor,
|
||||
**kwargs,
|
||||
)
|
||||
self.assertListEqual(output_ids_generate.tolist(), output_ids_group_beam_search.tolist())
|
||||
|
||||
# check `generate()` and `group_beam_search()` are equal for `num_return_sequences`
|
||||
num_return_sequences = 2
|
||||
if model.config.is_encoder_decoder:
|
||||
max_length = 4
|
||||
beam_kwargs, beam_scorer = self._get_diverse_beam_scorer_and_kwargs(
|
||||
input_ids.shape[0], max_length, num_return_sequences=num_return_sequences
|
||||
)
|
||||
|
||||
output_ids_generate = model.generate(
|
||||
input_ids,
|
||||
attention_mask=attention_mask,
|
||||
do_sample=False,
|
||||
max_length=max_length,
|
||||
**beam_kwargs,
|
||||
**logits_process_kwargs,
|
||||
)
|
||||
# group_beam_search does not automatically interleave `batch_size` dim for `num_beams`
|
||||
kwargs = {}
|
||||
if model.config.is_encoder_decoder:
|
||||
encoder_outputs, input_ids_clone, attention_mask_clone = self._get_encoder_outputs(
|
||||
model, input_ids, attention_mask, num_interleave=beam_scorer.num_beams
|
||||
)
|
||||
kwargs["encoder_outputs"] = encoder_outputs
|
||||
input_ids_clone = input_ids_clone.repeat_interleave(beam_scorer.num_beams, dim=0)
|
||||
else:
|
||||
attention_mask_clone = attention_mask.repeat_interleave(beam_scorer.num_beams, dim=0)
|
||||
input_ids_clone = input_ids.repeat_interleave(beam_scorer.num_beams, dim=0)
|
||||
|
||||
with torch.no_grad():
|
||||
output_ids_beam_search = model.group_beam_search(
|
||||
input_ids_clone,
|
||||
beam_scorer,
|
||||
max_length=max_length,
|
||||
attention_mask=attention_mask_clone,
|
||||
logits_processor=logits_processor,
|
||||
**kwargs,
|
||||
)
|
||||
self.assertListEqual(output_ids_generate.tolist(), output_ids_beam_search.tolist())
|
||||
|
||||
|
||||
@require_torch
|
||||
class UtilsFunctionsTest(unittest.TestCase):
|
||||
@@ -628,31 +512,3 @@ class UtilsFunctionsTest(unittest.TestCase):
|
||||
|
||||
self.assertTrue(torch.allclose(non_inf_expected_output, non_inf_output, atol=1e-12))
|
||||
self.assertTrue(torch.all(torch.eq(non_inf_expected_idx, non_inf_idx)))
|
||||
|
||||
|
||||
@require_torch
|
||||
class GenerationIntegrationTests(unittest.TestCase):
|
||||
@slow
|
||||
def test_diverse_beam_search(self):
|
||||
article = """Justin Timberlake and Jessica Biel, welcome to parenthood.
|
||||
The celebrity couple announced the arrival of their son, Silas Randall Timberlake, in statements to People.
|
||||
"Silas was the middle name of Timberlake's maternal grandfather Bill Bomar, who died in 2012, while Randall is the musician's own middle name, as well as his father's first," People reports.
|
||||
The couple announced the pregnancy in January, with an Instagram post. It is the first baby for both."""
|
||||
|
||||
bart_tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn")
|
||||
bart_model = BartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn").to(torch_device)
|
||||
input_ids = bart_tokenizer(article, return_tensors="pt").input_ids.to(torch_device)
|
||||
|
||||
outputs = bart_model.generate(
|
||||
input_ids, num_beams=4, num_return_sequences=2, num_beam_groups=4, diversity_penalty=2.0
|
||||
)
|
||||
|
||||
generated_text = bart_tokenizer.batch_decode(outputs, skip_special_tokens=True)
|
||||
|
||||
self.assertListEqual(
|
||||
generated_text,
|
||||
[
|
||||
"The couple announced the birth of their son, Silas Randall Timberlake, in a statement. Silas was the middle name of Timberlake's maternal grandfather Bill Bomar. Randall is the musician's own middle name, as well as his father's first. It is the first baby for both of them.",
|
||||
"Justin Timberlake and Jessica Biel have a son. The baby is named Silas Randall Timberlake. It is the first child for both. The couple announced the pregnancy in January. The name Silas is the middle name of Timberlake's maternal grandfather. It's also his own middle name.",
|
||||
],
|
||||
)
|
||||
+81
-169
@@ -13,7 +13,6 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
@@ -50,68 +49,37 @@ if is_torch_available():
|
||||
pipeline,
|
||||
)
|
||||
from transformers.models.bart.modeling_bart import (
|
||||
BartDecoder,
|
||||
BartEncoder,
|
||||
BartSinusoidalPositionalEmbedding,
|
||||
SinusoidalPositionalEmbedding,
|
||||
_prepare_bart_decoder_inputs,
|
||||
invert_mask,
|
||||
shift_tokens_right,
|
||||
)
|
||||
|
||||
|
||||
PGE_ARTICLE = """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."""
|
||||
|
||||
|
||||
def prepare_bart_inputs_dict(
|
||||
config,
|
||||
input_ids,
|
||||
attention_mask=None,
|
||||
):
|
||||
if attention_mask is None:
|
||||
attention_mask = input_ids.ne(config.pad_token_id)
|
||||
return {
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
}
|
||||
|
||||
|
||||
@require_torch
|
||||
class BartModelTester:
|
||||
class ModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
batch_size=13,
|
||||
seq_length=7,
|
||||
is_training=True,
|
||||
use_labels=False,
|
||||
vocab_size=99,
|
||||
hidden_size=16,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=4,
|
||||
intermediate_size=4,
|
||||
hidden_act="gelu",
|
||||
hidden_dropout_prob=0.1,
|
||||
attention_probs_dropout_prob=0.1,
|
||||
max_position_embeddings=20,
|
||||
eos_token_id=2,
|
||||
pad_token_id=1,
|
||||
bos_token_id=0,
|
||||
):
|
||||
self.parent = parent
|
||||
self.batch_size = batch_size
|
||||
self.seq_length = seq_length
|
||||
self.is_training = is_training
|
||||
self.use_labels = use_labels
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.intermediate_size = intermediate_size
|
||||
self.hidden_act = hidden_act
|
||||
self.hidden_dropout_prob = hidden_dropout_prob
|
||||
self.attention_probs_dropout_prob = attention_probs_dropout_prob
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.eos_token_id = eos_token_id
|
||||
self.pad_token_id = pad_token_id
|
||||
self.bos_token_id = bos_token_id
|
||||
self.batch_size = 13
|
||||
self.seq_length = 7
|
||||
self.is_training = True
|
||||
self.use_labels = False
|
||||
self.vocab_size = 99
|
||||
self.hidden_size = 16
|
||||
self.num_hidden_layers = 2
|
||||
self.num_attention_heads = 4
|
||||
self.intermediate_size = 4
|
||||
self.hidden_act = "gelu"
|
||||
self.hidden_dropout_prob = 0.1
|
||||
self.attention_probs_dropout_prob = 0.1
|
||||
self.max_position_embeddings = 20
|
||||
self.eos_token_id = 2
|
||||
self.pad_token_id = 1
|
||||
self.bos_token_id = 0
|
||||
torch.manual_seed(0)
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
@@ -143,67 +111,21 @@ class BartModelTester:
|
||||
config, inputs_dict = self.prepare_config_and_inputs()
|
||||
inputs_dict["decoder_input_ids"] = inputs_dict["input_ids"]
|
||||
inputs_dict["decoder_attention_mask"] = inputs_dict["attention_mask"]
|
||||
inputs_dict["use_cache"] = False
|
||||
return config, inputs_dict
|
||||
|
||||
def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict):
|
||||
model = BartModel(config=config).get_decoder().to(torch_device).eval()
|
||||
input_ids = inputs_dict["input_ids"]
|
||||
|
||||
# first forward pass
|
||||
outputs = model(input_ids, use_cache=True)
|
||||
|
||||
output, past_key_values = outputs.to_tuple()
|
||||
|
||||
# create hypothetical multiple next token and extent to next_input_ids
|
||||
next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size)
|
||||
|
||||
# append to next input_ids and
|
||||
next_input_ids = torch.cat([input_ids, next_tokens], dim=-1)
|
||||
|
||||
output_from_no_past = model(next_input_ids)["last_hidden_state"]
|
||||
output_from_past = model(next_tokens, past_key_values=past_key_values)["last_hidden_state"]
|
||||
|
||||
# select random slice
|
||||
random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item()
|
||||
output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx].detach()
|
||||
output_from_past_slice = output_from_past[:, :, random_slice_idx].detach()
|
||||
|
||||
self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])
|
||||
|
||||
# test that outputs are equal for slice
|
||||
self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-2))
|
||||
|
||||
def check_encoder_decoder_model_standalone(self, config, inputs_dict):
|
||||
model = BartModel(config=config).to(torch_device).eval()
|
||||
outputs = model(**inputs_dict)
|
||||
|
||||
encoder_last_hidden_state = outputs.encoder_last_hidden_state
|
||||
last_hidden_state = outputs.last_hidden_state
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
encoder = model.get_encoder()
|
||||
encoder.save_pretrained(tmpdirname)
|
||||
encoder = BartEncoder.from_pretrained(tmpdirname).to(torch_device)
|
||||
|
||||
encoder_last_hidden_state_2 = encoder(inputs_dict["input_ids"], attention_mask=inputs_dict["attention_mask"])[
|
||||
0
|
||||
]
|
||||
|
||||
self.parent.assertTrue((encoder_last_hidden_state_2 - encoder_last_hidden_state).abs().max().item() < 1e-3)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
decoder = model.get_decoder()
|
||||
decoder.save_pretrained(tmpdirname)
|
||||
decoder = BartDecoder.from_pretrained(tmpdirname).to(torch_device)
|
||||
|
||||
last_hidden_state_2 = decoder(
|
||||
input_ids=inputs_dict["decoder_input_ids"],
|
||||
attention_mask=inputs_dict["decoder_attention_mask"],
|
||||
encoder_hidden_states=encoder_last_hidden_state,
|
||||
encoder_attention_mask=inputs_dict["attention_mask"],
|
||||
)[0]
|
||||
|
||||
self.parent.assertTrue((last_hidden_state_2 - last_hidden_state).abs().max().item() < 1e-3)
|
||||
def prepare_bart_inputs_dict(
|
||||
config,
|
||||
input_ids,
|
||||
attention_mask=None,
|
||||
):
|
||||
if attention_mask is None:
|
||||
attention_mask = input_ids.ne(config.pad_token_id)
|
||||
return {
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
}
|
||||
|
||||
|
||||
@require_torch
|
||||
@@ -220,7 +142,7 @@ class BARTModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
test_missing_keys = False
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = BartModelTester(self)
|
||||
self.model_tester = ModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=BartConfig)
|
||||
|
||||
def test_config(self):
|
||||
@@ -247,25 +169,23 @@ class BARTModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
config, inputs_dict = self.model_tester.prepare_config_and_inputs()
|
||||
config.use_cache = False
|
||||
inputs_dict["input_ids"][:, -2:] = config.pad_token_id
|
||||
|
||||
decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_bart_decoder_inputs(
|
||||
config, inputs_dict["input_ids"]
|
||||
)
|
||||
model = BartModel(config).to(torch_device).eval()
|
||||
|
||||
decoder_features_with_created_mask = model(**inputs_dict)[0]
|
||||
|
||||
decoder_input_ids = shift_tokens_right(inputs_dict["input_ids"], config.pad_token_id)
|
||||
decoder_attention_mask = decoder_input_ids.ne(config.pad_token_id)
|
||||
decoder_attention_mask[:, 0] = decoder_attention_mask[:, 1]
|
||||
|
||||
decoder_features_with_passed_mask = model(
|
||||
decoder_attention_mask=decoder_attention_mask, decoder_input_ids=decoder_input_ids, **inputs_dict
|
||||
decoder_attention_mask=invert_mask(decoder_attn_mask), decoder_input_ids=decoder_input_ids, **inputs_dict
|
||||
)[0]
|
||||
assert_tensors_close(decoder_features_with_passed_mask, decoder_features_with_created_mask)
|
||||
useless_mask = torch.zeros_like(decoder_attention_mask)
|
||||
useless_mask = torch.zeros_like(decoder_attn_mask)
|
||||
decoder_features = model(decoder_attention_mask=useless_mask, **inputs_dict)[0]
|
||||
self.assertTrue(isinstance(decoder_features, torch.Tensor)) # no hidden states or attentions
|
||||
self.assertEqual(
|
||||
decoder_features.size(), (self.model_tester.batch_size, self.model_tester.seq_length, config.d_model)
|
||||
)
|
||||
if decoder_attention_mask.min().item() == 0: # some tokens were masked
|
||||
if decoder_attn_mask.min().item() < -1e3: # some tokens were masked
|
||||
self.assertFalse((decoder_features_with_created_mask == decoder_features).all().item())
|
||||
|
||||
# Test different encoder attention masks
|
||||
@@ -284,43 +204,13 @@ class BARTModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
model2, info = model_class.from_pretrained(tmpdirname, output_loading_info=True)
|
||||
self.assertEqual(info["missing_keys"], [])
|
||||
|
||||
def test_decoder_model_past_with_large_inputs(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_decoder_model_past_large_inputs(*config_and_inputs)
|
||||
|
||||
def test_encoder_decoder_model_standalone(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs_for_common()
|
||||
self.model_tester.check_encoder_decoder_model_standalone(*config_and_inputs)
|
||||
|
||||
# BartForSequenceClassification does not support inputs_embeds
|
||||
@unittest.skip("Passing inputs_embeds not implemented for Bart.")
|
||||
def test_inputs_embeds(self):
|
||||
config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
|
||||
pass
|
||||
|
||||
for model_class in (BartModel, BartForConditionalGeneration, BartForQuestionAnswering):
|
||||
model = model_class(config)
|
||||
model.to(torch_device)
|
||||
model.eval()
|
||||
|
||||
inputs = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class))
|
||||
|
||||
if not self.is_encoder_decoder:
|
||||
input_ids = inputs["input_ids"]
|
||||
del inputs["input_ids"]
|
||||
else:
|
||||
encoder_input_ids = inputs["input_ids"]
|
||||
decoder_input_ids = inputs.get("decoder_input_ids", encoder_input_ids)
|
||||
del inputs["input_ids"]
|
||||
inputs.pop("decoder_input_ids", None)
|
||||
|
||||
wte = model.get_input_embeddings()
|
||||
if not self.is_encoder_decoder:
|
||||
inputs["inputs_embeds"] = wte(input_ids)
|
||||
else:
|
||||
inputs["inputs_embeds"] = wte(encoder_input_ids)
|
||||
inputs["decoder_inputs_embeds"] = wte(decoder_input_ids)
|
||||
|
||||
with torch.no_grad():
|
||||
model(**inputs)[0]
|
||||
@unittest.skip("TODO: Decoder embeddings cannot be resized at the moment")
|
||||
def test_resize_embeddings_untied(self):
|
||||
pass
|
||||
|
||||
@require_sentencepiece
|
||||
@require_tokenizers
|
||||
@@ -496,6 +386,20 @@ class BartHeadTests(unittest.TestCase):
|
||||
model = BartForConditionalGeneration(config).eval().to(torch_device)
|
||||
model(**model.dummy_inputs)
|
||||
|
||||
def test_prepare_bart_decoder_inputs(self):
|
||||
config, *_ = self._get_config_and_data()
|
||||
input_ids = _long_tensor(([4, 4, 2]))
|
||||
decoder_input_ids = _long_tensor([[26388, 2, config.pad_token_id]])
|
||||
ignore = float("-inf")
|
||||
decoder_input_ids, decoder_attn_mask, causal_mask = _prepare_bart_decoder_inputs(
|
||||
config, input_ids, decoder_input_ids
|
||||
)
|
||||
expected_causal_mask = torch.tensor(
|
||||
[[0, ignore, ignore], [0, 0, ignore], [0, 0, 0]] # never attend to the final token, because its pad
|
||||
).to(input_ids.device)
|
||||
self.assertEqual(decoder_attn_mask.size(), decoder_input_ids.size())
|
||||
self.assertTrue(torch.eq(expected_causal_mask, causal_mask).all())
|
||||
|
||||
def test_resize_tokens_embeddings_more(self):
|
||||
config, input_ids, _ = self._get_config_and_data()
|
||||
|
||||
@@ -566,14 +470,14 @@ class BartModelIntegrationTests(unittest.TestCase):
|
||||
self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=TOLERANCE))
|
||||
|
||||
@slow
|
||||
def test_base_mask_filling(self):
|
||||
def test_bart_base_mask_filling(self):
|
||||
pbase = pipeline(task="fill-mask", model="facebook/bart-base")
|
||||
src_text = [" I went to the <mask>."]
|
||||
results = [x["token_str"] for x in pbase(src_text)]
|
||||
assert "Ġbathroom" in results
|
||||
|
||||
@slow
|
||||
def test_large_mask_filling(self):
|
||||
def test_bart_large_mask_filling(self):
|
||||
plarge = pipeline(task="fill-mask", model="facebook/bart-large")
|
||||
src_text = [" I went to the <mask>."]
|
||||
results = [x["token_str"] for x in plarge(src_text)]
|
||||
@@ -704,7 +608,7 @@ class BartModelIntegrationTests(unittest.TestCase):
|
||||
|
||||
|
||||
@require_torch
|
||||
class TestBartSinusoidalPositionalEmbeddings(unittest.TestCase):
|
||||
class TestSinusoidalPositionalEmbeddings(unittest.TestCase):
|
||||
desired_weights = [
|
||||
[0, 0, 0, 0, 0],
|
||||
[0.84147096, 0.82177866, 0.80180490, 0.78165019, 0.76140374],
|
||||
@@ -712,30 +616,38 @@ class TestBartSinusoidalPositionalEmbeddings(unittest.TestCase):
|
||||
]
|
||||
|
||||
def test_positional_emb_cache_logic(self):
|
||||
emb1 = BartSinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6, padding_idx=1).to(torch_device)
|
||||
no_cache = emb1((4, 10), past_key_values_length=0)
|
||||
yes_cache = emb1((4, 10), past_key_values_length=2)
|
||||
|
||||
self.assertTrue(no_cache.shape == yes_cache.shape == (10, 6))
|
||||
self.assertListEqual(no_cache[2:].tolist(), yes_cache[:-2].tolist())
|
||||
pad = 1
|
||||
input_ids = torch.tensor([[4, 10]], dtype=torch.long, device=torch_device)
|
||||
emb1 = SinusoidalPositionalEmbedding(num_positions=32, embedding_dim=6, padding_idx=pad).to(torch_device)
|
||||
no_cache = emb1(input_ids, use_cache=False)
|
||||
yes_cache = emb1(input_ids, use_cache=True)
|
||||
self.assertEqual((1, 1, 6), yes_cache.shape) # extra dim to allow broadcasting, feel free to delete!
|
||||
self.assertListEqual(no_cache[-1].tolist(), yes_cache[0][0].tolist())
|
||||
|
||||
def test_odd_embed_dim(self):
|
||||
# odd embedding_dim is allowed
|
||||
BartSinusoidalPositionalEmbedding(num_positions=4, embedding_dim=5, padding_idx=0).to(torch_device)
|
||||
SinusoidalPositionalEmbedding(num_positions=4, embedding_dim=5, padding_idx=0).to(torch_device)
|
||||
|
||||
# odd num_positions is allowed
|
||||
BartSinusoidalPositionalEmbedding(num_positions=5, embedding_dim=4, padding_idx=0).to(torch_device)
|
||||
SinusoidalPositionalEmbedding(num_positions=5, embedding_dim=4, padding_idx=0).to(torch_device)
|
||||
|
||||
def test_positional_emb_weights_against_marian(self):
|
||||
pad = 1
|
||||
emb1 = BartSinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512, padding_idx=pad).to(
|
||||
torch_device
|
||||
)
|
||||
emb1 = SinusoidalPositionalEmbedding(num_positions=512, embedding_dim=512, padding_idx=pad).to(torch_device)
|
||||
weights = emb1.weight.data[:3, :5].tolist()
|
||||
for i, (expected_weight, actual_weight) in enumerate(zip(self.desired_weights, weights)):
|
||||
for j in range(5):
|
||||
self.assertAlmostEqual(expected_weight[j], actual_weight[j], places=3)
|
||||
|
||||
# test that forward pass is just a lookup, there is no ignore padding logic
|
||||
input_ids = torch.tensor([[4, 10, pad, pad, pad]], dtype=torch.long, device=torch_device)
|
||||
no_cache_pad_zero = emb1(input_ids)
|
||||
self.assertTrue(
|
||||
torch.allclose(
|
||||
torch.tensor(self.desired_weights, device=torch_device), no_cache_pad_zero[:3, :5], atol=1e-3
|
||||
)
|
||||
)
|
||||
|
||||
def test_child_config_equivalence(self):
|
||||
"""Test that configs associated with children of BartForConditionalGeneration are identical."""
|
||||
child_classes = [BlenderbotConfig, MBartConfig, MarianConfig, PegasusConfig]
|
||||
|
||||
@@ -104,6 +104,9 @@ class BlenderbotTesterMixin(ModelTesterMixin, unittest.TestCase):
|
||||
self.model_tester = BlenderbotModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=BlenderbotConfig)
|
||||
|
||||
def test_inputs_embeds(self):
|
||||
pass
|
||||
|
||||
def test_initialization_module(self):
|
||||
config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
|
||||
model = BlenderbotForConditionalGeneration(config).model
|
||||
|
||||
@@ -302,8 +302,6 @@ class ModelTesterMixin:
|
||||
# Question Answering model returns start_logits and end_logits
|
||||
if model_class in MODEL_FOR_QUESTION_ANSWERING_MAPPING.values():
|
||||
correct_outlen += 1 # start_logits and end_logits instead of only 1 output
|
||||
if "past_key_values" in outputs:
|
||||
correct_outlen += 1 # past_key_values have been returned
|
||||
|
||||
self.assertEqual(out_len, correct_outlen)
|
||||
|
||||
@@ -388,7 +386,7 @@ class ModelTesterMixin:
|
||||
|
||||
try:
|
||||
if model.config.is_encoder_decoder:
|
||||
model.config.use_cache = False # FSTM still requires this hack -> FSTM should probably be refactored similar to BART afterward
|
||||
model.config.use_cache = False # TODO: this should be deleted after bug #7474 is solved
|
||||
input_ids = inputs["input_ids"]
|
||||
attention_mask = inputs["attention_mask"]
|
||||
decoder_input_ids = inputs["decoder_input_ids"]
|
||||
@@ -1022,6 +1020,7 @@ class ModelTesterMixin:
|
||||
)
|
||||
|
||||
def test_inputs_embeds(self):
|
||||
|
||||
config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
|
||||
|
||||
for model_class in self.all_model_classes:
|
||||
|
||||
@@ -57,7 +57,7 @@ class FlaxRobertaModelTest(unittest.TestCase):
|
||||
self.assertEqual(len(fx_outputs), len(pt_outputs), "Output lengths differ between Flax and PyTorch")
|
||||
|
||||
for fx_output, pt_output in zip(fx_outputs, pt_outputs.to_tuple()):
|
||||
self.assert_almost_equals(fx_output, pt_output.numpy(), 5e-3)
|
||||
self.assert_almost_equals(fx_output, pt_output.numpy(), 6e-4)
|
||||
|
||||
def test_multiple_sequences(self):
|
||||
tokenizer = RobertaTokenizerFast.from_pretrained("roberta-base")
|
||||
|
||||
@@ -20,7 +20,7 @@ from transformers import is_torch_available
|
||||
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
|
||||
|
||||
from .test_configuration_common import ConfigTester
|
||||
from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
|
||||
from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
@@ -128,6 +128,33 @@ class MobileBertModelTester:
|
||||
|
||||
return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
|
||||
def prepare_config_and_inputs_for_decoder(self):
|
||||
(
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
) = self.prepare_config_and_inputs()
|
||||
|
||||
config.is_decoder = True
|
||||
encoder_hidden_states = floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
|
||||
encoder_attention_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
|
||||
|
||||
return (
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
)
|
||||
|
||||
def create_and_check_mobilebert_model(
|
||||
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
@@ -141,6 +168,39 @@ class MobileBertModelTester:
|
||||
self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
|
||||
self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))
|
||||
|
||||
def create_and_check_mobilebert_model_as_decoder(
|
||||
self,
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
):
|
||||
model = MobileBertModel(config)
|
||||
model.to(torch_device)
|
||||
model.eval()
|
||||
result = model(
|
||||
input_ids,
|
||||
attention_mask=input_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
)
|
||||
result = model(
|
||||
input_ids,
|
||||
attention_mask=input_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
)
|
||||
result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
|
||||
|
||||
self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
|
||||
self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))
|
||||
|
||||
def create_and_check_mobilebert_for_masked_lm(
|
||||
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
@@ -292,6 +352,38 @@ class MobileBertModelTest(ModelTesterMixin, unittest.TestCase):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mobilebert_model(*config_and_inputs)
|
||||
|
||||
def test_mobilebert_model_as_decoder(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
|
||||
self.model_tester.create_and_check_mobilebert_model_as_decoder(*config_and_inputs)
|
||||
|
||||
def test_mobilebert_model_as_decoder_with_default_input_mask(self):
|
||||
# This regression test was failing with PyTorch < 1.3
|
||||
(
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
) = self.model_tester.prepare_config_and_inputs_for_decoder()
|
||||
|
||||
input_mask = None
|
||||
|
||||
self.model_tester.create_and_check_mobilebert_model_as_decoder(
|
||||
config,
|
||||
input_ids,
|
||||
token_type_ids,
|
||||
input_mask,
|
||||
sequence_labels,
|
||||
token_labels,
|
||||
choice_labels,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask,
|
||||
)
|
||||
|
||||
def test_for_masked_lm(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mobilebert_for_masked_lm(*config_and_inputs)
|
||||
|
||||
@@ -1,250 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Inc. team, Microsoft Corporation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import unittest
|
||||
|
||||
from transformers import is_torch_available
|
||||
from transformers.testing_utils import require_torch, slow, torch_device
|
||||
|
||||
from .test_configuration_common import ConfigTester
|
||||
from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
import torch
|
||||
|
||||
from transformers import (
|
||||
MPNetConfig,
|
||||
MPNetForMaskedLM,
|
||||
MPNetForMultipleChoice,
|
||||
MPNetForQuestionAnswering,
|
||||
MPNetForSequenceClassification,
|
||||
MPNetForTokenClassification,
|
||||
MPNetModel,
|
||||
)
|
||||
|
||||
|
||||
class MPNetModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
batch_size=13,
|
||||
seq_length=7,
|
||||
is_training=True,
|
||||
use_input_mask=True,
|
||||
use_token_type_ids=False,
|
||||
use_labels=True,
|
||||
vocab_size=99,
|
||||
hidden_size=64,
|
||||
num_hidden_layers=5,
|
||||
num_attention_heads=4,
|
||||
intermediate_size=64,
|
||||
hidden_act="gelu",
|
||||
hidden_dropout_prob=0.1,
|
||||
attention_probs_dropout_prob=0.1,
|
||||
max_position_embeddings=512,
|
||||
type_vocab_size=16,
|
||||
type_sequence_label_size=2,
|
||||
initializer_range=0.02,
|
||||
num_labels=3,
|
||||
num_choices=4,
|
||||
scope=None,
|
||||
):
|
||||
self.parent = parent
|
||||
self.batch_size = batch_size
|
||||
self.seq_length = seq_length
|
||||
self.is_training = is_training
|
||||
self.use_input_mask = use_input_mask
|
||||
self.use_token_type_ids = use_token_type_ids
|
||||
self.use_labels = use_labels
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.intermediate_size = intermediate_size
|
||||
self.hidden_act = hidden_act
|
||||
self.hidden_dropout_prob = hidden_dropout_prob
|
||||
self.attention_probs_dropout_prob = attention_probs_dropout_prob
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.type_vocab_size = type_vocab_size
|
||||
self.type_sequence_label_size = type_sequence_label_size
|
||||
self.initializer_range = initializer_range
|
||||
self.num_labels = num_labels
|
||||
self.num_choices = num_choices
|
||||
self.scope = scope
|
||||
|
||||
def get_large_model_config(self):
|
||||
return MPNetConfig.from_pretrained("microsoft/mpnet-base")
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
|
||||
|
||||
input_mask = None
|
||||
if self.use_input_mask:
|
||||
input_mask = random_attention_mask([self.batch_size, self.seq_length])
|
||||
|
||||
sequence_labels = None
|
||||
token_labels = None
|
||||
choice_labels = None
|
||||
if self.use_labels:
|
||||
sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
|
||||
token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
|
||||
choice_labels = ids_tensor([self.batch_size], self.num_choices)
|
||||
|
||||
config = MPNetConfig(
|
||||
vocab_size=self.vocab_size,
|
||||
hidden_size=self.hidden_size,
|
||||
num_hidden_layers=self.num_hidden_layers,
|
||||
num_attention_heads=self.num_attention_heads,
|
||||
intermediate_size=self.intermediate_size,
|
||||
hidden_act=self.hidden_act,
|
||||
hidden_dropout_prob=self.hidden_dropout_prob,
|
||||
attention_probs_dropout_prob=self.attention_probs_dropout_prob,
|
||||
max_position_embeddings=self.max_position_embeddings,
|
||||
initializer_range=self.initializer_range,
|
||||
)
|
||||
return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
|
||||
def create_and_check_mpnet_model(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
model = MPNetModel(config=config)
|
||||
model.to(torch_device)
|
||||
model.eval()
|
||||
result = model(input_ids, input_mask)
|
||||
result = model(input_ids)
|
||||
self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
|
||||
self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))
|
||||
|
||||
def create_and_check_mpnet_for_question_answering(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
model = MPNetForQuestionAnswering(config=config)
|
||||
model.to(torch_device)
|
||||
model.eval()
|
||||
result = model(
|
||||
input_ids,
|
||||
attention_mask=input_mask,
|
||||
start_positions=sequence_labels,
|
||||
end_positions=sequence_labels,
|
||||
)
|
||||
self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
|
||||
self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))
|
||||
|
||||
def create_and_check_mpnet_for_sequence_classification(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
config.num_labels = self.num_labels
|
||||
model = MPNetForSequenceClassification(config)
|
||||
model.to(torch_device)
|
||||
model.eval()
|
||||
result = model(input_ids, attention_mask=input_mask, labels=sequence_labels)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))
|
||||
|
||||
def create_and_check_mpnet_for_multiple_choice(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
config.num_choices = self.num_choices
|
||||
model = MPNetForMultipleChoice(config=config)
|
||||
model.to(torch_device)
|
||||
model.eval()
|
||||
multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
|
||||
multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
|
||||
result = model(
|
||||
multiple_choice_inputs_ids,
|
||||
attention_mask=multiple_choice_input_mask,
|
||||
labels=choice_labels,
|
||||
)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))
|
||||
|
||||
def create_and_check_mpnet_for_token_classification(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
config.num_labels = self.num_labels
|
||||
model = MPNetForTokenClassification(config=config)
|
||||
model.to(torch_device)
|
||||
model.eval()
|
||||
result = model(input_ids, attention_mask=input_mask, labels=token_labels)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config_and_inputs = self.prepare_config_and_inputs()
|
||||
(config, input_ids, input_mask, sequence_labels, token_labels, choice_labels) = config_and_inputs
|
||||
inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask}
|
||||
return config, inputs_dict
|
||||
|
||||
|
||||
@require_torch
|
||||
class MPNetModelTest(ModelTesterMixin, unittest.TestCase):
|
||||
|
||||
all_model_classes = (
|
||||
(
|
||||
MPNetForMaskedLM,
|
||||
MPNetForMultipleChoice,
|
||||
MPNetForQuestionAnswering,
|
||||
MPNetForSequenceClassification,
|
||||
MPNetForTokenClassification,
|
||||
MPNetModel,
|
||||
)
|
||||
if is_torch_available()
|
||||
else ()
|
||||
)
|
||||
test_pruning = False
|
||||
test_torchscript = True
|
||||
test_resize_embeddings = True
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = MPNetModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=MPNetConfig, hidden_size=37)
|
||||
|
||||
def test_config(self):
|
||||
self.config_tester.run_common_tests()
|
||||
|
||||
def test_mpnet_model(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_model(*config_and_inputs)
|
||||
|
||||
def test_for_sequence_classification(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_for_sequence_classification(*config_and_inputs)
|
||||
|
||||
def test_for_multiple_choice(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_for_multiple_choice(*config_and_inputs)
|
||||
|
||||
def test_for_token_classification(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_for_token_classification(*config_and_inputs)
|
||||
|
||||
def test_for_question_answering(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_for_question_answering(*config_and_inputs)
|
||||
|
||||
|
||||
@require_torch
|
||||
class MPNetModelIntegrationTest(unittest.TestCase):
|
||||
@slow
|
||||
def test_inference_no_head(self):
|
||||
model = MPNetModel.from_pretrained("microsoft/mpnet-base")
|
||||
input_ids = torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
|
||||
output = model(input_ids)[0]
|
||||
expected_shape = torch.Size((1, 11, 768))
|
||||
self.assertEqual(output.shape, expected_shape)
|
||||
expected_slice = torch.tensor(
|
||||
[[[-0.0550, 0.1943, -0.0740], [-0.0562, 0.2211, -0.0579], [-0.0437, 0.3337, -0.0641]]]
|
||||
)
|
||||
# compare the actual values for a slice.
|
||||
self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4))
|
||||
@@ -37,7 +37,7 @@ from transformers.testing_utils import (
|
||||
torch_device,
|
||||
)
|
||||
|
||||
from .test_modeling_bart import BartModelTester
|
||||
from .test_modeling_bart import ModelTester as BartModelTester
|
||||
from .test_modeling_dpr import DPRModelTester
|
||||
from .test_modeling_t5 import T5ModelTester
|
||||
|
||||
|
||||
+89
-177
@@ -14,50 +14,48 @@
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import copy
|
||||
import unittest
|
||||
|
||||
import copy
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from transformers import (
|
||||
MODEL_FOR_CAUSAL_LM_MAPPING,
|
||||
MODEL_FOR_MASKED_LM_MAPPING,
|
||||
MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
|
||||
MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
|
||||
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
|
||||
MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
|
||||
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
|
||||
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
|
||||
is_torch_available,
|
||||
)
|
||||
from transformers import is_torch_available, MODEL_FOR_MULTIPLE_CHOICE_MAPPING, MODEL_FOR_QUESTION_ANSWERING_MAPPING, \
|
||||
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING, MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING, \
|
||||
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, MODEL_FOR_CAUSAL_LM_MAPPING, MODEL_FOR_MASKED_LM_MAPPING, \
|
||||
MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
|
||||
from transformers.file_utils import cached_property
|
||||
from transformers.testing_utils import require_scatter, require_torch, slow, torch_device
|
||||
from transformers.testing_utils import require_torch, require_scatter, slow, torch_device
|
||||
|
||||
from .test_configuration_common import ConfigTester
|
||||
from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
|
||||
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
import torch
|
||||
|
||||
from transformers import (
|
||||
TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST,
|
||||
TapasTokenizer,
|
||||
TapasConfig,
|
||||
TapasForMaskedLM,
|
||||
TapasForQuestionAnswering,
|
||||
TapasForSequenceClassification,
|
||||
TapasModel,
|
||||
TapasTokenizer,
|
||||
)
|
||||
|
||||
from transformers.models.tapas.modeling_tapas import (
|
||||
IndexMap,
|
||||
ProductIndexMap,
|
||||
flatten,
|
||||
gather,
|
||||
flatten,
|
||||
range_index_map,
|
||||
reduce_max,
|
||||
reduce_sum,
|
||||
reduce_mean,
|
||||
reduce_sum,
|
||||
reduce_max,
|
||||
reduce_min,
|
||||
)
|
||||
|
||||
|
||||
@@ -169,6 +167,7 @@ class TapasModelTester:
|
||||
sequence_labels = None
|
||||
token_labels = None
|
||||
labels = None
|
||||
answer = None
|
||||
numeric_values = None
|
||||
numeric_values_scale = None
|
||||
float_answer = None
|
||||
@@ -303,7 +302,7 @@ class TapasModelTester:
|
||||
token_type_ids=token_type_ids,
|
||||
)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length))
|
||||
|
||||
|
||||
# inference: with aggregation head (WTQ, WikiSQL-supervised). Model returns logits and aggregation logits
|
||||
model = TapasForQuestionAnswering(config=config)
|
||||
model.to(torch_device)
|
||||
@@ -315,7 +314,7 @@ class TapasModelTester:
|
||||
)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length))
|
||||
self.parent.assertEqual(result.logits_aggregation.shape, (self.batch_size, self.num_aggregation_labels))
|
||||
|
||||
|
||||
# training: can happen in 3 main ways
|
||||
# case 1: conversational (SQA)
|
||||
model = TapasForQuestionAnswering(config=sqa_config)
|
||||
@@ -411,7 +410,7 @@ class TapasModelTest(ModelTesterMixin, unittest.TestCase):
|
||||
all_model_classes = (
|
||||
(
|
||||
TapasModel,
|
||||
TapasForMaskedLM,
|
||||
#TapasForMaskedLM,
|
||||
TapasForQuestionAnswering,
|
||||
TapasForSequenceClassification,
|
||||
)
|
||||
@@ -419,7 +418,7 @@ class TapasModelTest(ModelTesterMixin, unittest.TestCase):
|
||||
else None
|
||||
)
|
||||
test_pruning = False
|
||||
test_torchscript = False
|
||||
test_torchscript = True
|
||||
test_resize_embeddings = True
|
||||
test_head_masking = False
|
||||
|
||||
@@ -444,14 +443,10 @@ class TapasModelTest(ModelTesterMixin, unittest.TestCase):
|
||||
self.model_tester.batch_size, dtype=torch.long, device=torch_device
|
||||
)
|
||||
inputs_dict["numeric_values"] = torch.zeros(
|
||||
(self.model_tester.batch_size, self.model_tester.seq_length),
|
||||
dtype=torch.float,
|
||||
device=torch_device,
|
||||
(self.model_tester.batch_size, self.model_tester.seq_length), dtype=torch.float, device=torch_device
|
||||
)
|
||||
inputs_dict["numeric_values_scale"] = torch.zeros(
|
||||
(self.model_tester.batch_size, self.model_tester.seq_length),
|
||||
dtype=torch.float,
|
||||
device=torch_device,
|
||||
(self.model_tester.batch_size, self.model_tester.seq_length), dtype=torch.float, device=torch_device
|
||||
)
|
||||
inputs_dict["float_answer"] = torch.zeros(
|
||||
self.model_tester.batch_size, dtype=torch.float, device=torch_device
|
||||
@@ -500,22 +495,20 @@ class TapasModelTest(ModelTesterMixin, unittest.TestCase):
|
||||
|
||||
def prepare_tapas_single_inputs_for_inference():
|
||||
# Here we prepare a single table-question pair to test TAPAS inference on:
|
||||
data = {
|
||||
"Footballer": ["Lionel Messi", "Cristiano Ronaldo"],
|
||||
"Age": ["33", "35"],
|
||||
data = {'Footballer': ["Lionel Messi", "Cristiano Ronaldo"],
|
||||
'Age': ["33", "35"],
|
||||
}
|
||||
queries = "Which footballer is 33 years old?"
|
||||
table = pd.DataFrame.from_dict(data)
|
||||
|
||||
table = pd.DataFrame.from_dict(data)
|
||||
|
||||
return table, queries
|
||||
|
||||
|
||||
def prepare_tapas_batch_inputs_for_inference():
|
||||
# Here we prepare a batch of 2 table-question pairs to test TAPAS inference on:
|
||||
data = {
|
||||
"Footballer": ["Lionel Messi", "Cristiano Ronaldo"],
|
||||
"Age": ["33", "35"],
|
||||
"Number of goals": ["712", "750"],
|
||||
data = {'Footballer': ["Lionel Messi", "Cristiano Ronaldo"],
|
||||
'Age': ["33", "35"],
|
||||
'Number of goals': ["712", "750"]
|
||||
}
|
||||
queries = ["Which footballer is 33 years old?", "How many goals does Ronaldo have?"]
|
||||
table = pd.DataFrame.from_dict(data)
|
||||
@@ -525,10 +518,9 @@ def prepare_tapas_batch_inputs_for_inference():
|
||||
|
||||
def prepare_tapas_batch_inputs_for_training():
|
||||
# Here we prepare a DIFFERENT batch of 2 table-question pairs to test TAPAS training on:
|
||||
data = {
|
||||
"Footballer": ["Lionel Messi", "Cristiano Ronaldo"],
|
||||
"Age": ["33", "35"],
|
||||
"Number of goals": ["712", "750"],
|
||||
data = {'Footballer': ["Lionel Messi", "Cristiano Ronaldo"],
|
||||
'Age': ["33", "35"],
|
||||
'Number of goals': ["712", "750"]
|
||||
}
|
||||
queries = ["Which footballer is 33 years old?", "What's the total number of goals?"]
|
||||
table = pd.DataFrame.from_dict(data)
|
||||
@@ -549,11 +541,11 @@ class TapasModelIntegrationTest(unittest.TestCase):
|
||||
@cached_property
|
||||
def default_tokenizer(self):
|
||||
return TapasTokenizer.from_pretrained("nielsr/tapas-base-finetuned-wtq")
|
||||
|
||||
|
||||
@slow
|
||||
def test_inference_no_head(self):
|
||||
# ideally we want to test this with the weights of tapas_inter_masklm_base_reset,
|
||||
# but since it's not straightforward to do this with the TF 1 implementation, we test it with
|
||||
# but since it's not straightforward to do this with the TF 1 implementation, we test it with
|
||||
# the weights of the WTQ base model (i.e. tapas_wtq_wikisql_sqa_inter_masklm_base_reset)
|
||||
model = TapasModel.from_pretrained("nielsr/tapas-base-finetuned-wtq").to(torch_device)
|
||||
|
||||
@@ -564,23 +556,19 @@ class TapasModelIntegrationTest(unittest.TestCase):
|
||||
outputs = model(**inputs)
|
||||
# test the sequence output
|
||||
expected_slice = torch.tensor(
|
||||
[
|
||||
[
|
||||
[-0.141581565, -0.599805772, 0.747186482],
|
||||
[-0.143664181, -0.602008104, 0.749218345],
|
||||
[-0.15169853, -0.603363097, 0.741370678],
|
||||
]
|
||||
],
|
||||
device=torch_device,
|
||||
)
|
||||
[[[-0.141581565, -0.599805772, 0.747186482],
|
||||
[-0.143664181, -0.602008104, 0.749218345],
|
||||
[-0.15169853, -0.603363097, 0.741370678]]], device=torch_device)
|
||||
|
||||
self.assertTrue(torch.allclose(outputs.last_hidden_state[:, :3, :3], expected_slice, atol=TOLERANCE))
|
||||
|
||||
|
||||
# test the pooled output
|
||||
expected_slice = torch.tensor([[0.987518311, -0.970520139, -0.994303405]], device=torch_device)
|
||||
expected_slice = torch.tensor(
|
||||
[[0.987518311, -0.970520139, -0.994303405]], device=torch_device)
|
||||
|
||||
self.assertTrue(torch.allclose(outputs.pooler_output[:, :3], expected_slice, atol=TOLERANCE))
|
||||
|
||||
|
||||
|
||||
@unittest.skip(reason="Model not available yet")
|
||||
def test_inference_masked_lm(self):
|
||||
pass
|
||||
@@ -605,34 +593,11 @@ class TapasModelIntegrationTest(unittest.TestCase):
|
||||
expected_shape = torch.Size((1, 21))
|
||||
self.assertEqual(logits.shape, expected_shape)
|
||||
|
||||
expected_tensor = torch.tensor(
|
||||
[
|
||||
[
|
||||
-9997.22461,
|
||||
-9997.22461,
|
||||
-9997.22461,
|
||||
-9997.22461,
|
||||
-9997.22461,
|
||||
-9997.22461,
|
||||
-9997.22461,
|
||||
-9997.22461,
|
||||
-9997.22461,
|
||||
-16.2628059,
|
||||
-10004.082,
|
||||
15.4330549,
|
||||
15.4330549,
|
||||
15.4330549,
|
||||
-9990.42,
|
||||
-16.3270779,
|
||||
-16.3270779,
|
||||
-16.3270779,
|
||||
-16.3270779,
|
||||
-16.3270779,
|
||||
-10004.8506,
|
||||
]
|
||||
],
|
||||
device=torch_device,
|
||||
) # ok
|
||||
expected_tensor = torch.tensor([[-9997.22461, -9997.22461, -9997.22461, -9997.22461, -9997.22461,
|
||||
-9997.22461, -9997.22461, -9997.22461, -9997.22461, -16.2628059,
|
||||
-10004.082, 15.4330549, 15.4330549, 15.4330549, -9990.42,
|
||||
-16.3270779, -16.3270779, -16.3270779, -16.3270779, -16.3270779, -10004.8506]],
|
||||
device=torch_device) # ok
|
||||
|
||||
self.assertTrue(torch.allclose(logits, expected_tensor, atol=TOLERANCE))
|
||||
|
||||
@@ -642,7 +607,7 @@ class TapasModelIntegrationTest(unittest.TestCase):
|
||||
model = TapasForQuestionAnswering.from_pretrained("nielsr/tapas-base-finetuned-wtq").to(torch_device)
|
||||
|
||||
tokenizer = self.default_tokenizer
|
||||
# let's test on a batch
|
||||
# let's test on a batch
|
||||
table, queries = prepare_tapas_batch_inputs_for_inference()
|
||||
inputs = tokenizer(table=table, queries=queries, padding="longest", return_tensors="pt")
|
||||
inputs = {k: v.to(torch_device) for k, v in inputs.items()}
|
||||
@@ -653,57 +618,47 @@ class TapasModelIntegrationTest(unittest.TestCase):
|
||||
expected_shape = torch.Size((2, 28))
|
||||
self.assertEqual(logits.shape, expected_shape)
|
||||
|
||||
expected_slice = torch.tensor(
|
||||
[
|
||||
[-160.375504, -160.375504, -160.375504, -10072.3965, -10070.9414, -10094.9736],
|
||||
[-9861.6123, -9861.6123, -9861.6123, -9861.6123, -9891.01172, 146.600677],
|
||||
],
|
||||
device=torch_device,
|
||||
) # ok (batch size = 2)
|
||||
|
||||
self.assertTrue(torch.allclose(logits[:, -6:], expected_slice, atol=TOLERANCE))
|
||||
expected_slice = torch.tensor([[-160.375504, -160.375504, -160.375504, -10072.3965, -10070.9414, -10094.9736],
|
||||
[-9861.6123, -9861.6123, -9861.6123, -9861.6123, -9891.01172, 146.600677]],
|
||||
device=torch_device) # ok (batch size = 2)
|
||||
|
||||
self.assertTrue(torch.allclose(logits[:,-6:], expected_slice, atol=TOLERANCE))
|
||||
|
||||
# test the aggregation logits
|
||||
logits_aggregation = outputs.logits_aggregation
|
||||
expected_shape = torch.Size((2, 4))
|
||||
self.assertEqual(logits_aggregation.shape, expected_shape)
|
||||
expected_tensor = torch.tensor(
|
||||
[[18.8545208, -9.76614857, -6.3128891, -2.93525243], [-4.05782509, 40.0351, -5.35329962, 23.3978653]],
|
||||
device=torch_device,
|
||||
) # ok (batch size = 2)
|
||||
expected_tensor = torch.tensor([[18.8545208, -9.76614857, -6.3128891, -2.93525243],
|
||||
[-4.05782509, 40.0351, -5.35329962, 23.3978653]],
|
||||
device=torch_device) # ok (batch size = 2)
|
||||
|
||||
self.assertTrue(torch.allclose(logits_aggregation, expected_tensor, atol=TOLERANCE))
|
||||
|
||||
# test the predicted answer coordinates and aggregation indices
|
||||
EXPECTED_PREDICTED_ANSWER_COORDINATES = [[(0, 0)], [(1, 2)]]
|
||||
EXPECTED_PREDICTED_AGGREGATION_INDICES = [0, 1]
|
||||
|
||||
predicted_answer_coordinates, predicted_aggregation_indices = tokenizer.convert_logits_to_predictions(
|
||||
inputs, outputs.logits, outputs.logits_aggregation
|
||||
)
|
||||
|
||||
predicted_answer_coordinates, predicted_aggregation_indices = tokenizer.convert_logits_to_predictions(inputs,
|
||||
outputs.logits,
|
||||
outputs.logits_aggregation)
|
||||
|
||||
self.assertEqual(EXPECTED_PREDICTED_ANSWER_COORDINATES, predicted_answer_coordinates)
|
||||
self.assertEqual(EXPECTED_PREDICTED_AGGREGATION_INDICES, predicted_aggregation_indices)
|
||||
|
||||
|
||||
@slow
|
||||
def test_training_question_answering_head_weak_supervision(self):
|
||||
# note that nielsr/tapas-base-finetuned-wtq should correspond to tapas_wtq_wikisql_sqa_inter_masklm_base_reset
|
||||
model = TapasForQuestionAnswering.from_pretrained("nielsr/tapas-base-finetuned-wtq").to(torch_device)
|
||||
model.to(torch_device)
|
||||
# normally we should put the model in training mode but it's a pain to do this with the TF 1 implementation
|
||||
# normally we should put the model in training mode but it's a pain to do this with the TF 1 implementation
|
||||
|
||||
tokenizer = self.default_tokenizer
|
||||
# let's test on a batch
|
||||
# let's test on a batch
|
||||
table, queries, answer_coordinates, answer_text, float_answer = prepare_tapas_batch_inputs_for_training()
|
||||
inputs = tokenizer(
|
||||
table=table,
|
||||
queries=queries,
|
||||
answer_coordinates=answer_coordinates,
|
||||
answer_text=answer_text,
|
||||
padding="longest",
|
||||
return_tensors="pt",
|
||||
)
|
||||
|
||||
inputs = tokenizer(table=table, queries=queries, answer_coordinates=answer_coordinates,
|
||||
answer_text=answer_text, padding="longest", return_tensors="pt")
|
||||
|
||||
# prepare data (created by the tokenizer) and move to torch_device
|
||||
input_ids = inputs["input_ids"].to(torch_device)
|
||||
attention_mask = inputs["attention_mask"].to(torch_device)
|
||||
@@ -716,56 +671,38 @@ class TapasModelIntegrationTest(unittest.TestCase):
|
||||
float_answer = torch.FloatTensor(float_answer).to(torch_device)
|
||||
|
||||
# forward pass to get loss + logits:
|
||||
outputs = model(
|
||||
input_ids=input_ids,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids,
|
||||
labels=labels,
|
||||
numeric_values=numeric_values,
|
||||
numeric_values_scale=numeric_values_scale,
|
||||
float_answer=float_answer,
|
||||
)
|
||||
outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=labels,
|
||||
numeric_values=numeric_values, numeric_values_scale=numeric_values_scale,
|
||||
float_answer=float_answer)
|
||||
|
||||
# test the loss
|
||||
loss = outputs.loss
|
||||
expected_loss = torch.tensor(3.3527612686157227e-08, device=torch_device) # ok
|
||||
expected_loss = torch.tensor(3.3527612686157227e-08, device=torch_device) # ok
|
||||
self.assertTrue(torch.allclose(loss, expected_loss, atol=TOLERANCE))
|
||||
|
||||
# test the logits on the first example
|
||||
logits = outputs.logits
|
||||
expected_shape = torch.Size((2, 29))
|
||||
self.assertEqual(logits.shape, expected_shape)
|
||||
expected_slice = torch.tensor(
|
||||
[
|
||||
-160.0156,
|
||||
-160.0156,
|
||||
-160.0156,
|
||||
-160.0156,
|
||||
-160.0156,
|
||||
-10072.2266,
|
||||
-10070.8896,
|
||||
-10092.6006,
|
||||
-10092.6006,
|
||||
],
|
||||
device=torch_device,
|
||||
) # ok
|
||||
expected_slice = torch.tensor([-160.0156, -160.0156, -160.0156, -160.0156, -160.0156,
|
||||
-10072.2266, -10070.8896, -10092.6006, -10092.6006],
|
||||
device=torch_device) # ok
|
||||
|
||||
self.assertTrue(torch.allclose(logits[0, -9:], expected_slice, atol=TOLERANCE))
|
||||
self.assertTrue(torch.allclose(logits[0,-9:], expected_slice, atol=TOLERANCE))
|
||||
|
||||
|
||||
# test the aggregation logits on the second example
|
||||
logits_aggregation = outputs.logits_aggregation
|
||||
expected_shape = torch.Size((2, 4))
|
||||
self.assertEqual(logits_aggregation.shape, expected_shape)
|
||||
expected_slice = torch.tensor([-4.0538, 40.0304, -5.3554, 23.3965], device=torch_device) # ok
|
||||
expected_slice = torch.tensor([-4.0538, 40.0304, -5.3554, 23.3965], device=torch_device) # ok
|
||||
|
||||
self.assertTrue(torch.allclose(logits_aggregation[1, -4:], expected_slice, atol=TOLERANCE))
|
||||
self.assertTrue(torch.allclose(logits_aggregation[1,-4:], expected_slice, atol=TOLERANCE))
|
||||
|
||||
@slow
|
||||
def test_inference_question_answering_head_strong_supervision(self):
|
||||
# note that nielsr/tapas-base-finetuned-wikisql-supervised should correspond to tapas_wikisql_sqa_inter_masklm_base_reset
|
||||
model = TapasForQuestionAnswering.from_pretrained("nielsr/tapas-base-finetuned-wikisql-supervised").to(
|
||||
torch_device
|
||||
)
|
||||
model = TapasForQuestionAnswering.from_pretrained("nielsr/tapas-base-finetuned-wikisql-supervised").to(torch_device)
|
||||
|
||||
tokenizer = self.default_tokenizer
|
||||
table, queries = prepare_tapas_single_inputs_for_inference()
|
||||
@@ -776,34 +713,11 @@ class TapasModelIntegrationTest(unittest.TestCase):
|
||||
logits = outputs.logits
|
||||
expected_shape = torch.Size((1, 21))
|
||||
self.assertEqual(logits.shape, expected_shape)
|
||||
expected_tensor = torch.tensor(
|
||||
[
|
||||
[
|
||||
-10011.1084,
|
||||
-10011.1084,
|
||||
-10011.1084,
|
||||
-10011.1084,
|
||||
-10011.1084,
|
||||
-10011.1084,
|
||||
-10011.1084,
|
||||
-10011.1084,
|
||||
-10011.1084,
|
||||
-18.6185989,
|
||||
-10008.7969,
|
||||
17.6355762,
|
||||
17.6355762,
|
||||
17.6355762,
|
||||
-10002.4404,
|
||||
-18.7111301,
|
||||
-18.7111301,
|
||||
-18.7111301,
|
||||
-18.7111301,
|
||||
-18.7111301,
|
||||
-10007.0977,
|
||||
]
|
||||
],
|
||||
device=torch_device,
|
||||
) # ok
|
||||
expected_tensor = torch.tensor([[-10011.1084, -10011.1084, -10011.1084, -10011.1084, -10011.1084,
|
||||
-10011.1084, -10011.1084, -10011.1084, -10011.1084, -18.6185989,
|
||||
-10008.7969, 17.6355762, 17.6355762, 17.6355762, -10002.4404,
|
||||
-18.7111301, -18.7111301, -18.7111301, -18.7111301, -18.7111301, -10007.0977]],
|
||||
device=torch_device) # ok
|
||||
|
||||
self.assertTrue(torch.allclose(logits, expected_tensor, atol=TOLERANCE))
|
||||
|
||||
@@ -811,12 +725,11 @@ class TapasModelIntegrationTest(unittest.TestCase):
|
||||
logits_aggregation = outputs.logits_aggregation
|
||||
expected_shape = torch.Size((1, 4))
|
||||
self.assertEqual(logits_aggregation.shape, expected_shape)
|
||||
expected_tensor = torch.tensor(
|
||||
[[16.5659733, -3.06624889, -2.34152961, -0.970244825]], device=torch_device
|
||||
) # ok, PyTorch model outputs [[16.5679, -3.0668, -2.3442, -0.9674]]
|
||||
expected_tensor = torch.tensor([[16.5659733, -3.06624889, -2.34152961, -0.970244825]],
|
||||
device=torch_device) # ok, PyTorch model outputs [[16.5679, -3.0668, -2.3442, -0.9674]]
|
||||
|
||||
self.assertTrue(torch.allclose(logits_aggregation, expected_tensor, atol=TOLERANCE))
|
||||
|
||||
|
||||
@slow
|
||||
def test_inference_classification_head(self):
|
||||
# note that nielsr/tapas-base-finetuned-tabfact should correspond to tapas_tabfact_inter_masklm_base_reset
|
||||
@@ -832,9 +745,8 @@ class TapasModelIntegrationTest(unittest.TestCase):
|
||||
logits = outputs.logits
|
||||
expected_shape = torch.Size((1, 2))
|
||||
self.assertEqual(logits.shape, expected_shape)
|
||||
expected_tensor = torch.tensor(
|
||||
[[0.795137286, 9.5572]], device=torch_device
|
||||
) # ok. Note that the PyTorch model outputs [[0.8057, 9.5281]]
|
||||
expected_tensor = torch.tensor([[0.795137286, 9.5572]],
|
||||
device=torch_device) # ok. Note that the PyTorch model outputs [[0.8057, 9.5281]]
|
||||
|
||||
self.assertTrue(torch.allclose(outputs.logits, expected_tensor, atol=TOLERANCE))
|
||||
|
||||
@@ -1032,4 +944,4 @@ class TapasUtilitiesTest(unittest.TestCase):
|
||||
result = gather(values, index)
|
||||
|
||||
# We use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual
|
||||
np.testing.assert_array_equal(result.numpy(), [[[1, 2], [3, 4]], [[7, 8], [5, 6]]])
|
||||
np.testing.assert_array_equal(result.numpy(), [[[1, 2], [3, 4]], [[7, 8], [5, 6]]])
|
||||
@@ -344,6 +344,13 @@ class TFModelTesterMixin:
|
||||
tf_hidden_states[pt_nans] = 0
|
||||
|
||||
max_diff = np.amax(np.abs(tf_hidden_states - pt_hidden_states))
|
||||
# Debug info (remove when fixed)
|
||||
if max_diff >= 4e-2:
|
||||
print("===")
|
||||
print(model_class)
|
||||
print(config)
|
||||
print(inputs_dict)
|
||||
print(pt_inputs_dict)
|
||||
self.assertLessEqual(max_diff, 4e-2)
|
||||
|
||||
# Check we can load pt model in tf and vice-versa with checkpoint => model functions
|
||||
|
||||
@@ -1,237 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Inc. team, Microsoft Corporation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import unittest
|
||||
|
||||
from transformers import MPNetConfig, is_tf_available
|
||||
from transformers.testing_utils import require_tf, slow
|
||||
|
||||
from .test_configuration_common import ConfigTester
|
||||
from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
|
||||
|
||||
|
||||
if is_tf_available():
|
||||
import tensorflow as tf
|
||||
|
||||
from transformers.models.mpnet.modeling_tf_mpnet import (
|
||||
TFMPNetForMaskedLM,
|
||||
TFMPNetForMultipleChoice,
|
||||
TFMPNetForQuestionAnswering,
|
||||
TFMPNetForSequenceClassification,
|
||||
TFMPNetForTokenClassification,
|
||||
TFMPNetModel,
|
||||
)
|
||||
|
||||
|
||||
class TFMPNetModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
batch_size=13,
|
||||
seq_length=7,
|
||||
is_training=True,
|
||||
use_input_mask=True,
|
||||
use_token_type_ids=False,
|
||||
use_labels=True,
|
||||
vocab_size=99,
|
||||
hidden_size=64,
|
||||
num_hidden_layers=5,
|
||||
num_attention_heads=4,
|
||||
intermediate_size=64,
|
||||
hidden_act="gelu",
|
||||
hidden_dropout_prob=0.1,
|
||||
attention_probs_dropout_prob=0.1,
|
||||
max_position_embeddings=512,
|
||||
type_vocab_size=16,
|
||||
type_sequence_label_size=2,
|
||||
initializer_range=0.02,
|
||||
num_labels=3,
|
||||
num_choices=4,
|
||||
scope=None,
|
||||
):
|
||||
self.parent = parent
|
||||
self.batch_size = batch_size
|
||||
self.seq_length = seq_length
|
||||
self.is_training = is_training
|
||||
self.use_input_mask = use_input_mask
|
||||
self.use_token_type_ids = use_token_type_ids
|
||||
self.use_labels = use_labels
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.intermediate_size = intermediate_size
|
||||
self.hidden_act = hidden_act
|
||||
self.hidden_dropout_prob = hidden_dropout_prob
|
||||
self.attention_probs_dropout_prob = attention_probs_dropout_prob
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.type_vocab_size = type_vocab_size
|
||||
self.type_sequence_label_size = type_sequence_label_size
|
||||
self.initializer_range = initializer_range
|
||||
self.num_labels = num_labels
|
||||
self.num_choices = num_choices
|
||||
self.scope = scope
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
|
||||
|
||||
input_mask = None
|
||||
if self.use_input_mask:
|
||||
input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
|
||||
|
||||
sequence_labels = None
|
||||
token_labels = None
|
||||
choice_labels = None
|
||||
if self.use_labels:
|
||||
sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
|
||||
token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
|
||||
choice_labels = ids_tensor([self.batch_size], self.num_choices)
|
||||
|
||||
config = MPNetConfig(
|
||||
vocab_size=self.vocab_size,
|
||||
hidden_size=self.hidden_size,
|
||||
num_hidden_layers=self.num_hidden_layers,
|
||||
num_attention_heads=self.num_attention_heads,
|
||||
intermediate_size=self.intermediate_size,
|
||||
hidden_act=self.hidden_act,
|
||||
hidden_dropout_prob=self.hidden_dropout_prob,
|
||||
attention_probs_dropout_prob=self.attention_probs_dropout_prob,
|
||||
max_position_embeddings=self.max_position_embeddings,
|
||||
initializer_range=self.initializer_range,
|
||||
)
|
||||
return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
|
||||
def create_and_check_mpnet_model(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
model = TFMPNetModel(config=config)
|
||||
inputs = {"input_ids": input_ids, "attention_mask": input_mask}
|
||||
result = model(inputs)
|
||||
inputs = [input_ids, input_mask]
|
||||
result = model(inputs)
|
||||
self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
|
||||
|
||||
def create_and_check_mpnet_for_masked_lm(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
model = TFMPNetForMaskedLM(config=config)
|
||||
inputs = {"input_ids": input_ids, "attention_mask": input_mask}
|
||||
result = model(inputs)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
|
||||
|
||||
def create_and_check_mpnet_for_question_answering(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
model = TFMPNetForQuestionAnswering(config=config)
|
||||
inputs = {
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": input_mask,
|
||||
}
|
||||
result = model(inputs)
|
||||
self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
|
||||
self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))
|
||||
|
||||
def create_and_check_mpnet_for_sequence_classification(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
config.num_labels = self.num_labels
|
||||
model = TFMPNetForSequenceClassification(config)
|
||||
inputs = {"input_ids": input_ids, "attention_mask": input_mask}
|
||||
result = model(inputs)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))
|
||||
|
||||
def create_and_check_mpnet_for_multiple_choice(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
config.num_choices = self.num_choices
|
||||
model = TFMPNetForMultipleChoice(config)
|
||||
multiple_choice_inputs_ids = tf.tile(tf.expand_dims(input_ids, 1), (1, self.num_choices, 1))
|
||||
multiple_choice_input_mask = tf.tile(tf.expand_dims(input_mask, 1), (1, self.num_choices, 1))
|
||||
inputs = {
|
||||
"input_ids": multiple_choice_inputs_ids,
|
||||
"attention_mask": multiple_choice_input_mask,
|
||||
}
|
||||
result = model(inputs)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))
|
||||
|
||||
def create_and_check_mpnet_for_token_classification(
|
||||
self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
|
||||
):
|
||||
config.num_labels = self.num_labels
|
||||
model = TFMPNetForTokenClassification(config)
|
||||
inputs = {"input_ids": input_ids, "attention_mask": input_mask}
|
||||
result = model(inputs)
|
||||
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config_and_inputs = self.prepare_config_and_inputs()
|
||||
(config, input_ids, input_mask, sequence_labels, token_labels, choice_labels) = config_and_inputs
|
||||
inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask}
|
||||
return config, inputs_dict
|
||||
|
||||
|
||||
@require_tf
|
||||
class TFMPNetModelTest(TFModelTesterMixin, unittest.TestCase):
|
||||
|
||||
all_model_classes = (
|
||||
(
|
||||
TFMPNetForMaskedLM,
|
||||
TFMPNetForMultipleChoice,
|
||||
TFMPNetForQuestionAnswering,
|
||||
TFMPNetForSequenceClassification,
|
||||
TFMPNetForTokenClassification,
|
||||
TFMPNetModel,
|
||||
)
|
||||
if is_tf_available()
|
||||
else ()
|
||||
)
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = TFMPNetModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=MPNetConfig, hidden_size=37)
|
||||
|
||||
def test_config(self):
|
||||
self.config_tester.run_common_tests()
|
||||
|
||||
def test_mpnet_model(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_model(*config_and_inputs)
|
||||
|
||||
def test_for_masked_lm(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_for_masked_lm(*config_and_inputs)
|
||||
|
||||
def test_for_question_answering(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_for_question_answering(*config_and_inputs)
|
||||
|
||||
def test_for_sequence_classification(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_for_sequence_classification(*config_and_inputs)
|
||||
|
||||
def test_for_multiple_choice(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_for_multiple_choice(*config_and_inputs)
|
||||
|
||||
def test_for_token_classification(self):
|
||||
config_and_inputs = self.model_tester.prepare_config_and_inputs()
|
||||
self.model_tester.create_and_check_mpnet_for_token_classification(*config_and_inputs)
|
||||
|
||||
@slow
|
||||
def test_model_from_pretrained(self):
|
||||
for model_name in ["microsoft/mpnet-base"]:
|
||||
model = TFMPNetModel.from_pretrained(model_name)
|
||||
self.assertIsNotNone(model)
|
||||
@@ -1,79 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Inc. team, Microsoft Corporation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from transformers.models.mpnet.tokenization_mpnet import VOCAB_FILES_NAMES, MPNetTokenizer
|
||||
from transformers.testing_utils import require_tokenizers, slow
|
||||
|
||||
from .test_tokenization_common import TokenizerTesterMixin
|
||||
|
||||
|
||||
@require_tokenizers
|
||||
class MPNetTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
|
||||
tokenizer_class = MPNetTokenizer
|
||||
test_rust_tokenizer = False
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
vocab_tokens = [
|
||||
"[UNK]",
|
||||
"[CLS]",
|
||||
"[SEP]",
|
||||
"[PAD]",
|
||||
"[MASK]",
|
||||
"want",
|
||||
"##want",
|
||||
"##ed",
|
||||
"wa",
|
||||
"un",
|
||||
"runn",
|
||||
"##ing",
|
||||
",",
|
||||
"low",
|
||||
"lowest",
|
||||
]
|
||||
self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
|
||||
with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
|
||||
vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
|
||||
|
||||
def get_input_output_texts(self, tokenizer):
|
||||
input_text = "UNwant\u00E9d,running"
|
||||
output_text = "unwanted, running"
|
||||
return input_text, output_text
|
||||
|
||||
def test_full_tokenizer(self):
|
||||
tokenizer = self.tokenizer_class(self.vocab_file)
|
||||
|
||||
tokens = tokenizer.tokenize("UNwant\u00E9d,running")
|
||||
self.assertListEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"])
|
||||
self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [9, 6, 7, 12, 10, 11])
|
||||
|
||||
@slow
|
||||
def test_sequence_builders(self):
|
||||
tokenizer = self.tokenizer_class.from_pretrained("microsoft/mpnet-base")
|
||||
|
||||
text = tokenizer.encode("sequence builders", add_special_tokens=False)
|
||||
text_2 = tokenizer.encode("multi-sequence build", add_special_tokens=False)
|
||||
|
||||
encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)
|
||||
encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)
|
||||
|
||||
assert encoded_sentence == [0] + text + [2]
|
||||
assert encoded_pair == [0] + text + [2] + [2] + text_2 + [2]
|
||||
@@ -17,12 +17,13 @@ import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from typing import List
|
||||
|
||||
from typing import List, Tuple
|
||||
import numpy as np
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from transformers import AddedToken
|
||||
from transformers.testing_utils import require_tokenizers, slow, require_torch, is_pt_tf_cross_test
|
||||
from transformers.models.tapas.tokenization_tapas import (
|
||||
VOCAB_FILES_NAMES,
|
||||
BasicTokenizer,
|
||||
@@ -32,13 +33,11 @@ from transformers.models.tapas.tokenization_tapas import (
|
||||
_is_punctuation,
|
||||
_is_whitespace,
|
||||
)
|
||||
from transformers.testing_utils import is_pt_tf_cross_test, require_pandas, require_tokenizers, require_torch, slow
|
||||
|
||||
from .test_tokenization_common import TokenizerTesterMixin, filter_non_english, merge_model_tokenizer_mappings
|
||||
|
||||
|
||||
@require_tokenizers
|
||||
@require_pandas
|
||||
class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
tokenizer_class = TapasTokenizer
|
||||
test_rust_tokenizer = False
|
||||
@@ -46,9 +45,9 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
from_pretrained_filter = filter_non_english
|
||||
|
||||
def get_table(
|
||||
self,
|
||||
tokenizer: TapasTokenizer,
|
||||
length=5,
|
||||
self,
|
||||
tokenizer: TapasTokenizer,
|
||||
length=5,
|
||||
):
|
||||
toks = [tokenizer.decode([i], clean_up_tokenization_spaces=False) for i in range(len(tokenizer))]
|
||||
|
||||
@@ -62,9 +61,9 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
return table
|
||||
|
||||
def get_table_and_query(
|
||||
self,
|
||||
tokenizer: TapasTokenizer,
|
||||
length=5,
|
||||
self,
|
||||
tokenizer: TapasTokenizer,
|
||||
length=5,
|
||||
):
|
||||
toks = [tokenizer.decode([i], clean_up_tokenization_spaces=False) for i in range(len(tokenizer))]
|
||||
table = self.get_table(tokenizer, length=length - 3)
|
||||
@@ -73,14 +72,14 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
return table, query
|
||||
|
||||
def get_clean_sequence(
|
||||
self,
|
||||
tokenizer: TapasTokenizer,
|
||||
with_prefix_space=False,
|
||||
max_length=20,
|
||||
min_length=5,
|
||||
empty_table: bool = False,
|
||||
add_special_tokens: bool = True,
|
||||
return_table_and_query: bool = False,
|
||||
self,
|
||||
tokenizer: TapasTokenizer,
|
||||
with_prefix_space=False,
|
||||
max_length=20,
|
||||
min_length=5,
|
||||
empty_table: bool = False,
|
||||
add_special_tokens: bool = True,
|
||||
return_table_and_query: bool = False,
|
||||
):
|
||||
|
||||
toks = [tokenizer.decode([i], clean_up_tokenization_spaces=False) for i in range(len(tokenizer))]
|
||||
@@ -133,6 +132,13 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
output_text = "unwanted, running"
|
||||
return input_text, output_text
|
||||
|
||||
def test_full_tokenizer(self):
|
||||
tokenizer = self.tokenizer_class(self.vocab_file)
|
||||
|
||||
tokens = tokenizer.tokenize("UNwant\u00E9d,running")
|
||||
self.assertListEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"])
|
||||
self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [9, 6, 7, 12, 10, 11])
|
||||
|
||||
def test_rust_and_python_full_tokenizers(self):
|
||||
if not self.test_rust_tokenizer:
|
||||
return
|
||||
@@ -284,9 +290,7 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
|
||||
self.assertListEqual(
|
||||
[tokenizer.tokenize(t) for t in ["Test", "\xad", "test"]], [["[UNK]"], ["[EMPTY]"], ["[UNK]"]]
|
||||
)
|
||||
self.assertListEqual([tokenizer.tokenize(t) for t in ["Test", "\xad", "test"]], [["[UNK]"], ["[EMPTY]"], ["[UNK]"]])
|
||||
|
||||
@slow
|
||||
def test_sequence_builders(self):
|
||||
@@ -536,7 +540,8 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
right_padded_token_type_ids = right_padded_sequence["token_type_ids"]
|
||||
|
||||
assert (
|
||||
token_type_ids + [[token_type_padding_idx] * 7] * padding_size == right_padded_token_type_ids
|
||||
token_type_ids + [
|
||||
[token_type_padding_idx] * 7] * padding_size == right_padded_token_type_ids
|
||||
)
|
||||
assert [[token_type_padding_idx] * 7] * padding_size + token_type_ids == left_padded_token_type_ids
|
||||
|
||||
@@ -574,8 +579,8 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
table, query = self.get_table_and_query(tokenizer)
|
||||
|
||||
if (
|
||||
tokenizer.build_inputs_with_special_tokens.__qualname__.split(".")[0] != "PreTrainedTokenizer"
|
||||
and "token_type_ids" in tokenizer.model_input_names
|
||||
tokenizer.build_inputs_with_special_tokens.__qualname__.split(".")[0] != "PreTrainedTokenizer"
|
||||
and "token_type_ids" in tokenizer.model_input_names
|
||||
):
|
||||
information = tokenizer.encode_plus(table, query, add_special_tokens=True)
|
||||
sequences, mask = information["input_ids"], information["token_type_ids"]
|
||||
@@ -641,6 +646,40 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
assert sequence_length == padded_sequence_right_length
|
||||
assert encoded_sequence == padded_sequence_right
|
||||
|
||||
def test_padding_to_multiple_of(self):
|
||||
tokenizers = self.get_tokenizers()
|
||||
for tokenizer in tokenizers:
|
||||
with self.subTest(f"{tokenizer.__class__.__name__}"):
|
||||
if tokenizer.pad_token is None:
|
||||
self.skipTest("No padding token.")
|
||||
else:
|
||||
empty_tokens = tokenizer("", padding=True, pad_to_multiple_of=8)
|
||||
normal_tokens = tokenizer("This is a sample input", padding=True, pad_to_multiple_of=8)
|
||||
for key, value in empty_tokens.items():
|
||||
self.assertEqual(len(value) % 8, 0, "BatchEncoding.{} is not multiple of 8".format(key))
|
||||
for key, value in normal_tokens.items():
|
||||
self.assertEqual(len(value) % 8, 0, "BatchEncoding.{} is not multiple of 8".format(key))
|
||||
|
||||
normal_tokens = tokenizer("This", pad_to_multiple_of=8)
|
||||
for key, value in normal_tokens.items():
|
||||
self.assertNotEqual(len(value) % 8, 0, "BatchEncoding.{} is not multiple of 8".format(key))
|
||||
|
||||
# Should also work with truncation
|
||||
normal_tokens = tokenizer("This", padding=True, truncation=True, pad_to_multiple_of=8)
|
||||
for key, value in normal_tokens.items():
|
||||
self.assertEqual(len(value) % 8, 0, "BatchEncoding.{} is not multiple of 8".format(key))
|
||||
|
||||
# truncation to something which is not a multiple of pad_to_multiple_of raises an error
|
||||
self.assertRaises(
|
||||
ValueError,
|
||||
tokenizer.__call__,
|
||||
"This",
|
||||
padding=True,
|
||||
truncation=True,
|
||||
max_length=12,
|
||||
pad_to_multiple_of=8,
|
||||
)
|
||||
|
||||
def test_call(self):
|
||||
# Tests that all call wrap to encode_plus and batch_encode_plus
|
||||
tokenizers = self.get_tokenizers(do_lower_case=False)
|
||||
@@ -1018,9 +1057,7 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
if self.test_rust_tokenizer:
|
||||
fast_tokenizer = self.get_rust_tokenizer()
|
||||
encoded_sequence_fast = fast_tokenizer.encode_plus(table, sequence, return_tensors="np")
|
||||
batch_encoded_sequence_fast = fast_tokenizer.batch_encode_plus(
|
||||
table, [sequence, sequence], return_tensors="np"
|
||||
)
|
||||
batch_encoded_sequence_fast = fast_tokenizer.batch_encode_plus(table, [sequence, sequence], return_tensors="np")
|
||||
|
||||
# TODO: add forward through JAX/Flax when PR is merged
|
||||
# This is currently here to make flake8 happy !
|
||||
@@ -1102,21 +1139,11 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
tokenizer.encode(table=table, query=queries[0], max_length=i, truncation="drop_rows_to_fit")
|
||||
|
||||
for i in range(12, 512):
|
||||
new_encoded_inputs = tokenizer.encode(
|
||||
table=table, query=queries[0], max_length=i, truncation="drop_rows_to_fit"
|
||||
)
|
||||
new_encoded_inputs = tokenizer.encode(table=table, query=queries[0], max_length=i, truncation="drop_rows_to_fit")
|
||||
|
||||
# Ensure that the input IDs are less than the max length defined.
|
||||
self.assertLessEqual(len(new_encoded_inputs), i)
|
||||
|
||||
tokenizer.model_max_length = 20
|
||||
new_encoded_inputs = tokenizer.encode(table=table, query=queries[0], truncation=True)
|
||||
dropped_encoded_inputs = tokenizer.encode(table=table, query=queries[0], truncation="drop_rows_to_fit")
|
||||
|
||||
# Ensure that the input IDs are still truncated when no max_length is specified
|
||||
self.assertListEqual(new_encoded_inputs, dropped_encoded_inputs)
|
||||
self.assertLessEqual(len(new_encoded_inputs), 20)
|
||||
|
||||
@is_pt_tf_cross_test
|
||||
def test_batch_encode_plus_tensors(self):
|
||||
tokenizers = self.get_tokenizers(do_lower_case=False)
|
||||
@@ -1153,9 +1180,7 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
)
|
||||
else:
|
||||
pytorch_tensor = tokenizer.batch_encode_plus(table, sequences, padding=True, return_tensors="pt")
|
||||
tensorflow_tensor = tokenizer.batch_encode_plus(
|
||||
table, sequences, padding="longest", return_tensors="tf"
|
||||
)
|
||||
tensorflow_tensor = tokenizer.batch_encode_plus(table, sequences, padding="longest", return_tensors="tf")
|
||||
encoded_sequences = tokenizer.batch_encode_plus(table, sequences, padding=True)
|
||||
|
||||
for key in encoded_sequences.keys():
|
||||
@@ -3419,7 +3444,3 @@ class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
self.assertListEqual(segment_ids.tolist(), expected_results["segment_ids"])
|
||||
self.assertListEqual(column_ids.tolist(), expected_results["column_ids"])
|
||||
self.assertListEqual(row_ids.tolist(), expected_results["row_ids"])
|
||||
|
||||
@unittest.skip("Skip this test while all models are still to be uploaded.")
|
||||
def test_pretrained_model_lists(self):
|
||||
pass
|
||||
+1
-7
@@ -28,8 +28,6 @@ PATH_TO_DOC = "docs/source/model_doc"
|
||||
# Update this list for models that are not tested with a comment explaining the reason it should not be.
|
||||
# Being in this list is an exception and should **not** be the rule.
|
||||
IGNORE_NON_TESTED = [
|
||||
"BartDecoder", # Building part of bigger (tested) model.
|
||||
"BartEncoder", # Building part of bigger (tested) model.
|
||||
"BertLMHeadModel", # Needs to be setup as decoder.
|
||||
"DPREncoder", # Building part of bigger (tested) model.
|
||||
"DPRSpanPredictor", # Building part of bigger (tested) model.
|
||||
@@ -60,11 +58,9 @@ TEST_FILES_WITH_NO_COMMON_TESTS = [
|
||||
# Update this list for models that are not documented with a comment explaining the reason it should not be.
|
||||
# Being in this list is an exception and should **not** be the rule.
|
||||
IGNORE_NON_DOCUMENTED = [
|
||||
"BartDecoder", # Building part of bigger (documented) model.
|
||||
"BartEncoder", # Building part of bigger (documented) model.
|
||||
"DPREncoder", # Building part of bigger (documented) model.
|
||||
"DPRSpanPredictor", # Building part of bigger (documented) model.
|
||||
"T5Stack", # Building part of bigger (documented) model.
|
||||
"T5Stack", # Building part of bigger (tested) model.
|
||||
"TFDPREncoder", # Building part of bigger (documented) model.
|
||||
"TFDPRSpanPredictor", # Building part of bigger (documented) model.
|
||||
]
|
||||
@@ -82,8 +78,6 @@ MODEL_NAME_TO_DOC_FILE = {
|
||||
# Update this list for models that are not in any of the auto MODEL_XXX_MAPPING. Being in this list is an exception and
|
||||
# should **not** be the rule.
|
||||
IGNORE_NON_AUTO_CONFIGURED = [
|
||||
"BartDecoder",
|
||||
"BartEncoder",
|
||||
"DPRContextEncoder",
|
||||
"DPREncoder",
|
||||
"DPRReader",
|
||||
|
||||
Reference in new issue
Block a user