diff --git a/.circleci/config.yml b/.circleci/config.yml index 01e6d82b..bfa3b943 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,9 +1,11 @@ version: 2 jobs: - build_py3_torch_and_tf: + run_tests_py3_torch_and_tf: working_directory: ~/transformers docker: - image: circleci/python:3.5 + environment: + OMP_NUM_THREADS: 1 resource_class: xlarge parallelism: 1 steps: @@ -11,65 +13,67 @@ jobs: - run: sudo pip install torch - run: sudo pip install tensorflow - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./transformers/tests/ --cov + - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov - run: codecov - build_py3_torch: + run_tests_py3_torch: working_directory: ~/transformers docker: - image: circleci/python:3.5 + environment: + OMP_NUM_THREADS: 1 resource_class: xlarge parallelism: 1 steps: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./transformers/tests/ --cov - - run: python -m pytest -sv ./examples/ + - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov - run: codecov - build_py3_tf: + run_tests_py3_tf: working_directory: ~/transformers docker: - image: circleci/python:3.5 + environment: + OMP_NUM_THREADS: 1 resource_class: xlarge parallelism: 1 steps: - checkout - run: sudo pip install tensorflow - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./transformers/tests/ --cov + - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov - run: codecov - build_py2_torch: + run_tests_py3_custom_tokenizers: working_directory: ~/transformers - resource_class: large - parallelism: 1 docker: - - image: circleci/python:2.7 + - image: circleci/python:3.5 + steps: + - checkout + - run: sudo pip install --progress-bar off . + - run: sudo pip install pytest pytest-xdist + - run: sudo pip install mecab-python3 + - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py + run_examples_py3_torch: + working_directory: ~/transformers + docker: + - image: circleci/python:3.5 + environment: + OMP_NUM_THREADS: 1 + resource_class: xlarge + parallelism: 1 steps: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov - - run: python -m pytest -sv ./transformers/tests/ --cov - - run: codecov - build_py2_tf: - working_directory: ~/transformers - resource_class: large - parallelism: 1 - docker: - - image: circleci/python:2.7 - steps: - - checkout - - run: sudo pip install tensorflow - - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov - - run: python -m pytest -sv ./transformers/tests/ --cov - - run: codecov + - run: sudo pip install pytest pytest-xdist + - run: sudo pip install tensorboardX scikit-learn + - run: python -m pytest -n 8 --dist=loadfile -s -v ./examples/ deploy_doc: working_directory: ~/transformers docker: @@ -82,6 +86,16 @@ jobs: - run: sudo pip install --progress-bar off -r docs/requirements.txt - run: sudo pip install --progress-bar off -r requirements.txt - run: ./.circleci/deploy.sh + check_repository_consistency: + working_directory: ~/transformers + docker: + - image: circleci/python:3.5 + resource_class: small + parallelism: 1 + steps: + - checkout + - run: sudo pip install requests + - run: python ./utils/link_tester.py workflow_filters: &workflow_filters filters: branches: @@ -91,9 +105,10 @@ workflows: version: 2 build_and_test: jobs: - - build_py3_torch_and_tf - - build_py3_torch - - build_py3_tf - - build_py2_torch - - build_py2_tf + - check_repository_consistency + - run_examples_py3_torch + - run_tests_py3_custom_tokenizers + - run_tests_py3_torch_and_tf + - run_tests_py3_torch + - run_tests_py3_tf - deploy_doc: *workflow_filters diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 98151963..a32581ba 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,3 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 +deploy_doc "3616209" v2.2.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 136ef8df..7d7f2c73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,7 @@ Follow these steps to start contributing: ```bash $ git clone git@github.com:/transformers.git $ cd transformers - $ git remote add upstream git@github.com:huggingface/transformers.git + $ git remote add upstream https://github.com/huggingface/transformers.git ``` 3. Create a new branch to hold your development changes: @@ -168,7 +168,7 @@ Follow these steps to start contributing: to be merged; 4. Make sure pre-existing tests still pass; 5. Add high-coverage tests. No quality test, no merge; -6. All public methods must have informative doctrings; +6. All public methods must have informative docstrings; ### Style guide diff --git a/README.md b/README.md index 40d6acb7..416adcc1 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,12 @@ Choose the right framework for every part of a model's lifetime | [Online demo](#online-demo) | Experimenting with this repo’s text generation capabilities | | [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | | [Quick tour: TF 2.0 and PyTorch ](#Quick-tour-TF-20-training-and-PyTorch-interoperability) | Train a TF 2.0 model in 10 lines of code, load it in PyTorch | +| [Quick tour: pipelines](#quick-tour-of-pipelines) | Using Pipelines: Wrapper around tokenizer and models to use finetuned models | | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | +| [Quick tour: Share your models ](#Quick-tour-of-model-sharing) | Upload and share your fine-tuned models with the community | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation](https://huggingface.co/transformers/) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) | Full API documentation and more | +| [Documentation][(v2.3.0)](https://huggingface.co/transformers/v2.3.0)[(v2.2.0/v2.2.1/v2.2.2)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | ## Installation @@ -86,21 +88,41 @@ When TensorFlow 2.0 and/or PyTorch has been installed, you can install from sour pip install [--editable] . ``` +### Run the examples + +Examples are included in the repository but are not shipped with the library. +Therefore, in order to run the latest versions of the examples you also need to install from source. To do so, create a new virtual environment and follow these steps: + +```bash +git clone https://github.com/huggingface/transformers +cd transformers +pip install [--editable] . +``` + ### Tests A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). -These tests can be run using `pytest` (install pytest if needed with `pip install pytest`). +These tests can be run using `unittest` or `pytest` (install pytest if needed with `pip install pytest`). Depending on which framework is installed (TensorFlow 2.0 and/or PyTorch), the irrelevant tests will be skipped. Ensure that both frameworks are installed if you want to execute all tests. You can run the tests from the root of the cloned repository with the commands: +```bash +python -m unittest discover -s transformers/tests -p "*test.py" -t . +python -m unittest discover -s examples -p "*test.py" -t examples +``` + +or + ```bash python -m pytest -sv ./transformers/tests/ python -m pytest -sv ./examples/ ``` +By default, slow tests are skipped. Set the `RUN_SLOW` environment variable to `yes` to run them. + ### Do you want to run a Transformer model on a mobile device? You should check out our [`swift-coreml-transformers`](https://github.com/huggingface/swift-coreml-transformers) repo. @@ -120,9 +142,13 @@ At some point in the future, you'll be able to seamlessly move from pre-training 5. **[XLNet](https://github.com/zihangdai/xlnet/)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. 6. **[XLM](https://github.com/facebookresearch/XLM/)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. -8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). +8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/master/examples/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT. 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. -10. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. +10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. +11. **[ALBERT](https://github.com/google-research/ALBERT)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. +12. **[T5](https://github.com/google-research/text-to-text-transfer-transformer)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu. +13. **[XLM-RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/xlmr)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. +14. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). @@ -144,7 +170,7 @@ import torch from transformers import * # Transformers has a unified API -# for 8 transformer architectures and 30 pretrained weights. +# for 10 transformer architectures and 30 pretrained weights. # Model | Tokenizer | Pretrained weights shortcut MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'), (OpenAIGPTModel, OpenAIGPTTokenizer, 'openai-gpt'), @@ -154,7 +180,9 @@ MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'), (XLNetModel, XLNetTokenizer, 'xlnet-base-cased'), (XLMModel, XLMTokenizer, 'xlm-mlm-enfr-1024'), (DistilBertModel, DistilBertTokenizer, 'distilbert-base-uncased'), - (RobertaModel, RobertaTokenizer, 'roberta-base')] + (RobertaModel, RobertaTokenizer, 'roberta-base'), + (XLMRobertaModel, XLMRobertaTokenizer, 'xlm-roberta-base'), + ] # To use TensorFlow 2.0 versions of the models, simply prefix the class names with 'TF', e.g. `TFRobertaModel` is the TF 2.0 counterpart of the PyTorch model `RobertaModel` @@ -252,6 +280,11 @@ print("sentence_2 is", "a paraphrase" if pred_2 else "not a paraphrase", "of sen ## Quick tour of the fine-tuning/usage scripts +**Important** +Before running the fine-tuning scripts, please read the +[instructions](#run-the-examples) on how to +setup your environment to run the examples. + The library comprises several example scripts with SOTA performances for NLU and NLG tasks: - `run_glue.py`: an example fine-tuning Bert, XLNet and XLM on nine different GLUE tasks (*sequence-level classification*) @@ -418,6 +451,76 @@ python ./examples/run_generation.py \ --repetition_penalty=1.2 \ ``` +## Quick tour of model sharing + +New in `v2.2.2`: you can now upload and share your fine-tuned models with the community, using the CLI that's built-in to the library. + +**First, create an account on [https://huggingface.co/join](https://huggingface.co/join)**. Then: + +```shell +transformers-cli login +# log in using the same credentials as on huggingface.co +``` +Upload your model: +```shell +transformers-cli upload ./path/to/pretrained_model/ + +# ^^ Upload folder containing weights/tokenizer/config +# saved via `.save_pretrained()` + +transformers-cli upload ./config.json [--filename folder/foobar.json] + +# ^^ Upload a single file +# (you can optionally override its filename, which can be nested inside a folder) +``` + +Your model will then be accessible through its identifier, a concatenation of your username and the folder name above: +```python +"username/model_name" +``` + +Anyone can load it from code: +```python +tokenizer = AutoTokenizer.from_pretrained("username/pretrained_model") +model = AutoModel.from_pretrained("username/pretrained_model") +``` + +Finally, list all your files on S3: +```shell +transformers-cli ls +# List all your S3 objects. +``` + +## Quick tour of pipelines + +New in version `v2.3`: `Pipeline` are high-level objects which automatically handle tokenization, running your data through a transformers model +and outputting the result in a structured object. + +You can create `Pipeline` objects for the following down-stream tasks: + + - `feature-extraction`: Generates a tensor representation for the input sequence + - `ner`: Generates named entity mapping for each word in the input sequence. + - `sentiment-analysis`: Gives the polarity (positive / negative) of the whole input sequence. + - `question-answering`: Provided some context and a question refering to the context, it will extract the answer to the question + in the context. + +```python +from transformers import pipeline + +# Allocate a pipeline for sentiment-analysis +nlp = pipeline('sentiment-analysis') +nlp('We are very happy to include pipeline into the transformers repository.') +>>> {'label': 'POSITIVE', 'score': 0.99893874} + +# Allocate a pipeline for question-answering +nlp = pipeline('question-answering') +nlp({ + 'question': 'What is the name of the repository ?', + 'context': 'Pipeline have been included in the huggingface/transformers repository' +}) +>>> {'score': 0.28756016668193496, 'start': 35, 'end': 59, 'answer': 'huggingface/transformers'} +``` + ## Migrating from pytorch-transformers to transformers Here is a quick summary of what you should take care of when migrating from `pytorch-transformers` to `transformers`. diff --git a/deploy_multi_version_doc.sh b/deploy_multi_version_doc.sh new file mode 100644 index 00000000..bd567213 --- /dev/null +++ b/deploy_multi_version_doc.sh @@ -0,0 +1,22 @@ +cd docs + +function deploy_doc(){ + echo "Creating doc at commit $1 and pushing to folder $2" + git checkout $1 + if [ ! -z "$2" ] + then + echo "Pushing version" $2 + make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2 + else + echo "Pushing master" + make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir + fi +} + +deploy_doc "master" +deploy_doc "b33a385" v1.0.0 +deploy_doc "fe02e45" v1.1.0 +deploy_doc "89fd345" v1.2.0 +deploy_doc "fc9faa8" v2.0.0 +deploy_doc "3ddce1d" v2.1.1 +deploy_doc "f2f3294" v2.2.0 \ No newline at end of file diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js index 2c7836fd..ec804b37 100644 --- a/docs/source/_static/js/custom.js +++ b/docs/source/_static/js/custom.js @@ -1,5 +1,5 @@ function addIcon() { - const huggingFaceLogo = "https://huggingface.co/assets/transformers-docs/huggingface_logo.svg"; + const huggingFaceLogo = "https://huggingface.co/landing/assets/transformers-docs/huggingface_logo.svg"; const image = document.createElement("img"); image.setAttribute("src", huggingFaceLogo); @@ -24,10 +24,10 @@ function addCustomFooter() { social.classList.add("footer__Social"); const imageDetails = [ - { link: "https://huggingface.co", imageLink: "https://huggingface.co/assets/transformers-docs/website.svg" }, - { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/twitter.svg" }, - { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/github.svg" }, - { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/assets/transformers-docs/linkedin.svg" } + { link: "https://huggingface.co", imageLink: "https://huggingface.co/landing/assets/transformers-docs/website.svg" }, + { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/twitter.svg" }, + { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/github.svg" }, + { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/landing/assets/transformers-docs/linkedin.svg" } ]; imageDetails.forEach(imageLinks => { diff --git a/docs/source/conf.py b/docs/source/conf.py index 00c020ab..41a65eec 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ author = u'huggingface' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'2.1.1' +release = u'2.3.0' # -- General configuration --------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 4cd1f48b..0ac9c740 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -47,6 +47,10 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train 6. `XLM `_ (from Facebook) released together with the paper `Cross-lingual Language Model Pretraining `_ by Guillaume Lample and Alexis Conneau. 7. `RoBERTa `_ (from Facebook), released together with the paper a `Robustly Optimized BERT Pretraining Approach `_ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. `DistilBERT `_ (from HuggingFace) released together with the paper `DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into `DistilGPT2 `_. +9. `CTRL `_ (from Salesforce), released together with the paper `CTRL: A Conditional Transformer Language Model for Controllable Generation `_ by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. +10. `CamemBERT `_ (from FAIR, Inria, Sorbonne Université) released together with the paper `CamemBERT: a Tasty French Language Model `_ by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suarez, Yoann Dupont, Laurent Romary, Eric Villemonte de la Clergerie, Djame Seddah, and Benoît Sagot. +11. `ALBERT `_ (from Google Research), released together with the paper a `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations `_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. +12. `XLM-RoBERTa `_ (from Facebook AI), released together with the paper `Unsupervised Cross-lingual Representation Learning at Scale `_ by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. .. toctree:: :maxdepth: 2 @@ -55,6 +59,7 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train installation quickstart pretrained_models + model_sharing examples notebooks serialization @@ -89,3 +94,5 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train model_doc/roberta model_doc/distilbert model_doc/ctrl + model_doc/camembert + model_doc/albert diff --git a/docs/source/installation.md b/docs/source/installation.md index 11beb1ab..6263f760 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -24,15 +24,24 @@ pip install [--editable] . An extensive test suite is included to test the library behavior and several examples. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). -Tests can be run using `pytest` (install pytest if needed with `pip install pytest`). +Tests can be run using `unittest` or `pytest` (install pytest if needed with `pip install pytest`). Run all the tests from the root of the cloned repository with the commands: +```bash +python -m unittest discover -s transformers/tests -p "*test.py" -t . +python -m unittest discover -s examples -p "*test.py" -t examples +``` + +or + ``` bash python -m pytest -sv ./transformers/tests/ python -m pytest -sv ./examples/ ``` +By default, slow tests are skipped. Set the `RUN_SLOW` environment variable to `yes` to run them. + ## OpenAI GPT original tokenization workflow If you want to reproduce the original tokenization process of the `OpenAI GPT` paper, you will need to install `ftfy` (use version 4.4.3 if you are using Python 2) and `SpaCy`: diff --git a/docs/source/main_classes/optimizer_schedules.rst b/docs/source/main_classes/optimizer_schedules.rst index b30a2e0e..22ed1b28 100644 --- a/docs/source/main_classes/optimizer_schedules.rst +++ b/docs/source/main_classes/optimizer_schedules.rst @@ -5,6 +5,7 @@ The ``.optimization`` module provides: - an optimizer with weight decay fixed that can be used to fine-tuned models, and - several schedules in the form of schedule objects that inherit from ``_LRSchedule``: +- a gradient accumulation class to accumulate the gradients of multiple batches ``AdamW`` ~~~~~~~~~~~~~~~~ @@ -12,6 +13,15 @@ The ``.optimization`` module provides: .. autoclass:: transformers.AdamW :members: +``AdamWeightDecay`` +~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AdamWeightDecay + :members: + +.. autofunction:: transformers.create_optimizer + :members: + Schedules ---------------------------------------------------- @@ -49,3 +59,17 @@ Learning Rate Schedules .. image:: /imgs/warmup_linear_schedule.png :target: /imgs/warmup_linear_schedule.png :alt: + +``Warmup`` +~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.Warmup + :members: + +Gradient Strategies +---------------------------------------------------- + +``GradientAccumulator`` +~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.GradientAccumulator diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index a85c1269..46839ce6 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -54,5 +54,100 @@ Additionally, the following method can be used to load values from a data file Example usage ^^^^^^^^^^^^^^^^^^^^^^^^^ +An example using these processors is given in the `run_glue.py `__ script. + + +XNLI +~~~~~~~~~~~~~~~~~~~~~ + +`The Cross-Lingual NLI Corpus (XNLI) `__ is a benchmark that evaluates +the quality of cross-lingual text representations. +XNLI is crowd-sourced dataset based on `MultiNLI `: pairs of text are labeled with textual entailment +annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). + +It was released together with the paper +`XNLI: Evaluating Cross-lingual Sentence Representations `__ + +This library hosts the processor to load the XNLI data: + - :class:`~transformers.data.processors.utils.XnliProcessor` + +Please note that since the gold labels are available on the test set, evaluation is performed on the test set. + An example using these processors is given in the -`run_glue.py `__ script. \ No newline at end of file +`run_xnli.py `__ script. + + +SQuAD +~~~~~~~~~~~~~~~~~~~~~ + +`The Stanford Question Answering Dataset (SQuAD) `__ is a benchmark that evaluates +the performance of models on question answering. Two versions are available, v1.1 and v2.0. The first version (v1.1) was released together with the paper +`SQuAD: 100,000+ Questions for Machine Comprehension of Text `__. The second version (v2.0) was released alongside +the paper `Know What You Don't Know: Unanswerable Questions for SQuAD `__. + +This library hosts a processor for each of the two versions: + +Processors +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Those processors are: + - :class:`~transformers.data.processors.utils.SquadV1Processor` + - :class:`~transformers.data.processors.utils.SquadV2Processor` + +They both inherit from the abstract class :class:`~transformers.data.processors.utils.SquadProcessor` + +.. autoclass:: transformers.data.processors.squad.SquadProcessor + :members: + +Additionally, the following method can be used to convert SQuAD examples into :class:`~transformers.data.processors.utils.SquadFeatures` +that can be used as model inputs. + +.. automethod:: transformers.data.processors.squad.squad_convert_examples_to_features + +These processors as well as the aforementionned method can be used with files containing the data as well as with the `tensorflow_datasets` package. +Examples are given below. + + +Example usage +^^^^^^^^^^^^^^^^^^^^^^^^^ +Here is an example using the processors as well as the conversion method using data files: + +Example:: + + # Loading a V2 processor + processor = SquadV2Processor() + examples = processor.get_dev_examples(squad_v2_data_dir) + + # Loading a V1 processor + processor = SquadV1Processor() + examples = processor.get_dev_examples(squad_v1_data_dir) + + features = squad_convert_examples_to_features( + examples=examples, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + doc_stride=args.doc_stride, + max_query_length=max_query_length, + is_training=not evaluate, + ) + +Using `tensorflow_datasets` is as easy as using a data file: + +Example:: + + # tensorflow_datasets only handle Squad V1. + tfds_examples = tfds.load("squad") + examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate) + + features = squad_convert_examples_to_features( + examples=examples, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + doc_stride=args.doc_stride, + max_query_length=max_query_length, + is_training=not evaluate, + ) + + +Another example using these processors is given in the +`run_squad.py `__ script. diff --git a/docs/source/migration.md b/docs/source/migration.md index d04b66d5..f50d1dff 100644 --- a/docs/source/migration.md +++ b/docs/source/migration.md @@ -104,6 +104,6 @@ for batch in train_data: loss = model(batch) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm) # Gradient clipping is not in AdamW anymore (so you can use amp without issue) - scheduler.step() optimizer.step() + scheduler.step() ``` diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst new file mode 100644 index 00000000..92970c93 --- /dev/null +++ b/docs/source/model_doc/albert.rst @@ -0,0 +1,64 @@ +ALBERT +---------------------------------------------------- + +``AlbrtConfig`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertConfig + :members: + + +``AlbertTokenizer`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertTokenizer + :members: + + +``AlbertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertModel + :members: + + +``AlbertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForMaskedLM + :members: + + +``AlbertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForSequenceClassification + :members: + + +``AlbertForQuestionAnswering`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForQuestionAnswering + :members: + + +``TFAlbertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertModel + :members: + + +``TFAlbertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertForMaskedLM + :members: + + +``TFAlbertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertForSequenceClassification + :members: diff --git a/docs/source/model_doc/camembert.rst b/docs/source/model_doc/camembert.rst new file mode 100644 index 00000000..82ca9de9 --- /dev/null +++ b/docs/source/model_doc/camembert.rst @@ -0,0 +1,50 @@ +CamemBERT +---------------------------------------------------- + +``CamembertConfig`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertConfig + :members: + + +``CamembertTokenizer`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertTokenizer + :members: + + +``CamembertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertModel + :members: + + +``CamembertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForMaskedLM + :members: + + +``CamembertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForSequenceClassification + :members: + + +``CamembertForMultipleChoice`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForMultipleChoice + :members: + + +``CamembertForTokenClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForTokenClassification + :members: diff --git a/docs/source/model_sharing.md b/docs/source/model_sharing.md new file mode 100644 index 00000000..95baafb5 --- /dev/null +++ b/docs/source/model_sharing.md @@ -0,0 +1,40 @@ +# Model upload and sharing + +Starting with `v2.2.2`, you can now upload and share your fine-tuned models with the community, using the CLI that's built-in to the library. + +**First, create an account on [https://huggingface.co/join](https://huggingface.co/join)**. Then: + +```shell +transformers-cli login +# log in using the same credentials as on huggingface.co +``` +Upload your model: +```shell +transformers-cli upload ./path/to/pretrained_model/ + +# ^^ Upload folder containing weights/tokenizer/config +# saved via `.save_pretrained()` + +transformers-cli upload ./config.json [--filename folder/foobar.json] + +# ^^ Upload a single file +# (you can optionally override its filename, which can be nested inside a folder) +``` + +Your model will then be accessible through its identifier, a concatenation of your username and the folder name above: +```python +"username/pretrained_model" +``` + +Anyone can load it from code: +```python +tokenizer = AutoTokenizer.from_pretrained("username/pretrained_model") +model = AutoModel.from_pretrained("username/pretrained_model") +``` + +Finally, list all your files on S3: +```shell +transformers-cli ls +# List all your S3 objects. +``` + diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index edb47e7f..eb7b41ff 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -3,6 +3,7 @@ Pretrained models Here is the full list of the currently provided pretrained models together with a short presentation of each model. +For a list that includes community-uploaded models, refer to `https://huggingface.co/models `__. +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | Architecture | Shortcut name | Details of the model | @@ -61,6 +62,32 @@ Here is the full list of the currently provided pretrained models together with | | ``bert-base-german-dbmdz-uncased`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | Trained on uncased German text by DBMDZ | | | | (see `details on dbmdz repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-japanese`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on Japanese text. Text is tokenized with MeCab and WordPiece. | +| | | | `MeCab `__ is required for tokenization. | +| | | (see `details on cl-tohoku repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-japanese-whole-word-masking`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on Japanese text using Whole-Word-Masking. Text is tokenized with MeCab and WordPiece. | +| | | | `MeCab `__ is required for tokenization. | +| | | (see `details on cl-tohoku repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-japanese-char`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on Japanese text. Text is tokenized into characters. | +| | | (see `details on cl-tohoku repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-japanese-char-whole-word-masking`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on Japanese text using Whole-Word-Masking. Text is tokenized into characters. | +| | | (see `details on cl-tohoku repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-finnish-cased-v1`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on cased Finnish text. | +| | | (see `details on turkunlp.org `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-finnish-uncased-v1`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on uncased Finnish text. | +| | | (see `details on turkunlp.org `__). | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | GPT | ``openai-gpt`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | OpenAI GPT English model | @@ -128,6 +155,10 @@ Here is the full list of the currently provided pretrained models together with | | | | ``roberta-large`` fine-tuned on `MNLI `__. | | | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``distilroberta-base`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | +| | | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint. | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``roberta-base-openai-detector`` | | 12-layer, 768-hidden, 12-heads, 125M parameters | | | | | ``roberta-base`` fine-tuned by OpenAI on the outputs of the 1.5B-parameter GPT-2 model. | | | | (see `details `__) | @@ -148,12 +179,74 @@ Here is the full list of the currently provided pretrained models together with | | | | The DistilGPT2 model distilled from the GPT2 model `gpt2` checkpoint. | | | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -| | ``distilroberta-base`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | -| | | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint. | +| | ``distilbert-base-german-cased`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | +| | | | The German DistilBERT model distilled from the German DBMDZ BERT model `bert-base-german-dbmdz-cased` checkpoint. | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``distilbert-base-multilingual-cased`` | | 6-layer, 768-hidden, 12-heads, 134M parameters | +| | | | The multilingual DistilBERT model distilled from the Multilingual BERT model `bert-base-multilingual-cased` checkpoint. | | | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | CTRL | ``ctrl`` | | 48-layer, 1280-hidden, 16-heads, 1.6B parameters | | | | | Salesforce's Large-sized CTRL English model | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| CamemBERT | ``camembert-base`` | | 12-layer, 768-hidden, 12-heads, 110M parameters | +| | | | CamemBERT using the BERT-base architecture | +| | | (see `details `__) | ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| ALBERT | ``albert-base-v1`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | +| | | | ALBERT base model | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-large-v1`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | +| | | | ALBERT large model | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xlarge-v1`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | +| | | | ALBERT xlarge model | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xxlarge-v1`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | +| | | | ALBERT xxlarge model | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-base-v2`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | +| | | | ALBERT base model with no dropout, additional training data and longer training | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-large-v2`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | +| | | | ALBERT large model with no dropout, additional training data and longer training | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xlarge-v2`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | +| | | | ALBERT xlarge model with no dropout, additional training data and longer training | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | +| | | | ALBERT xxlarge model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| T5 | ``t5-small`` | | ~60M parameters with 6-layers, 512-hidden-state, 2048 feed-forward hidden-state, 8-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``t5-base`` | | ~220M parameters with 12-layers, 768-hidden-state, 3072 feed-forward hidden-state, 12-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``t5-large`` | | ~770M parameters with 24-layers, 1024-hidden-state, 4096 feed-forward hidden-state, 16-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``t5-3B`` | | ~2.8B parameters with 24-layers, 1024-hidden-state, 16384 feed-forward hidden-state, 32-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``t5-11B`` | | ~11B parameters with 24-layers, 1024-hidden-state, 65536 feed-forward hidden-state, 128-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| XLM-RoBERTa | ``xlm-roberta-base`` | | ~125M parameters with 12-layers, 768-hidden-state, 3072 feed-forward hidden-state, 8-heads, | +| | | | Trained on on 2.5 TB of newly created clean CommonCrawl data in 100 languages | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``xlm-roberta-large`` | | ~355M parameters with 24-layers, 1027-hidden-state, 4096 feed-forward hidden-state, 16-heads, | +| | | | Trained on 2.5 TB of newly created clean CommonCrawl data in 100 languages | ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -.. `__ \ No newline at end of file + +.. `__ diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index 530aff8e..60e2cf3f 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -219,4 +219,97 @@ sequence = tokenizer.decode(generated) print(sequence) ``` -The model only requires a single token as input as all the previous tokens' key/value pairs are contained in the `past`. \ No newline at end of file +The model only requires a single token as input as all the previous tokens' key/value pairs are contained in the `past`. + +### Model2Model example + +Encoder-decoder architectures require two tokenized inputs: one for the encoder and the other one for the decoder. Let's assume that we want to use `Model2Model` for generative question answering, and start by tokenizing the question and answer that will be fed to the model. + +```python +import torch +from transformers import BertTokenizer, Model2Model + +# OPTIONAL: if you want to have more information on what's happening under the hood, activate the logger as follows +import logging +logging.basicConfig(level=logging.INFO) + +# Load pre-trained model tokenizer (vocabulary) +tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + +# Encode the input to the encoder (the question) +question = "Who was Jim Henson?" +encoded_question = tokenizer.encode(question) + +# Encode the input to the decoder (the answer) +answer = "Jim Henson was a puppeteer" +encoded_answer = tokenizer.encode(answer) + +# Convert inputs to PyTorch tensors +question_tensor = torch.tensor([encoded_question]) +answer_tensor = torch.tensor([encoded_answer]) +``` + +Let's see how we can use `Model2Model` to get the value of the loss associated with this (question, answer) pair: + +```python +# In order to compute the loss we need to provide language model +# labels (the token ids that the model should have produced) to +# the decoder. +lm_labels = encoded_answer +labels_tensor = torch.tensor([lm_labels]) + +# Load pre-trained model (weights) +model = Model2Model.from_pretrained('bert-base-uncased') + +# Set the model in evaluation mode to deactivate the DropOut modules +# This is IMPORTANT to have reproducible results during evaluation! +model.eval() + +# If you have a GPU, put everything on cuda +question_tensor = question_tensor.to('cuda') +answer_tensor = answer_tensor.to('cuda') +labels_tensor = labels_tensor.to('cuda') +model.to('cuda') + +# Predict hidden states features for each layer +with torch.no_grad(): + # See the models docstrings for the detail of the inputs + outputs = model(question_tensor, answer_tensor, decoder_lm_labels=labels_tensor) + # Transformers models always output tuples. + # See the models docstrings for the detail of all the outputs + # In our case, the first element is the value of the LM loss + lm_loss = outputs[0] +``` + +This loss can be used to fine-tune `Model2Model` on the question answering task. Assuming that we fine-tuned the model, let us now see how to generate an answer: + +```python +# Let's re-use the previous question +question = "Who was Jim Henson?" +encoded_question = tokenizer.encode(question) +question_tensor = torch.tensor([encoded_question]) + +# This time we try to generate the answer, so we start with an empty sequence +answer = "[CLS]" +encoded_answer = tokenizer.encode(answer, add_special_tokens=False) +answer_tensor = torch.tensor([encoded_answer]) + +# Load pre-trained model (weights) +model = Model2Model.from_pretrained('fine-tuned-weights') +model.eval() + +# If you have a GPU, put everything on cuda +question_tensor = encoded_question.to('cuda') +answer_tensor = encoded_answer.to('cuda') +model.to('cuda') + +# Predict all tokens +with torch.no_grad(): + outputs = model(question_tensor, answer_tensor) + predictions = outputs[0] + +# confirm we were able to predict 'jim' +predicted_index = torch.argmax(predictions[0, -1]).item() +predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0] +assert predicted_token == 'jim' +``` diff --git a/examples/README.md b/examples/README.md index 638a9fb0..fb619d58 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,6 +3,17 @@ In this section a few examples are put together. All of these examples work for several models, making use of the very similar API between the different models. +**Important** +To run the latest versions of the examples, you have to install from source and install some specific requirements for the examples. +Execute the following steps in a new virtual environment: + +```bash +git clone https://github.com/huggingface/transformers +cd transformers +pip install [--editable] . +pip install -r ./examples/requirements.txt +``` + | Section | Description | |----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| | [TensorFlow 2.0 models on GLUE](#TensorFlow-2.0-Bert-models-on-GLUE) | Examples running BERT TensorFlow 2.0 model on the GLUE tasks. @@ -12,7 +23,7 @@ similar API between the different models. | [SQuAD](#squad) | Using BERT/RoBERTa/XLNet/XLM for question answering, examples with distributed training. | | [Multiple Choice](#multiple-choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. | [Named Entity Recognition](#named-entity-recognition) | Using BERT for Named Entity Recognition (NER) on the CoNLL 2003 dataset, examples with distributed training. | -| [Abstractive summarization](#abstractive-summarization) | Fine-tuning the library models for abstractive summarization tasks on the CNN/Daily Mail dataset. | +| [XNLI](#xnli) | Examples running BERT/XLM on the XNLI benchmark. | ## TensorFlow 2.0 Bert models on GLUE @@ -506,7 +517,8 @@ Larger batch size may improve the performance while costing more memory. ## Named Entity Recognition -Based on the script [`run_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/run_ner.py). +Based on the scripts [`run_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/run_ner.py) for Pytorch and +[`run_tf_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/run_tf_ner.py) for Tensorflow 2. This example fine-tune Bert Multilingual on GermEval 2014 (German NER). Details and results for the fine-tuning provided by @stefan-it. @@ -551,7 +563,7 @@ The GermEval 2014 dataset has much more labels than CoNLL-2002/2003 datasets, so cat train.txt dev.txt test.txt | cut -d " " -f 2 | grep -v "^$"| sort | uniq > labels.txt ``` -### Training +### Prepare the run Additional environment variables must be set: @@ -563,6 +575,8 @@ export SAVE_STEPS=750 export SEED=1 ``` +### Run the Pytorch version + To start training, just run: ```bash @@ -583,7 +597,7 @@ python3 run_ner.py --data_dir ./ \ If your GPU supports half-precision training, just add the `--fp16` flag. After training, the model will be both evaluated on development and test datasets. -### Evaluation +#### Evaluation Evaluation on development dataset outputs the following for our example: @@ -605,7 +619,7 @@ On the test dataset the following results could be achieved: 10/04/2019 00:42:42 - INFO - __main__ - recall = 0.8624150210424085 ``` -### Comparing BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) +#### Comparing BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) Here is a small comparison between BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) with the same hyperparameters as specified in the [example documentation](https://huggingface.co/transformers/examples.html#named-entity-recognition) (one run): @@ -615,30 +629,108 @@ Here is a small comparison between BERT (large, cased), RoBERTa (large, cased) a | `roberta-large` | 95.96 | 91.87 | `distilbert-base-uncased` | 94.34 | 90.32 -## Abstractive summarization +### Run the Tensorflow 2 version -Based on the script -[`run_summarization_finetuning.py`](https://github.com/huggingface/transformers/blob/master/examples/run_summarization_finetuning.py). - -Before running this script you should download **both** CNN and Daily Mail -datasets from [Kyunghyun Cho's website](https://cs.nyu.edu/~kcho/DMQA/) (the -links next to "Stories") in the same folder. Then uncompress the archives by running: +To start training, just run: ```bash -tar -xvf cnn_stories.tgz && tar -xvf dailymail_stories.tgz +python3 run_tf_ner.py --data_dir ./ \ +--model_type bert \ +--labels ./labels.txt \ +--model_name_or_path $BERT_MODEL \ +--output_dir $OUTPUT_DIR \ +--max_seq_length $MAX_LENGTH \ +--num_train_epochs $NUM_EPOCHS \ +--per_device_train_batch_size $BATCH_SIZE \ +--save_steps $SAVE_STEPS \ +--seed $SEED \ +--do_train \ +--do_eval \ +--do_predict ``` -note that the finetuning script **will not work** if you do not download both -datasets. We will refer as `$DATA_PATH` the path to where you uncompressed both -archive. +Such as the Pytorch version, if your GPU supports half-precision training, just add the `--fp16` flag. After training, the model will be both evaluated on development and test datasets. + +#### Evaluation + +Evaluation on development dataset outputs the following for our example: +```bash + precision recall f1-score support + + LOCderiv 0.7619 0.6154 0.6809 52 + PERpart 0.8724 0.8997 0.8858 4057 + OTHpart 0.9360 0.9466 0.9413 711 + ORGpart 0.7015 0.6989 0.7002 269 + LOCpart 0.7668 0.8488 0.8057 496 + LOC 0.8745 0.9191 0.8963 235 + ORGderiv 0.7723 0.8571 0.8125 91 + OTHderiv 0.4800 0.6667 0.5581 18 + OTH 0.5789 0.6875 0.6286 16 + PERderiv 0.5385 0.3889 0.4516 18 + PER 0.5000 0.5000 0.5000 2 + ORG 0.0000 0.0000 0.0000 3 + +micro avg 0.8574 0.8862 0.8715 5968 +macro avg 0.8575 0.8862 0.8713 5968 +``` + +On the test dataset the following results could be achieved: +```bash + precision recall f1-score support + + PERpart 0.8847 0.8944 0.8896 9397 + OTHpart 0.9376 0.9353 0.9365 1639 + ORGpart 0.7307 0.7044 0.7173 697 + LOC 0.9133 0.9394 0.9262 561 + LOCpart 0.8058 0.8157 0.8107 1150 + ORG 0.0000 0.0000 0.0000 8 + OTHderiv 0.5882 0.4762 0.5263 42 + PERderiv 0.6571 0.5227 0.5823 44 + OTH 0.4906 0.6667 0.5652 39 + ORGderiv 0.7016 0.7791 0.7383 172 + LOCderiv 0.8256 0.6514 0.7282 109 + PER 0.0000 0.0000 0.0000 11 + +micro avg 0.8722 0.8774 0.8748 13869 +macro avg 0.8712 0.8774 0.8740 13869 +``` + +## XNLI + +Based on the script [`run_xnli.py`](https://github.com/huggingface/transformers/blob/master/examples/run_xnli.py). + +[XNLI](https://www.nyu.edu/projects/bowman/xnli/) is crowd-sourced dataset based on [MultiNLI](http://www.nyu.edu/projects/bowman/multinli/). It is an evaluation benchmark for cross-lingual text representations. Pairs of text are labeled with textual entailment annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). + +#### Fine-tuning on XNLI + +This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in 106 mins +on a single tesla V100 16GB. The data for XNLI can be downloaded with the following links and should be both saved (and un-zipped) in a +`$XNLI_DIR` directory. + +* [XNLI 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-1.0.zip) +* [XNLI-MT 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-MT-1.0.zip) ```bash -export DATA_PATH=/path/to/dataset/ +export XNLI_DIR=/path/to/XNLI -python run_summarization_finetuning.py \ - --output_dir=output \ - --model_type=bert2bert \ - --model_name_or_path=bert2bert \ - --do_train \ - --data_path=$DATA_PATH \ +python run_xnli.py \ + --model_type bert \ + --model_name_or_path bert-base-multilingual-cased \ + --language de \ + --train_language en \ + --do_train \ + --do_eval \ + --data_dir $XNLI_DIR \ + --per_gpu_train_batch_size 32 \ + --learning_rate 5e-5 \ + --num_train_epochs 2.0 \ + --max_seq_length 128 \ + --output_dir /tmp/debug_xnli/ \ + --save_steps -1 +``` + +Training with the previously defined hyper-parameters yields the following results on the **test** set: + +```bash +acc = 0.7093812375249501 ``` diff --git a/examples/contrib/run_camembert.py b/examples/contrib/run_camembert.py new file mode 100644 index 00000000..28144d51 --- /dev/null +++ b/examples/contrib/run_camembert.py @@ -0,0 +1,48 @@ +from pathlib import Path +import tarfile +import urllib.request + +import torch + +from transformers.tokenization_camembert import CamembertTokenizer +from transformers.modeling_camembert import CamembertForMaskedLM + + +def fill_mask(masked_input, model, tokenizer, topk=5): + # Adapted from https://github.com/pytorch/fairseq/blob/master/fairseq/models/roberta/hub_interface.py + assert masked_input.count('') == 1 + input_ids = torch.tensor(tokenizer.encode(masked_input, add_special_tokens=True)).unsqueeze(0) # Batch size 1 + logits = model(input_ids)[0] # The last hidden-state is the first element of the output tuple + masked_index = (input_ids.squeeze() == tokenizer.mask_token_id).nonzero().item() + logits = logits[0, masked_index, :] + prob = logits.softmax(dim=0) + values, indices = prob.topk(k=topk, dim=0) + topk_predicted_token_bpe = ' '.join([tokenizer.convert_ids_to_tokens(indices[i].item()) + for i in range(len(indices))]) + masked_token = tokenizer.mask_token + topk_filled_outputs = [] + for index, predicted_token_bpe in enumerate(topk_predicted_token_bpe.split(' ')): + predicted_token = predicted_token_bpe.replace('\u2581', ' ') + if " {0}".format(masked_token) in masked_input: + topk_filled_outputs.append(( + masked_input.replace( + ' {0}'.format(masked_token), predicted_token + ), + values[index].item(), + predicted_token, + )) + else: + topk_filled_outputs.append(( + masked_input.replace(masked_token, predicted_token), + values[index].item(), + predicted_token, + )) + return topk_filled_outputs + + +tokenizer = CamembertTokenizer.from_pretrained('camembert-base') +model = CamembertForMaskedLM.from_pretrained('camembert-base') +model.eval() + +masked_input = "Le camembert est :)" +print(fill_mask(masked_input, model, tokenizer, topk=3)) diff --git a/examples/distillation/README.md b/examples/distillation/README.md index 8efd1ea6..24a1677d 100644 --- a/examples/distillation/README.md +++ b/examples/distillation/README.md @@ -2,6 +2,10 @@ This folder contains the original code used to train Distil* as well as examples showcasing how to use DistilBERT, DistilRoBERTa and DistilGPT2. +**December 6th, 2019 - Update** We release **DistilmBERT**: 92% of `bert-base-multilingual-cased` on XNLI. The model supports 104 different languages listed [here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages). + +**November 19th, 2019 - Update** We release German **DistilBERT**: 98.8% of `bert-base-german-dbmdz-cased` on NER tasks. + **October 23rd, 2019 - Update** We release **DistilRoBERTa**: 95% of `RoBERTa-base`'s performance on GLUE, twice as fast as RoBERTa while being 35% smaller. **October 3rd, 2019 - Update** We release our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108) explaining our approach on **DistilBERT**. It includes updated results and further experiments. We applied the same method to GPT2 and release the weights of **DistilGPT2**. DistilGPT2 is two times faster and 33% smaller than GPT2. **The paper superseeds our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. Please use the paper as a reference when comparing/reporting results on DistilBERT.** @@ -15,8 +19,9 @@ Distil* is a class of compressed models that started with DistilBERT. DistilBERT We have applied the same method to other Transformer architectures and released the weights: - GPT2: on the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test set of 15.0 compared to 18.5 for **DistilGPT2** (after fine-tuning on the train set). -- RoBERTa: **DistilRoBERTa** reaches 95% of `RoBERTa-base` performance on GLUE while being twice faster and 35% smaller. -- and more to come! 🤗🤗🤗 +- RoBERTa: **DistilRoBERTa** reaches 95% of `RoBERTa-base`'s performance on GLUE while being twice faster and 35% smaller. +- German BERT: **German DistilBERT** reaches 99% of `bert-base-german-dbmdz-cased`'s performance on German NER (CoNLL-2003). +- Multilingual BERT: **DistilmBERT** reaches 92% of Multilingual BERT's performance on XNLI while being twice faster and 25% smaller. The model supports 104 languages listed [here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages). For more information on DistilBERT, please refer to our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108). @@ -27,7 +32,7 @@ Here are the results on the dev sets of GLUE: | BERT-base | **77.6** | 48.9 | 84.3 | 88.6 | 89.3 | 89.5 | 71.3 | 91.7 | 91.2 | 43.7 | | DistilBERT | **76.8** | 49.1 | 81.8 | 90.2 | 90.2 | 89.2 | 62.9 | 92.7 | 90.7 | 44.4 | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| RoBERTa-base (reported) | **83.2**/**86.4**2 | 63.6 | 87.6 | 90.2 | 92.8 | 91.9 | 78.7 | 94.8 | 91.2 | 57.73 | +| RoBERTa-base (reported) | **83.2**/**86.4**2 | 63.6 | 87.6 | 90.2 | 92.8 | 91.9 | 78.7 | 94.8 | 91.2 | 57.73 | | DistilRoBERTa1 | **79.0**/**82.3**2 | 59.4 | 83.9 | 86.6 | 90.8 | 89.4 | 67.9 | 92.5 | 88.3 | 52.1 | 1 We did not use the MNLI checkpoint for fine-tuning but directy perform transfer learning on the pre-trained DistilRoBERTa. @@ -36,6 +41,14 @@ Here are the results on the dev sets of GLUE: 3 We compute this score ourselves for completeness. +Here are the results on the *test* sets for 6 of the languages available in XNLI. The results are computed in the zero shot setting (trained on the English portion and evaluated on the target language portion): + +| Model | English | Spanish | Chinese | German | Arabic | Urdu | +| :---: | :---: | :---: | :---: | :---: | :---: | :---:| +| mBERT base cased (computed) | 82.1 | 74.6 | 69.1 | 72.3 | 66.4 | 58.5 | +| mBERT base uncased (reported)| 81.4 | 74.3 | 63.8 | 70.5 | 62.1 | 58.3 | +| DistilmBERT | 78.2 | 69.1 | 64.0 | 66.3 | 59.1 | 54.7 | + ## Setup This part of the library has only be tested with Python3.6+. There are few specific dependencies to install before launching a distillation, you can install them with the command `pip install -r requirements.txt`. @@ -45,13 +58,14 @@ This part of the library has only be tested with Python3.6+. There are few speci ## How to use DistilBERT -Transformers includes two pre-trained Distil* models, currently only provided for English (we are investigating the possibility to train and release a multilingual version of DistilBERT): +Transformers includes five pre-trained Distil* models, currently only provided for English and German (we are investigating the possibility to train and release a multilingual version of DistilBERT): - `distilbert-base-uncased`: DistilBERT English language model pretrained on the same data used to pretrain Bert (concatenation of the Toronto Book Corpus and full English Wikipedia) using distillation with the supervision of the `bert-base-uncased` version of Bert. The model has 6 layers, 768 dimension and 12 heads, totalizing 66M parameters. - `distilbert-base-uncased-distilled-squad`: A finetuned version of `distilbert-base-uncased` finetuned using (a second step of) knwoledge distillation on SQuAD 1.0. This model reaches a F1 score of 86.9 on the dev set (for comparison, Bert `bert-base-uncased` version reaches a 88.5 F1 score). +- `distilbert-base-german-cased`: DistilBERT German language model pretrained on 1/2 of the data used to pretrain Bert using distillation with the supervision of the `bert-base-german-dbmdz-cased` version of German DBMDZ Bert. For NER tasks the model reaches a F1 score of 83.49 on the CoNLL-2003 test set (for comparison, `bert-base-german-dbmdz-cased` reaches a 84.52 F1 score), and a F1 score of 85.23 on the GermEval 2014 test set (`bert-base-german-dbmdz-cased` reaches a 86.89 F1 score). - `distilgpt2`: DistilGPT2 English language model pretrained with the supervision of `gpt2` (the smallest version of GPT2) on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset. The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 124M parameters for GPT2). On average, DistilGPT2 is two times faster than GPT2. - `distilroberta-base`: DistilRoBERTa English language model pretrained with the supervision of `roberta-base` solely on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset (it is ~4 times less training data than the teacher RoBERTa). The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 125M parameters for RoBERTa-base). On average DistilRoBERTa is twice as fast as Roberta-base. -- and more to come! 🤗🤗🤗 +- `distilbert-base-multilingual-cased`: DistilmBERT multilingual model pretrained with the supervision of `bert-base-multilingual-cased` on the concatenation of Wikipedia in 104 different languages. The model supports the 104 languages listed [here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages). The model has 6 layers, 768 dimension and 12 heads, totalizing 134M parameters (compared to 177M parameters for mBERT-base). On average DistilmBERT is twice as fast as mBERT-base. Using DistilBERT is very similar to using BERT. DistilBERT share the same tokenizer as BERT's `bert-base-uncased` even though we provide a link to this tokenizer under the `DistilBertTokenizer` name to have a consistent naming between the library models. @@ -67,6 +81,7 @@ last_hidden_states = outputs[0] # The last hidden-state is the first element of Similarly, using the other Distil* models simply consists in calling the base classes with a different pretrained checkpoint: - DistilGPT2: `model = GPT2Model.from_pretrained('distilgpt2')` - DistilRoBERTa: `model = RobertaModel.from_pretrained('distilroberta-base')` +- DistilmBERT: `model = DistilBertModel.from_pretrained('distilbert-base-multilingual-cased')` ## How to train Distil* diff --git a/examples/distillation/distiller.py b/examples/distillation/distiller.py index 0442072e..1e33190a 100644 --- a/examples/distillation/distiller.py +++ b/examples/distillation/distiller.py @@ -21,7 +21,6 @@ import psutil import time from tqdm import trange, tqdm import numpy as np -import psutil import torch import torch.nn as nn diff --git a/examples/distillation/requirements.txt b/examples/distillation/requirements.txt index d76273b3..491924ee 100644 --- a/examples/distillation/requirements.txt +++ b/examples/distillation/requirements.txt @@ -3,4 +3,4 @@ tensorboard>=1.14.0 tensorboardX==1.8 psutil==5.6.3 scipy==1.3.1 -transformers==2.0.0 +transformers diff --git a/examples/pplm/README.md b/examples/pplm/README.md new file mode 100644 index 00000000..b1220585 --- /dev/null +++ b/examples/pplm/README.md @@ -0,0 +1,54 @@ +# Plug and Play Language Models: a Simple Approach to Controlled Text Generation + +Authors: [Sumanth Dathathri](https://dathath.github.io/), [Andrea Madotto](https://andreamad8.github.io/), Janice Lan, Jane Hung, Eric Frank, [Piero Molino](https://w4nderlu.st/), [Jason Yosinski](http://yosinski.com/), and [Rosanne Liu](http://www.rosanneliu.com/) + +This folder contains the original code used to run the Plug and Play Language Model (PPLM). + +Paper link: https://arxiv.org/abs/1912.02164 + +Blog link: https://eng.uber.com/pplm + +Please check out the repo under uber-research for more information: https://github.com/uber-research/PPLM + + +## Setup + +```bash +git clone https://github.com/huggingface/transformers && cd transformers +pip install [--editable] . +pip install nltk torchtext # additional requirements. +cd examples/pplm +``` + +## PPLM-BoW + +### Example command for bag-of-words control + +```bash +python run_pplm.py -B military --cond_text "The potato" --length 50 --gamma 1.5 --num_iterations 3 --num_samples 10 --stepsize 0.03 --window_length 5 --kl_scale 0.01 --gm_scale 0.99 --colorama --sample +``` + +### Tuning hyperparameters for bag-of-words control + +1. Increase `--stepsize` to intensify topic control, and decrease its value to soften the control. `--stepsize 0` recovers the original uncontrolled GPT-2 model. + +2. If the language being generated is repetitive (For e.g. "science science experiment experiment"), there are several options to consider:
+ a) Reduce the `--stepsize`
+ b) Increase `--kl_scale` (the KL-loss coefficient) or decrease `--gm_scale` (the gm-scaling term)
+ c) Add `--grad-length xx` where xx is an (integer <= length, e.g. `--grad-length 30`).
+ + +## PPLM-Discrim + +### Example command for discriminator based sentiment control + +```bash +python run_pplm.py -D sentiment --class_label 2 --cond_text "My dog died" --length 50 --gamma 1.0 --num_iterations 10 --num_samples 10 --stepsize 0.04 --kl_scale 0.01 --gm_scale 0.95 --sample +``` + +### Tuning hyperparameters for discriminator control + +1. Increase `--stepsize` to intensify topic control, and decrease its value to soften the control. `--stepsize 0` recovers the original uncontrolled GPT-2 model. + +2. Use `--class_label 3` for negative, and `--class_label 2` for positive + diff --git a/examples/pplm/imgs/headfigure.png b/examples/pplm/imgs/headfigure.png new file mode 100644 index 00000000..f4c11ad5 Binary files /dev/null and b/examples/pplm/imgs/headfigure.png differ diff --git a/examples/pplm/imgs/wooly.png b/examples/pplm/imgs/wooly.png new file mode 100644 index 00000000..190d3afd Binary files /dev/null and b/examples/pplm/imgs/wooly.png differ diff --git a/examples/pplm/pplm_classification_head.py b/examples/pplm/pplm_classification_head.py new file mode 100644 index 00000000..9aae0f17 --- /dev/null +++ b/examples/pplm/pplm_classification_head.py @@ -0,0 +1,18 @@ +import torch + +class ClassificationHead(torch.nn.Module): + """Classification Head for transformer encoders""" + + def __init__(self, class_size, embed_size): + super(ClassificationHead, self).__init__() + self.class_size = class_size + self.embed_size = embed_size + # self.mlp1 = torch.nn.Linear(embed_size, embed_size) + # self.mlp2 = (torch.nn.Linear(embed_size, class_size)) + self.mlp = torch.nn.Linear(embed_size, class_size) + + def forward(self, hidden_state): + # hidden_state = F.relu(self.mlp1(hidden_state)) + # hidden_state = self.mlp2(hidden_state) + logits = self.mlp(hidden_state) + return logits diff --git a/examples/pplm/run_pplm.py b/examples/pplm/run_pplm.py new file mode 100644 index 00000000..095dc39a --- /dev/null +++ b/examples/pplm/run_pplm.py @@ -0,0 +1,879 @@ +#! /usr/bin/env python3 +# coding=utf-8 + +#Copyright (c) 2019 Uber Technologies, Inc. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. + +""" +Example command with bag of words: +python examples/run_pplm.py -B space --cond_text "The president" --length 100 --gamma 1.5 --num_iterations 3 --num_samples 10 --stepsize 0.01 --window_length 5 --kl_scale 0.01 --gm_scale 0.95 + +Example command with discriminator: +python examples/run_pplm.py -D sentiment --class_label 3 --cond_text "The lake" --length 10 --gamma 1.0 --num_iterations 30 --num_samples 10 --stepsize 0.01 --kl_scale 0.01 --gm_scale 0.95 +""" + +import argparse +import json +from operator import add +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch.autograd import Variable +from tqdm import trange + +from transformers import GPT2Tokenizer +from transformers.file_utils import cached_path +from transformers.modeling_gpt2 import GPT2LMHeadModel +from pplm_classification_head import ClassificationHead + +PPLM_BOW = 1 +PPLM_DISCRIM = 2 +PPLM_BOW_DISCRIM = 3 +SMALL_CONST = 1e-15 +BIG_CONST = 1e10 + +BAG_OF_WORDS_ARCHIVE_MAP = { + 'legal': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/legal.txt", + 'military': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/military.txt", + 'politics': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/politics.txt", + 'religion': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/religion.txt", + 'science': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/science.txt", + 'space': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/space.txt", + 'technology': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/technology.txt", +} + +DISCRIMINATOR_MODELS_PARAMS = { + "clickbait": { + "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/clickbait_classifier_head.pt", + "class_size": 2, + "embed_size": 1024, + "class_vocab": {"non_clickbait": 0, "clickbait": 1}, + "default_class": 1, + "pretrained_model": "gpt2-medium", + }, + "sentiment": { + "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/SST_classifier_head.pt", + "class_size": 5, + "embed_size": 1024, + "class_vocab": {"very_positive": 2, "very_negative": 3}, + "default_class": 3, + "pretrained_model": "gpt2-medium", + }, +} + + +def to_var(x, requires_grad=False, volatile=False, device='cuda'): + if torch.cuda.is_available() and device == 'cuda': + x = x.cuda() + elif device != 'cuda': + x = x.to(device) + return Variable(x, requires_grad=requires_grad, volatile=volatile) + + +def top_k_filter(logits, k, probs=False): + """ + Masks everything but the k top entries as -infinity (1e10). + Used to mask logits such that e^-infinity -> 0 won't contribute to the + sum of the denominator. + """ + if k == 0: + return logits + else: + values = torch.topk(logits, k)[0] + batch_mins = values[:, -1].view(-1, 1).expand_as(logits) + if probs: + return torch.where(logits < batch_mins, + torch.ones_like(logits) * 0.0, logits) + return torch.where(logits < batch_mins, + torch.ones_like(logits) * -BIG_CONST, + logits) + + +def perturb_past( + past, + model, + last, + unpert_past=None, + unpert_logits=None, + accumulated_hidden=None, + grad_norms=None, + stepsize=0.01, + one_hot_bows_vectors=None, + classifier=None, + class_label=None, + loss_type=0, + num_iterations=3, + horizon_length=1, + window_length=0, + decay=False, + gamma=1.5, + kl_scale=0.01, + device='cuda', +): + # Generate inital perturbed past + grad_accumulator = [ + (np.zeros(p.shape).astype("float32")) + for p in past + ] + + if accumulated_hidden is None: + accumulated_hidden = 0 + + if decay: + decay_mask = torch.arange( + 0., + 1.0 + SMALL_CONST, + 1.0 / (window_length) + )[1:] + else: + decay_mask = 1.0 + + # TODO fix this comment (SUMANTH) + # Generate a mask is gradient perturbated is based on a past window + _, _, _, curr_length, _ = past[0].shape + + if curr_length > window_length and window_length > 0: + ones_key_val_shape = ( + tuple(past[0].shape[:-2]) + + tuple([window_length]) + + tuple(past[0].shape[-1:]) + ) + + zeros_key_val_shape = ( + tuple(past[0].shape[:-2]) + + tuple([curr_length - window_length]) + + tuple(past[0].shape[-1:]) + ) + + ones_mask = torch.ones(ones_key_val_shape) + ones_mask = decay_mask * ones_mask.permute(0, 1, 2, 4, 3) + ones_mask = ones_mask.permute(0, 1, 2, 4, 3) + + window_mask = torch.cat( + (ones_mask, torch.zeros(zeros_key_val_shape)), + dim=-2 + ).to(device) + else: + window_mask = torch.ones_like(past[0]).to(device) + + # accumulate perturbations for num_iterations + loss_per_iter = [] + new_accumulated_hidden = None + for i in range(num_iterations): + print("Iteration ", i + 1) + curr_perturbation = [ + to_var(torch.from_numpy(p_), requires_grad=True, device=device) + for p_ in grad_accumulator + ] + + # Compute hidden using perturbed past + perturbed_past = list(map(add, past, curr_perturbation)) + _, _, _, curr_length, _ = curr_perturbation[0].shape + all_logits, _, all_hidden = model(last, past=perturbed_past) + hidden = all_hidden[-1] + new_accumulated_hidden = accumulated_hidden + torch.sum( + hidden, + dim=1 + ).detach() + # TODO: Check the layer-norm consistency of this with trained discriminator (Sumanth) + logits = all_logits[:, -1, :] + probs = F.softmax(logits, dim=-1) + + loss = 0.0 + loss_list = [] + if loss_type == PPLM_BOW or loss_type == PPLM_BOW_DISCRIM: + for one_hot_bow in one_hot_bows_vectors: + bow_logits = torch.mm(probs, torch.t(one_hot_bow)) + bow_loss = -torch.log(torch.sum(bow_logits)) + loss += bow_loss + loss_list.append(bow_loss) + print(" pplm_bow_loss:", loss.data.cpu().numpy()) + + if loss_type == 2 or loss_type == 3: + ce_loss = torch.nn.CrossEntropyLoss() + # TODO why we need to do this assignment and not just using unpert_past? (Sumanth) + curr_unpert_past = unpert_past + curr_probs = torch.unsqueeze(probs, dim=1) + wte = model.resize_token_embeddings() + for _ in range(horizon_length): + inputs_embeds = torch.matmul(curr_probs, wte.weight.data) + _, curr_unpert_past, curr_all_hidden = model( + past=curr_unpert_past, + inputs_embeds=inputs_embeds + ) + curr_hidden = curr_all_hidden[-1] + new_accumulated_hidden = new_accumulated_hidden + torch.sum( + curr_hidden, dim=1) + + prediction = classifier(new_accumulated_hidden / + (curr_length + 1 + horizon_length)) + + label = torch.tensor(prediction.shape[0] * [class_label], + device=device, + dtype=torch.long) + discrim_loss = ce_loss(prediction, label) + print(" pplm_discrim_loss:", discrim_loss.data.cpu().numpy()) + loss += discrim_loss + loss_list.append(discrim_loss) + + kl_loss = 0.0 + if kl_scale > 0.0: + unpert_probs = F.softmax(unpert_logits[:, -1, :], dim=-1) + unpert_probs = ( + unpert_probs + SMALL_CONST * + (unpert_probs <= SMALL_CONST).float().to(device).detach() + ) + correction = SMALL_CONST * (probs <= SMALL_CONST).float().to( + device).detach() + corrected_probs = probs + correction.detach() + kl_loss = kl_scale * ( + (corrected_probs * (corrected_probs / unpert_probs).log()).sum() + ) + print(' kl_loss', kl_loss.data.cpu().numpy()) + loss += kl_loss + + loss_per_iter.append(loss.data.cpu().numpy()) + print(' pplm_loss', (loss - kl_loss).data.cpu().numpy()) + + # compute gradients + loss.backward() + + # calculate gradient norms + if grad_norms is not None and loss_type == PPLM_BOW: + grad_norms = [ + torch.max(grad_norms[index], torch.norm(p_.grad * window_mask)) + for index, p_ in enumerate(curr_perturbation) + ] + else: + grad_norms = [ + (torch.norm(p_.grad * window_mask) + SMALL_CONST) + for index, p_ in enumerate(curr_perturbation) + ] + + # normalize gradients + grad = [ + -stepsize * + (p_.grad * window_mask / grad_norms[ + index] ** gamma).data.cpu().numpy() + for index, p_ in enumerate(curr_perturbation) + ] + + # accumulate gradient + grad_accumulator = list(map(add, grad, grad_accumulator)) + + # reset gradients, just to make sure + for p_ in curr_perturbation: + p_.grad.data.zero_() + + # removing past from the graph + new_past = [] + for p_ in past: + new_past.append(p_.detach()) + past = new_past + + # apply the accumulated perturbations to the past + grad_accumulator = [ + to_var(torch.from_numpy(p_), requires_grad=True, device=device) + for p_ in grad_accumulator + ] + pert_past = list(map(add, past, grad_accumulator)) + + return pert_past, new_accumulated_hidden, grad_norms, loss_per_iter + + +def get_classifier( + name: Optional[str], class_label: Union[str, int], + device: str +) -> Tuple[Optional[ClassificationHead], Optional[int]]: + if name is None: + return None, None + + params = DISCRIMINATOR_MODELS_PARAMS[name] + classifier = ClassificationHead( + class_size=params['class_size'], + embed_size=params['embed_size'] + ).to(device) + if "url" in params: + resolved_archive_file = cached_path(params["url"]) + elif "path" in params: + resolved_archive_file = params["path"] + else: + raise ValueError("Either url or path have to be specified " + "in the discriminator model parameters") + classifier.load_state_dict( + torch.load(resolved_archive_file, map_location=device)) + classifier.eval() + + if isinstance(class_label, str): + if class_label in params["class_vocab"]: + label_id = params["class_vocab"][class_label] + else: + label_id = params["default_class"] + print("class_label {} not in class_vocab".format(class_label)) + print("available values are: {}".format(params["class_vocab"])) + print("using default class {}".format(label_id)) + + elif isinstance(class_label, int): + if class_label in set(params["class_vocab"].values()): + label_id = class_label + else: + label_id = params["default_class"] + print("class_label {} not in class_vocab".format(class_label)) + print("available values are: {}".format(params["class_vocab"])) + print("using default class {}".format(label_id)) + + else: + label_id = params["default_class"] + + return classifier, label_id + + +def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str], tokenizer) -> \ + List[List[List[int]]]: + bow_indices = [] + for id_or_path in bag_of_words_ids_or_paths: + if id_or_path in BAG_OF_WORDS_ARCHIVE_MAP: + filepath = cached_path(BAG_OF_WORDS_ARCHIVE_MAP[id_or_path]) + else: + filepath = id_or_path + with open(filepath, "r") as f: + words = f.read().strip().split("\n") + bow_indices.append( + [tokenizer.encode(word.strip(), add_prefix_space=True) for word in + words]) + return bow_indices + + +def build_bows_one_hot_vectors(bow_indices, tokenizer, device='cuda'): + if bow_indices is None: + return None + + one_hot_bows_vectors = [] + for single_bow in bow_indices: + single_bow = list(filter(lambda x: len(x) <= 1, single_bow)) + single_bow = torch.tensor(single_bow).to(device) + num_words = single_bow.shape[0] + one_hot_bow = torch.zeros(num_words, tokenizer.vocab_size).to(device) + one_hot_bow.scatter_(1, single_bow, 1) + one_hot_bows_vectors.append(one_hot_bow) + return one_hot_bows_vectors + + +def full_text_generation( + model, + tokenizer, + context=None, + num_samples=1, + device="cuda", + bag_of_words=None, + discrim=None, + class_label=None, + length=100, + stepsize=0.02, + temperature=1.0, + top_k=10, + sample=False, + num_iterations=3, + grad_length=10000, + horizon_length=1, + window_length=0, + decay=False, + gamma=1.5, + gm_scale=0.9, + kl_scale=0.01, + **kwargs +): + classifier, class_id = get_classifier( + discrim, + class_label, + device + ) + + bow_indices = [] + if bag_of_words: + bow_indices = get_bag_of_words_indices(bag_of_words.split(";"), + tokenizer) + + if bag_of_words and classifier: + print("Both PPLM-BoW and PPLM-Discrim are on. This is not optimized.") + loss_type = PPLM_BOW_DISCRIM + + elif bag_of_words: + loss_type = PPLM_BOW + print("Using PPLM-BoW") + + elif classifier is not None: + loss_type = PPLM_DISCRIM + print("Using PPLM-Discrim") + + else: + raise Exception("Specify either a bag of words or a discriminator") + + unpert_gen_tok_text, _, _ = generate_text_pplm( + model=model, + tokenizer=tokenizer, + context=context, + device=device, + length=length, + sample=sample, + perturb=False + ) + if device == 'cuda': + torch.cuda.empty_cache() + + pert_gen_tok_texts = [] + discrim_losses = [] + losses_in_time = [] + + for i in range(num_samples): + pert_gen_tok_text, discrim_loss, loss_in_time = generate_text_pplm( + model=model, + tokenizer=tokenizer, + context=context, + device=device, + perturb=True, + bow_indices=bow_indices, + classifier=classifier, + class_label=class_id, + loss_type=loss_type, + length=length, + stepsize=stepsize, + temperature=temperature, + top_k=top_k, + sample=sample, + num_iterations=num_iterations, + grad_length=grad_length, + horizon_length=horizon_length, + window_length=window_length, + decay=decay, + gamma=gamma, + gm_scale=gm_scale, + kl_scale=kl_scale, + ) + pert_gen_tok_texts.append(pert_gen_tok_text) + if classifier is not None: + discrim_losses.append(discrim_loss.data.cpu().numpy()) + losses_in_time.append(loss_in_time) + + if device == 'cuda': + torch.cuda.empty_cache() + + return unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time + + +def generate_text_pplm( + model, + tokenizer, + context=None, + past=None, + device="cuda", + perturb=True, + bow_indices=None, + classifier=None, + class_label=None, + loss_type=0, + length=100, + stepsize=0.02, + temperature=1.0, + top_k=10, + sample=False, + num_iterations=3, + grad_length=10000, + horizon_length=1, + window_length=0, + decay=False, + gamma=1.5, + gm_scale=0.9, + kl_scale=0.01, +): + output_so_far = None + if context: + context_t = torch.tensor(context, device=device, dtype=torch.long) + while len(context_t.shape) < 2: + context_t = context_t.unsqueeze(0) + output_so_far = context_t + + # collect one hot vectors for bags of words + one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices, tokenizer, + device) + + grad_norms = None + last = None + unpert_discrim_loss = 0 + loss_in_time = [] + for i in trange(length, ascii=True): + + # Get past/probs for current output, except for last word + # Note that GPT takes 2 inputs: past + current_token + + # run model forward to obtain unperturbed + if past is None and output_so_far is not None: + last = output_so_far[:, -1:] + if output_so_far.shape[1] > 1: + _, past, _ = model(output_so_far[:, :-1]) + + unpert_logits, unpert_past, unpert_all_hidden = model(output_so_far) + unpert_last_hidden = unpert_all_hidden[-1] + + # check if we are abowe grad max length + if i >= grad_length: + current_stepsize = stepsize * 0 + else: + current_stepsize = stepsize + + # modify the past if necessary + if not perturb or num_iterations == 0: + pert_past = past + + else: + accumulated_hidden = unpert_last_hidden[:, :-1, :] + accumulated_hidden = torch.sum(accumulated_hidden, dim=1) + + if past is not None: + pert_past, _, grad_norms, loss_this_iter = perturb_past( + past, + model, + last, + unpert_past=unpert_past, + unpert_logits=unpert_logits, + accumulated_hidden=accumulated_hidden, + grad_norms=grad_norms, + stepsize=current_stepsize, + one_hot_bows_vectors=one_hot_bows_vectors, + classifier=classifier, + class_label=class_label, + loss_type=loss_type, + num_iterations=num_iterations, + horizon_length=horizon_length, + window_length=window_length, + decay=decay, + gamma=gamma, + kl_scale=kl_scale, + device=device, + ) + loss_in_time.append(loss_this_iter) + else: + pert_past = past + + pert_logits, past, pert_all_hidden = model(last, past=pert_past) + pert_logits = pert_logits[:, -1, :] / temperature # + SMALL_CONST + pert_probs = F.softmax(pert_logits, dim=-1) + + if classifier is not None: + ce_loss = torch.nn.CrossEntropyLoss() + prediction = classifier(torch.mean(unpert_last_hidden, dim=1)) + label = torch.tensor([class_label], device=device, + dtype=torch.long) + unpert_discrim_loss = ce_loss(prediction, label) + print( + "unperturbed discrim loss", + unpert_discrim_loss.data.cpu().numpy() + ) + else: + unpert_discrim_loss = 0 + + # Fuse the modified model and original model + if perturb: + + unpert_probs = F.softmax(unpert_logits[:, -1, :], dim=-1) + + pert_probs = ((pert_probs ** gm_scale) * ( + unpert_probs ** (1 - gm_scale))) # + SMALL_CONST + pert_probs = top_k_filter(pert_probs, k=top_k, + probs=True) # + SMALL_CONST + + # rescale + if torch.sum(pert_probs) <= 1: + pert_probs = pert_probs / torch.sum(pert_probs) + + else: + pert_logits = top_k_filter(pert_logits, k=top_k) # + SMALL_CONST + pert_probs = F.softmax(pert_logits, dim=-1) + + # sample or greedy + if sample: + last = torch.multinomial(pert_probs, num_samples=1) + + else: + _, last = torch.topk(pert_probs, k=1, dim=-1) + + # update context/output_so_far appending the new token + output_so_far = ( + last if output_so_far is None + else torch.cat((output_so_far, last), dim=1) + ) + + print(tokenizer.decode(output_so_far.tolist()[0])) + + return output_so_far, unpert_discrim_loss, loss_in_time + + +def set_generic_model_params(discrim_weights, discrim_meta): + if discrim_weights is None: + raise ValueError('When using a generic discriminator, ' + 'discrim_weights need to be specified') + if discrim_meta is None: + raise ValueError('When using a generic discriminator, ' + 'discrim_meta need to be specified') + + with open(discrim_meta, 'r') as discrim_meta_file: + meta = json.load(discrim_meta_file) + meta['path'] = discrim_weights + DISCRIMINATOR_MODELS_PARAMS['generic'] = meta + + +def run_pplm_example( + pretrained_model="gpt2-medium", + cond_text="", + uncond=False, + num_samples=1, + bag_of_words=None, + discrim=None, + discrim_weights=None, + discrim_meta=None, + class_label=-1, + length=100, + stepsize=0.02, + temperature=1.0, + top_k=10, + sample=False, + num_iterations=3, + grad_length=10000, + horizon_length=1, + window_length=0, + decay=False, + gamma=1.5, + gm_scale=0.9, + kl_scale=0.01, + seed=0, + no_cuda=False, + colorama=False +): + # set Random seed + torch.manual_seed(seed) + np.random.seed(seed) + + # set the device + device = "cuda" if torch.cuda.is_available() and not no_cuda else "cpu" + + if discrim == 'generic': + set_generic_model_params(discrim_weights, discrim_meta) + + if discrim is not None: + pretrained_model = DISCRIMINATOR_MODELS_PARAMS[discrim][ + "pretrained_model" + ] + print("discrim = {}, pretrained_model set " + "to discriminator's = {}".format(discrim, pretrained_model)) + + # load pretrained model + model = GPT2LMHeadModel.from_pretrained( + pretrained_model, + output_hidden_states=True + ) + model.to(device) + model.eval() + + # load tokenizer + tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model) + + # Freeze GPT-2 weights + for param in model.parameters(): + param.requires_grad = False + + # figure out conditioning text + if uncond: + tokenized_cond_text = tokenizer.encode( + [tokenizer.bos_token] + ) + else: + raw_text = cond_text + while not raw_text: + print("Did you forget to add `--cond_text`? ") + raw_text = input("Model prompt >>> ") + tokenized_cond_text = tokenizer.encode(tokenizer.bos_token + raw_text) + + print("= Prefix of sentence =") + print(tokenizer.decode(tokenized_cond_text)) + print() + + # generate unperturbed and perturbed texts + + # full_text_generation returns: + # unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time + unpert_gen_tok_text, pert_gen_tok_texts, _, _ = full_text_generation( + model=model, + tokenizer=tokenizer, + context=tokenized_cond_text, + device=device, + num_samples=num_samples, + bag_of_words=bag_of_words, + discrim=discrim, + class_label=class_label, + length=length, + stepsize=stepsize, + temperature=temperature, + top_k=top_k, + sample=sample, + num_iterations=num_iterations, + grad_length=grad_length, + horizon_length=horizon_length, + window_length=window_length, + decay=decay, + gamma=gamma, + gm_scale=gm_scale, + kl_scale=kl_scale, + ) + + # untokenize unperturbed text + unpert_gen_text = tokenizer.decode(unpert_gen_tok_text.tolist()[0]) + + print("=" * 80) + print("= Unperturbed generated text =") + print(unpert_gen_text) + print() + + generated_texts = [] + + bow_word_ids = set() + if bag_of_words and colorama: + bow_indices = get_bag_of_words_indices(bag_of_words.split(";"), + tokenizer) + for single_bow_list in bow_indices: + # filtering all words in the list composed of more than 1 token + filtered = list(filter(lambda x: len(x) <= 1, single_bow_list)) + # w[0] because we are sure w has only 1 item because previous fitler + bow_word_ids.update(w[0] for w in filtered) + + # iterate through the perturbed texts + for i, pert_gen_tok_text in enumerate(pert_gen_tok_texts): + try: + # untokenize unperturbed text + if colorama: + import colorama + + pert_gen_text = '' + for word_id in pert_gen_tok_text.tolist()[0]: + if word_id in bow_word_ids: + pert_gen_text += '{}{}{}'.format( + colorama.Fore.RED, + tokenizer.decode([word_id]), + colorama.Style.RESET_ALL + ) + else: + pert_gen_text += tokenizer.decode([word_id]) + else: + pert_gen_text = tokenizer.decode(pert_gen_tok_text.tolist()[0]) + + print("= Perturbed generated text {} =".format(i + 1)) + print(pert_gen_text) + print() + except: + pass + + # keep the prefix, perturbed seq, original seq for each index + generated_texts.append( + (tokenized_cond_text, pert_gen_tok_text, unpert_gen_tok_text) + ) + + return + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + "--pretrained_model", + "-M", + type=str, + default="gpt2-medium", + help="pretrained model name or path to local checkpoint", + ) + parser.add_argument( + "--cond_text", type=str, default="The lake", + help="Prefix texts to condition on" + ) + parser.add_argument( + "--uncond", action="store_true", + help="Generate from end-of-text as prefix" + ) + parser.add_argument( + "--num_samples", + type=int, + default=1, + help="Number of samples to generate from the modified latents", + ) + parser.add_argument( + "--bag_of_words", + "-B", + type=str, + default=None, + help="Bags of words used for PPLM-BoW. " + "Either a BOW id (see list in code) or a filepath. " + "Multiple BoWs separated by ;", + ) + parser.add_argument( + "--discrim", + "-D", + type=str, + default=None, + choices=("clickbait", "sentiment", "toxicity", "generic"), + help="Discriminator to use", + ) + parser.add_argument('--discrim_weights', type=str, default=None, + help='Weights for the generic discriminator') + parser.add_argument('--discrim_meta', type=str, default=None, + help='Meta information for the generic discriminator') + parser.add_argument( + "--class_label", + type=int, + default=-1, + help="Class label used for the discriminator", + ) + parser.add_argument("--length", type=int, default=100) + parser.add_argument("--stepsize", type=float, default=0.02) + parser.add_argument("--temperature", type=float, default=1.0) + parser.add_argument("--top_k", type=int, default=10) + parser.add_argument( + "--sample", action="store_true", + help="Generate from end-of-text as prefix" + ) + parser.add_argument("--num_iterations", type=int, default=3) + parser.add_argument("--grad_length", type=int, default=10000) + parser.add_argument( + "--window_length", + type=int, + default=0, + help="Length of past which is being optimized; " + "0 corresponds to infinite window length", + ) + parser.add_argument( + "--horizon_length", + type=int, + default=1, + help="Length of future to optimize over", + ) + parser.add_argument("--decay", action="store_true", + help="whether to decay or not") + parser.add_argument("--gamma", type=float, default=1.5) + parser.add_argument("--gm_scale", type=float, default=0.9) + parser.add_argument("--kl_scale", type=float, default=0.01) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--no_cuda", action="store_true", help="no cuda") + parser.add_argument("--colorama", action="store_true", + help="colors keywords") + + args = parser.parse_args() + run_pplm_example(**vars(args)) diff --git a/examples/pplm/run_pplm_discrim_train.py b/examples/pplm/run_pplm_discrim_train.py new file mode 100644 index 00000000..3055139d --- /dev/null +++ b/examples/pplm/run_pplm_discrim_train.py @@ -0,0 +1,588 @@ +#! /usr/bin/env python3 +# coding=utf-8 + +#Copyright (c) 2019 Uber Technologies, Inc. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. + +import argparse +import csv +import json +import math +import time + +import numpy as np +import torch +import torch.nn.functional as F +import torch.optim +import torch.optim as optim +import torch.utils.data as data +from nltk.tokenize.treebank import TreebankWordDetokenizer +from torchtext import data as torchtext_data +from torchtext import datasets +from tqdm import tqdm, trange + +from transformers import GPT2Tokenizer, GPT2LMHeadModel +from pplm_classification_head import ClassificationHead + +torch.manual_seed(0) +np.random.seed(0) +EPSILON = 1e-10 +example_sentence = "This is incredible! I love it, this is the best chicken I have ever had." +max_length_seq = 100 + + + + +class Discriminator(torch.nn.Module): + """Transformer encoder followed by a Classification Head""" + + def __init__( + self, + class_size, + pretrained_model="gpt2-medium", + cached_mode=False, + device='cpu' + ): + super(Discriminator, self).__init__() + self.tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model) + self.encoder = GPT2LMHeadModel.from_pretrained(pretrained_model) + self.embed_size = self.encoder.transformer.config.hidden_size + self.classifier_head = ClassificationHead( + class_size=class_size, + embed_size=self.embed_size + ) + self.cached_mode = cached_mode + self.device = device + + def get_classifier(self): + return self.classifier_head + + def train_custom(self): + for param in self.encoder.parameters(): + param.requires_grad = False + self.classifier_head.train() + + def avg_representation(self, x): + mask = x.ne(0).unsqueeze(2).repeat( + 1, 1, self.embed_size + ).float().to(self.device).detach() + hidden, _ = self.encoder.transformer(x) + masked_hidden = hidden * mask + avg_hidden = torch.sum(masked_hidden, dim=1) / ( + torch.sum(mask, dim=1).detach() + EPSILON + ) + return avg_hidden + + def forward(self, x): + if self.cached_mode: + avg_hidden = x.to(self.device) + else: + avg_hidden = self.avg_representation(x.to(self.device)) + + logits = self.classifier_head(avg_hidden) + probs = F.log_softmax(logits, dim=-1) + + return probs + + +class Dataset(data.Dataset): + def __init__(self, X, y): + """Reads source and target sequences from txt files.""" + self.X = X + self.y = y + + def __len__(self): + return len(self.X) + + def __getitem__(self, index): + """Returns one data pair (source and target).""" + data = {} + data["X"] = self.X[index] + data["y"] = self.y[index] + return data + + +def collate_fn(data): + def pad_sequences(sequences): + lengths = [len(seq) for seq in sequences] + + padded_sequences = torch.zeros( + len(sequences), + max(lengths) + ).long() # padding value = 0 + + for i, seq in enumerate(sequences): + end = lengths[i] + padded_sequences[i, :end] = seq[:end] + + return padded_sequences, lengths + + item_info = {} + for key in data[0].keys(): + item_info[key] = [d[key] for d in data] + + x_batch, _ = pad_sequences(item_info["X"]) + y_batch = torch.tensor(item_info["y"], dtype=torch.long) + + return x_batch, y_batch + + +def cached_collate_fn(data): + item_info = {} + for key in data[0].keys(): + item_info[key] = [d[key] for d in data] + + x_batch = torch.cat(item_info["X"], 0) + y_batch = torch.tensor(item_info["y"], dtype=torch.long) + + return x_batch, y_batch + + +def train_epoch(data_loader, discriminator, optimizer, + epoch=0, log_interval=10, device='cpu'): + samples_so_far = 0 + discriminator.train_custom() + for batch_idx, (input_t, target_t) in enumerate(data_loader): + input_t, target_t = input_t.to(device), target_t.to(device) + + optimizer.zero_grad() + + output_t = discriminator(input_t) + loss = F.nll_loss(output_t, target_t) + loss.backward(retain_graph=True) + optimizer.step() + + samples_so_far += len(input_t) + + if batch_idx % log_interval == 0: + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch + 1, + samples_so_far, len(data_loader.dataset), + 100 * samples_so_far / len(data_loader.dataset), loss.item() + ) + ) + + +def evaluate_performance(data_loader, discriminator, device='cpu'): + discriminator.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for input_t, target_t in data_loader: + input_t, target_t = input_t.to(device), target_t.to(device) + output_t = discriminator(input_t) + # sum up batch loss + test_loss += F.nll_loss(output_t, target_t, reduction="sum").item() + # get the index of the max log-probability + pred_t = output_t.argmax(dim=1, keepdim=True) + correct += pred_t.eq(target_t.view_as(pred_t)).sum().item() + + test_loss /= len(data_loader.dataset) + + print( + "Performance on test set: " + "Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)".format( + test_loss, correct, len(data_loader.dataset), + 100. * correct / len(data_loader.dataset) + ) + ) + + +def predict(input_sentence, model, classes, cached=False, device='cpu'): + input_t = model.tokenizer.encode(input_sentence) + input_t = torch.tensor([input_t], dtype=torch.long, device=device) + if cached: + input_t = model.avg_representation(input_t) + + log_probs = model(input_t).data.cpu().numpy().flatten().tolist() + print("Input sentence:", input_sentence) + print("Predictions:", ", ".join( + "{}: {:.4f}".format(c, math.exp(log_prob)) for c, log_prob in + zip(classes, log_probs) + )) + + +def get_cached_data_loader(dataset, batch_size, discriminator, + shuffle=False, device='cpu'): + data_loader = torch.utils.data.DataLoader(dataset=dataset, + batch_size=batch_size, + collate_fn=collate_fn) + + xs = [] + ys = [] + for batch_idx, (x, y) in enumerate(tqdm(data_loader, ascii=True)): + with torch.no_grad(): + x = x.to(device) + avg_rep = discriminator.avg_representation(x).cpu().detach() + avg_rep_list = torch.unbind(avg_rep.unsqueeze(1)) + xs += avg_rep_list + ys += y.cpu().numpy().tolist() + + data_loader = torch.utils.data.DataLoader( + dataset=Dataset(xs, ys), + batch_size=batch_size, + shuffle=shuffle, + collate_fn=cached_collate_fn) + + return data_loader + + +def train_discriminator( + dataset, dataset_fp=None, pretrained_model="gpt2-medium", + epochs=10, batch_size=64, log_interval=10, + save_model=False, cached=False, no_cuda=False): + device = "cuda" if torch.cuda.is_available() and not no_cuda else "cpu" + + print("Preprocessing {} dataset...".format(dataset)) + start = time.time() + + if dataset == "SST": + idx2class = ["positive", "negative", "very positive", "very negative", + "neutral"] + class2idx = {c: i for i, c in enumerate(idx2class)} + + discriminator = Discriminator( + class_size=len(idx2class), + pretrained_model=pretrained_model, + cached_mode=cached, + device=device + ).to(device) + + text = torchtext_data.Field() + label = torchtext_data.Field(sequential=False) + train_data, val_data, test_data = datasets.SST.splits( + text, + label, + fine_grained=True, + train_subtrees=True, + ) + + x = [] + y = [] + for i in trange(len(train_data), ascii=True): + seq = TreebankWordDetokenizer().detokenize( + vars(train_data[i])["text"] + ) + seq = discriminator.tokenizer.encode(seq) + seq = torch.tensor([50256] + seq, device=device, dtype=torch.long) + x.append(seq) + y.append(class2idx[vars(train_data[i])["label"]]) + train_dataset = Dataset(x, y) + + test_x = [] + test_y = [] + for i in trange(len(test_data), ascii=True): + seq = TreebankWordDetokenizer().detokenize( + vars(test_data[i])["text"] + ) + seq = discriminator.tokenizer.encode(seq) + seq = torch.tensor([50256] + seq, device=device, dtype=torch.long) + test_x.append(seq) + test_y.append(class2idx[vars(test_data[i])["label"]]) + test_dataset = Dataset(test_x, test_y) + + discriminator_meta = { + "class_size": len(idx2class), + "embed_size": discriminator.embed_size, + "pretrained_model": pretrained_model, + "class_vocab": class2idx, + "default_class": 2, + } + + elif dataset == "clickbait": + idx2class = ["non_clickbait", "clickbait"] + class2idx = {c: i for i, c in enumerate(idx2class)} + + discriminator = Discriminator( + class_size=len(idx2class), + pretrained_model=pretrained_model, + cached_mode=cached, + device=device + ).to(device) + + with open("datasets/clickbait/clickbait_train_prefix.txt") as f: + data = [] + for i, line in enumerate(f): + try: + data.append(eval(line)) + except: + print("Error evaluating line {}: {}".format( + i, line + )) + continue + x = [] + y = [] + with open("datasets/clickbait/clickbait_train_prefix.txt") as f: + for i, line in enumerate(tqdm(f, ascii=True)): + try: + d = eval(line) + seq = discriminator.tokenizer.encode(d["text"]) + + if len(seq) < max_length_seq: + seq = torch.tensor( + [50256] + seq, device=device, dtype=torch.long + ) + else: + print("Line {} is longer than maximum length {}".format( + i, max_length_seq + )) + continue + x.append(seq) + y.append(d["label"]) + except: + print("Error evaluating / tokenizing" + " line {}, skipping it".format(i)) + pass + + full_dataset = Dataset(x, y) + train_size = int(0.9 * len(full_dataset)) + test_size = len(full_dataset) - train_size + train_dataset, test_dataset = torch.utils.data.random_split( + full_dataset, [train_size, test_size] + ) + + discriminator_meta = { + "class_size": len(idx2class), + "embed_size": discriminator.embed_size, + "pretrained_model": pretrained_model, + "class_vocab": class2idx, + "default_class": 1, + } + + elif dataset == "toxic": + idx2class = ["non_toxic", "toxic"] + class2idx = {c: i for i, c in enumerate(idx2class)} + + discriminator = Discriminator( + class_size=len(idx2class), + pretrained_model=pretrained_model, + cached_mode=cached, + device=device + ).to(device) + + x = [] + y = [] + with open("datasets/toxic/toxic_train.txt") as f: + for i, line in enumerate(tqdm(f, ascii=True)): + try: + d = eval(line) + seq = discriminator.tokenizer.encode(d["text"]) + + if len(seq) < max_length_seq: + seq = torch.tensor( + [50256] + seq, device=device, dtype=torch.long + ) + else: + print("Line {} is longer than maximum length {}".format( + i, max_length_seq + )) + continue + x.append(seq) + y.append(int(np.sum(d["label"]) > 0)) + except: + print("Error evaluating / tokenizing" + " line {}, skipping it".format(i)) + pass + + full_dataset = Dataset(x, y) + train_size = int(0.9 * len(full_dataset)) + test_size = len(full_dataset) - train_size + train_dataset, test_dataset = torch.utils.data.random_split( + full_dataset, [train_size, test_size] + ) + + discriminator_meta = { + "class_size": len(idx2class), + "embed_size": discriminator.embed_size, + "pretrained_model": pretrained_model, + "class_vocab": class2idx, + "default_class": 0, + } + + else: # if dataset == "generic": + # This assumes the input dataset is a TSV with the following structure: + # class \t text + + if dataset_fp is None: + raise ValueError("When generic dataset is selected, " + "dataset_fp needs to be specified aswell.") + + classes = set() + with open(dataset_fp) as f: + csv_reader = csv.reader(f, delimiter="\t") + for row in tqdm(csv_reader, ascii=True): + if row: + classes.add(row[0]) + + idx2class = sorted(classes) + class2idx = {c: i for i, c in enumerate(idx2class)} + + discriminator = Discriminator( + class_size=len(idx2class), + pretrained_model=pretrained_model, + cached_mode=cached, + device=device + ).to(device) + + x = [] + y = [] + with open(dataset_fp) as f: + csv_reader = csv.reader(f, delimiter="\t") + for i, row in enumerate(tqdm(csv_reader, ascii=True)): + if row: + label = row[0] + text = row[1] + + try: + seq = discriminator.tokenizer.encode(text) + if (len(seq) < max_length_seq): + seq = torch.tensor( + [50256] + seq, + device=device, + dtype=torch.long + ) + + else: + print( + "Line {} is longer than maximum length {}".format( + i, max_length_seq + )) + continue + + x.append(seq) + y.append(class2idx[label]) + + except: + print("Error tokenizing line {}, skipping it".format(i)) + pass + + full_dataset = Dataset(x, y) + train_size = int(0.9 * len(full_dataset)) + test_size = len(full_dataset) - train_size + train_dataset, test_dataset = torch.utils.data.random_split( + full_dataset, + [train_size, test_size] + ) + + discriminator_meta = { + "class_size": len(idx2class), + "embed_size": discriminator.embed_size, + "pretrained_model": pretrained_model, + "class_vocab": class2idx, + "default_class": 0, + } + + end = time.time() + print("Preprocessed {} data points".format( + len(train_dataset) + len(test_dataset)) + ) + print("Data preprocessing took: {:.3f}s".format(end - start)) + + if cached: + print("Building representation cache...") + + start = time.time() + + train_loader = get_cached_data_loader( + train_dataset, batch_size, discriminator, + shuffle=True, device=device + ) + + test_loader = get_cached_data_loader( + test_dataset, batch_size, discriminator, device=device + ) + + end = time.time() + print("Building representation cache took: {:.3f}s".format(end - start)) + + else: + train_loader = torch.utils.data.DataLoader(dataset=train_dataset, + batch_size=batch_size, + shuffle=True, + collate_fn=collate_fn) + test_loader = torch.utils.data.DataLoader(dataset=test_dataset, + batch_size=batch_size, + collate_fn=collate_fn) + + if save_model: + with open("{}_classifier_head_meta.json".format(dataset), + "w") as meta_file: + json.dump(discriminator_meta, meta_file) + + optimizer = optim.Adam(discriminator.parameters(), lr=0.0001) + + for epoch in range(epochs): + start = time.time() + print("\nEpoch", epoch + 1) + + train_epoch( + discriminator=discriminator, + data_loader=train_loader, + optimizer=optimizer, + epoch=epoch, + log_interval=log_interval, + device=device + ) + evaluate_performance( + data_loader=test_loader, + discriminator=discriminator, + device=device + ) + + end = time.time() + print("Epoch took: {:.3f}s".format(end - start)) + + print("\nExample prediction") + predict(example_sentence, discriminator, idx2class, + cached=cached, device=device) + + if save_model: + # torch.save(discriminator.state_dict(), + # "{}_discriminator_{}.pt".format( + # args.dataset, epoch + 1 + # )) + torch.save(discriminator.get_classifier().state_dict(), + "{}_classifier_head_epoch_{}.pt".format(dataset, + epoch + 1)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Train a discriminator on top of GPT-2 representations") + parser.add_argument("--dataset", type=str, default="SST", + choices=("SST", "clickbait", "toxic", "generic"), + help="dataset to train the discriminator on." + "In case of generic, the dataset is expected" + "to be a TSBV file with structure: class \\t text") + parser.add_argument("--dataset_fp", type=str, default="", + help="File path of the dataset to use. " + "Needed only in case of generic datadset") + parser.add_argument("--pretrained_model", type=str, default="gpt2-medium", + help="Pretrained model to use as encoder") + parser.add_argument("--epochs", type=int, default=10, metavar="N", + help="Number of training epochs") + parser.add_argument("--batch_size", type=int, default=64, metavar="N", + help="input batch size for training (default: 64)") + parser.add_argument("--log_interval", type=int, default=10, metavar="N", + help="how many batches to wait before logging training status") + parser.add_argument("--save_model", action="store_true", + help="whether to save the model") + parser.add_argument("--cached", action="store_true", + help="whether to cache the input representations") + parser.add_argument("--no_cuda", action="store_true", + help="use to turn off cuda") + args = parser.parse_args() + + train_discriminator(**(vars(args))) diff --git a/examples/run_generation.py b/examples/run_generation.py index 2d917660..fa52905b 100644 --- a/examples/run_generation.py +++ b/examples/run_generation.py @@ -247,7 +247,11 @@ def main(): out = out[:, len(context_tokens):].tolist() for o in out: text = tokenizer.decode(o, clean_up_tokenization_spaces=True) - text = text[: text.find(args.stop_token) if args.stop_token else None] + if args.stop_token: + index = text.find(args.stop_token) + if index == -1: + index = None + text = text[:index] print(text) diff --git a/examples/run_glue.py b/examples/run_glue.py index 19316cb0..954a8fbf 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -22,6 +22,7 @@ import glob import logging import os import random +import json import numpy as np import torch @@ -47,7 +48,14 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForSequenceClassification, - DistilBertTokenizer) + DistilBertTokenizer, + AlbertConfig, + AlbertForSequenceClassification, + AlbertTokenizer, + XLMRobertaConfig, + XLMRobertaForSequenceClassification, + XLMRobertaTokenizer, + ) from transformers import AdamW, get_linear_schedule_with_warmup @@ -66,7 +74,9 @@ MODEL_CLASSES = { 'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), 'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer), + 'albert': (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer), + 'xlmroberta': (XLMRobertaConfig, XLMRobertaForSequenceClassification, XLMRobertaTokenizer), } @@ -99,6 +109,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: @@ -158,7 +169,7 @@ def train(args, train_dataset, model, tokenizer): loss.backward() tr_loss += loss.item() - if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) else: @@ -170,15 +181,23 @@ def train(args, train_dataset, model, tokenizer): global_step += 1 if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: - # Log metrics + logs = {} if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well results = evaluate(args, model, tokenizer) for key, value in results.items(): - tb_writer.add_scalar('eval_{}'.format(key), value, global_step) - tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step) - tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) + eval_key = 'eval_{}'.format(key) + logs[eval_key] = value + + loss_scalar = (tr_loss - logging_loss) / args.logging_steps + learning_rate_scalar = scheduler.get_lr()[0] + logs['learning_rate'] = learning_rate_scalar + logs['loss'] = loss_scalar logging_loss = tr_loss + for key, value in logs.items(): + tb_writer.add_scalar(key, value, global_step) + print(json.dumps({**logs, **{'step': global_step}})) + if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: # Save model checkpoint output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) @@ -189,11 +208,6 @@ def train(args, train_dataset, model, tokenizer): torch.save(args, os.path.join(output_dir, 'training_args.bin')) logger.info("Saving model checkpoint to %s", output_dir) - if args.tpu: - args.xla_model.optimizer_step(optimizer, barrier=True) - model.zero_grad() - global_step += 1 - if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() break @@ -221,7 +235,7 @@ def evaluate(args, model, tokenizer, prefix=""): args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu eval @@ -294,9 +308,9 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): else: logger.info("Creating features from dataset file at %s", args.data_dir) label_list = processor.get_labels() - if task in ['mnli', 'mnli-mm'] and args.model_type in ['roberta']: + if task in ['mnli', 'mnli-mm'] and args.model_type in ['roberta', 'xlmroberta']: # HACK(label indices are swapped in RoBERTa pretrained model) - label_list[1], label_list[2] = label_list[2], label_list[1] + label_list[1], label_list[2] = label_list[2], label_list[1] examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) features = convert_examples_to_features(examples, tokenizer, @@ -322,7 +336,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): all_labels = torch.tensor([f.label for f in features], dtype=torch.long) elif output_mode == "regression": all_labels = torch.tensor([f.label for f in features], dtype=torch.float) - + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) return dataset @@ -366,11 +380,11 @@ def main(): parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for evaluation.") parser.add_argument('--gradient_accumulation_steps', type=int, default=1, - help="Number of updates steps to accumulate before performing a backward/update pass.") + help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.0, type=float, - help="Weight deay if we apply some.") + help="Weight decay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, @@ -397,15 +411,6 @@ def main(): parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") - parser.add_argument('--tpu', action='store_true', - help="Whether to run on the TPU defined in the environment variables") - parser.add_argument('--tpu_ip_address', type=str, default='', - help="TPU IP address if none are set in the environment variables") - parser.add_argument('--tpu_name', type=str, default='', - help="TPU name if none are set in the environment variables") - parser.add_argument('--xrt_tpu_config', type=str, default='', - help="XRT TPU config if none are set in the environment variables") - parser.add_argument('--fp16', action='store_true', help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") parser.add_argument('--fp16_opt_level', type=str, default='O1', @@ -439,23 +444,6 @@ def main(): args.n_gpu = 1 args.device = device - if args.tpu: - if args.tpu_ip_address: - os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address - if args.tpu_name: - os.environ["TPU_NAME"] = args.tpu_name - if args.xrt_tpu_config: - os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config - - assert "TPU_IP_ADDRESS" in os.environ - assert "TPU_NAME" in os.environ - assert "XRT_TPU_CONFIG" in os.environ - - import torch_xla - import torch_xla.core.xla_model as xm - args.device = xm.xla_device() - args.xla_model = xm - # Setup logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', @@ -509,7 +497,7 @@ def main(): # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() - if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # Create output directory if needed if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.makedirs(args.output_dir) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 52a1b75a..75848d5a 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -47,7 +47,8 @@ from transformers import (WEIGHTS_NAME, AdamW, get_linear_schedule_with_warmup, GPT2Config, GPT2LMHeadModel, GPT2Tokenizer, OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer, RobertaConfig, RobertaForMaskedLM, RobertaTokenizer, - DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer) + DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer, + CamembertConfig, CamembertForMaskedLM, CamembertTokenizer) logger = logging.getLogger(__name__) @@ -58,7 +59,8 @@ MODEL_CLASSES = { 'openai-gpt': (OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer), 'bert': (BertConfig, BertForMaskedLM, BertTokenizer), 'roberta': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer), + 'camembert': (CamembertConfig, CamembertForMaskedLM, CamembertTokenizer) } @@ -68,7 +70,7 @@ class TextDataset(Dataset): directory, filename = os.path.split(file_path) cached_features_file = os.path.join(directory, args.model_name_or_path + '_cached_lm_' + str(block_size) + '_' + filename) - if os.path.exists(cached_features_file): + if os.path.exists(cached_features_file) and not args.overwrite_cache: logger.info("Loading features from cached file %s", cached_features_file) with open(cached_features_file, 'rb') as handle: self.examples = pickle.load(handle) @@ -186,6 +188,13 @@ def train(args, train_dataset, model, tokenizer): ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) + + # Check if saved optimizer or scheduler states exist + if os.path.isfile(os.path.join(args.model_name_or_path, 'optimizer.pt')) and os.path.isfile(os.path.join(args.model_name_or_path, 'scheduler.pt')): + # Load in optimizer and scheduler states + optimizer.load_state_dict(torch.load(os.path.join(args.model_name_or_path, 'optimizer.pt'))) + scheduler.load_state_dict(torch.load(os.path.join(args.model_name_or_path, 'scheduler.pt'))) + if args.fp16: try: from apex import amp @@ -214,13 +223,37 @@ def train(args, train_dataset, model, tokenizer): logger.info(" Total optimization steps = %d", t_total) global_step = 0 + epochs_trained = 0 + steps_trained_in_current_epoch = 0 + # Check if continuing training from a checkpoint + if os.path.exists(args.model_name_or_path): + # set global_step to gobal_step of last saved checkpoint from model path + global_step = int(args.model_name_or_path.split('-')[-1].split('/')[0]) + epochs_trained = global_step // (len(train_dataloader) // args.gradient_accumulation_steps) + steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps) + + logger.info(" Continuing training from checkpoint, will skip to saved global_step") + logger.info(" Continuing training from epoch %d", epochs_trained) + logger.info(" Continuing training from global step %d", global_step) + logger.info(" Will skip the first %d steps in the first epoch", steps_trained_in_current_epoch) + tr_loss, logging_loss = 0.0, 0.0 + + model_to_resize = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_resize.resize_token_embeddings(len(tokenizer)) + model.zero_grad() - train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) + train_iterator = trange(epochs_trained, int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproducibility (even between python 2 and 3) for _ in train_iterator: epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) for step, batch in enumerate(epoch_iterator): + + # Skip past any already trained steps if resuming training + if steps_trained_in_current_epoch > 0: + steps_trained_in_current_epoch -= 1 + continue + inputs, labels = mask_tokens(batch, tokenizer, args) if args.mlm else (batch, batch) inputs = inputs.to(args.device) labels = labels.to(args.device) @@ -268,11 +301,17 @@ def train(args, train_dataset, model, tokenizer): os.makedirs(output_dir) model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training model_to_save.save_pretrained(output_dir) + tokenizer.save_pretrained(output_dir) + torch.save(args, os.path.join(output_dir, 'training_args.bin')) logger.info("Saving model checkpoint to %s", output_dir) _rotate_checkpoints(args, checkpoint_prefix) + torch.save(optimizer.state_dict(), os.path.join(output_dir, 'optimizer.pt')) + torch.save(scheduler.state_dict(), os.path.join(output_dir, 'scheduler.pt')) + logger.info("Saving optimizer and scheduler states to %s", output_dir) + if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() break @@ -297,7 +336,7 @@ def evaluate(args, model, tokenizer, prefix=""): args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu evaluate @@ -391,7 +430,7 @@ def main(): parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.0, type=float, - help="Weight deay if we apply some.") + help="Weight decay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, @@ -431,7 +470,7 @@ def main(): parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") args = parser.parse_args() - if args.model_type in ["bert", "roberta", "distilbert"] and not args.mlm: + if args.model_type in ["bert", "roberta", "distilbert", "camembert"] and not args.mlm: raise ValueError("BERT and RoBERTa do not have LM heads but masked LM heads. They must be run using the --mlm " "flag (masked language modeling).") if args.eval_data_file is None and args.do_eval: diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py index 30c33329..9d1ca7f3 100644 --- a/examples/run_multiple_choice.py +++ b/examples/run_multiple_choice.py @@ -226,7 +226,7 @@ def evaluate(args, model, tokenizer, prefix="", test=False): args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu evaluate diff --git a/examples/run_ner.py b/examples/run_ner.py index 4359e587..6426a6d1 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -37,17 +37,22 @@ from transformers import AdamW, get_linear_schedule_with_warmup from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer from transformers import DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer +from transformers import CamembertConfig, CamembertForTokenClassification, CamembertTokenizer +from transformers import XLMRobertaConfig, XLMRobertaForTokenClassification, XLMRobertaTokenizer logger = logging.getLogger(__name__) ALL_MODELS = sum( - (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)), + (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig, + CamembertConfig, XLMRobertaConfig)), ()) MODEL_CLASSES = { "bert": (BertConfig, BertForTokenClassification, BertTokenizer), "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer), - "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer) + "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer), + "camembert": (CamembertConfig, CamembertForTokenClassification, CamembertTokenizer), + "xlmroberta": (XLMRobertaConfig, XLMRobertaForTokenClassification, XLMRobertaTokenizer), } @@ -125,7 +130,7 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id): "attention_mask": batch[1], "labels": batch[3]} if args.model_type != "distilbert": - inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids + inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids outputs = model(**inputs) loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) @@ -215,7 +220,7 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix="" "attention_mask": batch[1], "labels": batch[3]} if args.model_type != "distilbert": - inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids + inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] diff --git a/examples/run_squad.py b/examples/run_squad.py index bd44979e..26983a2b 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -16,6 +16,8 @@ """ Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet).""" from __future__ import absolute_import, division, print_function +from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor, SquadResult +from transformers.data.metrics.squad_metrics import compute_predictions_logits, compute_predictions_log_probs, squad_evaluate import argparse import logging @@ -23,11 +25,9 @@ import os import random import glob import timeit - import numpy as np import torch -from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, - TensorDataset) +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset) from torch.utils.data.distributed import DistributedSampler try: @@ -43,18 +43,12 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLMTokenizer, XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer, - DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) + DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer, + AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer, + XLMConfig, XLMForQuestionAnswering, XLMTokenizer, + ) -from transformers import AdamW, get_linear_schedule_with_warmup - -from utils_squad import (read_squad_examples, convert_examples_to_features, - RawResult, write_predictions, - RawResultExtended, write_predictions_extended) - -# The follwing import is the official SQuAD evaluation script (2.0). -# You can remove it from the dependencies if you are using this script outside of the library -# We've added it here for automated tests (see examples/test_examples.py file) -from utils_squad_evaluate import EVAL_OPTS, main as evaluate_on_squad +from transformers import AdamW, get_linear_schedule_with_warmup, squad_convert_examples_to_features logger = logging.getLogger(__name__) @@ -65,7 +59,8 @@ MODEL_CLASSES = { 'bert': (BertConfig, BertForQuestionAnswering, BertTokenizer), 'xlnet': (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer), 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer), + 'albert': (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer), } def set_seed(args): @@ -98,14 +93,16 @@ def train(args, train_dataset, model, tokenizer): optimizer_grouped_parameters = [ {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} - ] + ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) + if args.fp16: try: from apex import amp except ImportError: raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) # multi-gpu training (should be after apex fp16 initialization) @@ -128,22 +125,28 @@ def train(args, train_dataset, model, tokenizer): logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) logger.info(" Total optimization steps = %d", t_total) - global_step = 0 + global_step = 1 tr_loss, logging_loss = 0.0, 0.0 model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproductibility (even between python 2 and 3) + for _ in train_iterator: epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) for step, batch in enumerate(epoch_iterator): model.train() batch = tuple(t.to(args.device) for t in batch) - inputs = {'input_ids': batch[0], - 'attention_mask': batch[1], - 'start_positions': batch[3], - 'end_positions': batch[4]} + + inputs = { + 'input_ids': batch[0], + 'attention_mask': batch[1], + 'start_positions': batch[3], + 'end_positions': batch[4] + } + if args.model_type != 'distilbert': inputs['token_type_ids'] = None if args.model_type == 'xlm' else batch[2] + if args.model_type in ['xlnet', 'xlm']: inputs.update({'cls_index': batch[5], 'p_mask': batch[6]}) @@ -175,8 +178,8 @@ def train(args, train_dataset, model, tokenizer): model.zero_grad() global_step += 1 + # Log metrics if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: - # Log metrics if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well results = evaluate(args, model, tokenizer) for key, value in results.items(): @@ -185,8 +188,8 @@ def train(args, train_dataset, model, tokenizer): tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) logging_loss = tr_loss + # Save model checkpoint if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: - # Save model checkpoint output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) if not os.path.exists(output_dir): os.makedirs(output_dir) @@ -215,50 +218,72 @@ def evaluate(args, model, tokenizer, prefix=""): os.makedirs(args.output_dir) args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) + # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset) + eval_sampler = SequentialSampler(dataset) eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu evaluate - if args.n_gpu > 1: + if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): model = torch.nn.DataParallel(model) # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(dataset)) logger.info(" Batch size = %d", args.eval_batch_size) + all_results = [] start_time = timeit.default_timer() + for batch in tqdm(eval_dataloader, desc="Evaluating"): model.eval() batch = tuple(t.to(args.device) for t in batch) + with torch.no_grad(): - inputs = {'input_ids': batch[0], - 'attention_mask': batch[1] - } + inputs = { + 'input_ids': batch[0], + 'attention_mask': batch[1] + } + if args.model_type != 'distilbert': inputs['token_type_ids'] = None if args.model_type == 'xlm' else batch[2] # XLM don't use segment_ids + example_indices = batch[3] + + # XLNet and XLM use more arguments for their predictions if args.model_type in ['xlnet', 'xlm']: - inputs.update({'cls_index': batch[4], - 'p_mask': batch[5]}) + inputs.update({'cls_index': batch[4], 'p_mask': batch[5]}) + outputs = model(**inputs) for i, example_index in enumerate(example_indices): eval_feature = features[example_index.item()] unique_id = int(eval_feature.unique_id) - if args.model_type in ['xlnet', 'xlm']: - # XLNet uses a more complex post-processing procedure - result = RawResultExtended(unique_id = unique_id, - start_top_log_probs = to_list(outputs[0][i]), - start_top_index = to_list(outputs[1][i]), - end_top_log_probs = to_list(outputs[2][i]), - end_top_index = to_list(outputs[3][i]), - cls_logits = to_list(outputs[4][i])) + + output = [to_list(output[i]) for output in outputs] + + # Some models (XLNet, XLM) use 5 arguments for their predictions, while the other "simpler" + # models only use two. + if len(output) >= 5: + start_logits = output[0] + start_top_index = output[1] + end_logits = output[2] + end_top_index = output[3] + cls_logits = output[4] + + result = SquadResult( + unique_id, start_logits, end_logits, + start_top_index=start_top_index, + end_top_index=end_top_index, + cls_logits=cls_logits + ) + else: - result = RawResult(unique_id = unique_id, - start_logits = to_list(outputs[0][i]), - end_logits = to_list(outputs[1][i])) + start_logits, end_logits = output + result = SquadResult( + unique_id, start_logits, end_logits + ) + all_results.append(result) evalTime = timeit.default_timer() - start_time @@ -267,63 +292,84 @@ def evaluate(args, model, tokenizer, prefix=""): # Compute predictions output_prediction_file = os.path.join(args.output_dir, "predictions_{}.json".format(prefix)) output_nbest_file = os.path.join(args.output_dir, "nbest_predictions_{}.json".format(prefix)) + if args.version_2_with_negative: output_null_log_odds_file = os.path.join(args.output_dir, "null_odds_{}.json".format(prefix)) else: output_null_log_odds_file = None + # XLNet and XLM use a more complex post-processing procedure if args.model_type in ['xlnet', 'xlm']: - # XLNet uses a more complex post-processing procedure - write_predictions_extended(examples, features, all_results, args.n_best_size, + start_n_top = model.config.start_n_top if hasattr(model, "config") else model.module.config.start_n_top + end_n_top = model.config.end_n_top if hasattr(model, "config") else model.module.config.end_n_top + + predictions = compute_predictions_log_probs(examples, features, all_results, args.n_best_size, args.max_answer_length, output_prediction_file, - output_nbest_file, output_null_log_odds_file, args.predict_file, - model.config.start_n_top, model.config.end_n_top, + output_nbest_file, output_null_log_odds_file, + start_n_top, end_n_top, args.version_2_with_negative, tokenizer, args.verbose_logging) else: - write_predictions(examples, features, all_results, args.n_best_size, + predictions = compute_predictions_logits(examples, features, all_results, args.n_best_size, args.max_answer_length, args.do_lower_case, output_prediction_file, output_nbest_file, output_null_log_odds_file, args.verbose_logging, args.version_2_with_negative, args.null_score_diff_threshold) - # Evaluate with the official SQuAD script - evaluate_options = EVAL_OPTS(data_file=args.predict_file, - pred_file=output_prediction_file, - na_prob_file=output_null_log_odds_file) - results = evaluate_on_squad(evaluate_options) + # Compute the F1 and exact scores. + results = squad_evaluate(examples, predictions) return results - def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=False): if args.local_rank not in [-1, 0] and not evaluate: torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache # Load data features from cache or dataset file - input_file = args.predict_file if evaluate else args.train_file - cached_features_file = os.path.join(os.path.dirname(input_file), 'cached_{}_{}_{}'.format( + input_dir = args.data_dir if args.data_dir else "." + cached_features_file = os.path.join(input_dir, 'cached_{}_{}_{}'.format( 'dev' if evaluate else 'train', list(filter(None, args.model_name_or_path.split('/'))).pop(), - str(args.max_seq_length))) + str(args.max_seq_length)) + ) + + # Init features and dataset from cache if it exists if os.path.exists(cached_features_file) and not args.overwrite_cache and not output_examples: logger.info("Loading features from cached file %s", cached_features_file) - features = torch.load(cached_features_file) + features_and_dataset = torch.load(cached_features_file) + features, dataset = features_and_dataset["features"], features_and_dataset["dataset"] else: - logger.info("Creating features from dataset file at %s", input_file) - examples = read_squad_examples(input_file=input_file, - is_training=not evaluate, - version_2_with_negative=args.version_2_with_negative) - features = convert_examples_to_features(examples=examples, - tokenizer=tokenizer, - max_seq_length=args.max_seq_length, - doc_stride=args.doc_stride, - max_query_length=args.max_query_length, - is_training=not evaluate, - cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0, - pad_token_segment_id=3 if args.model_type in ['xlnet'] else 0, - cls_token_at_end=True if args.model_type in ['xlnet'] else False, - sequence_a_is_doc=True if args.model_type in ['xlnet'] else False) + logger.info("Creating features from dataset file at %s", input_dir) + + if not args.data_dir and ((evaluate and not args.predict_file) or (not evaluate and not args.train_file)): + try: + import tensorflow_datasets as tfds + except ImportError: + raise ImportError("If not data_dir is specified, tensorflow_datasets needs to be installed.") + + if args.version_2_with_negative: + logger.warn("tensorflow_datasets does not handle version 2 of SQuAD.") + + tfds_examples = tfds.load("squad") + examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate) + else: + processor = SquadV2Processor() if args.version_2_with_negative else SquadV1Processor() + + if evaluate: + examples = processor.get_dev_examples(args.data_dir, filename=args.predict_file) + else: + examples = processor.get_train_examples(args.data_dir, filename=args.train_file) + + features, dataset = squad_convert_examples_to_features( + examples=examples, + tokenizer=tokenizer, + max_seq_length=args.max_seq_length, + doc_stride=args.doc_stride, + max_query_length=args.max_query_length, + is_training=not evaluate, + return_dataset='pt' + ) + if args.local_rank in [-1, 0]: logger.info("Saving features into cached file %s", cached_features_file) - torch.save(features, cached_features_file) + torch.save({"features": features, "dataset": dataset}, cached_features_file) if args.local_rank == 0 and not evaluate: torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache @@ -355,10 +401,6 @@ def main(): parser = argparse.ArgumentParser() ## Required parameters - parser.add_argument("--train_file", default=None, type=str, required=True, - help="SQuAD json for training. E.g., train-v1.1.json") - parser.add_argument("--predict_file", default=None, type=str, required=True, - help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json") parser.add_argument("--model_type", default=None, type=str, required=True, help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) parser.add_argument("--model_name_or_path", default=None, type=str, required=True, @@ -367,6 +409,15 @@ def main(): help="The output directory where the model checkpoints and predictions will be written.") ## Other parameters + parser.add_argument("--data_dir", default=None, type=str, + help="The input data dir. Should contain the .json files for the task." + + "If no data dir or train/predict files are specified, will run with tensorflow_datasets.") + parser.add_argument("--train_file", default=None, type=str, + help="The input training file. If a data dir is specified, will look for the file there" + + "If no data dir or train/predict files are specified, will run with tensorflow_datasets.") + parser.add_argument("--predict_file", default=None, type=str, + help="The input evaluation file. If a data dir is specified, will look for the file there" + + "If no data dir or train/predict files are specified, will run with tensorflow_datasets.") parser.add_argument("--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name") parser.add_argument("--tokenizer_name", default="", type=str, @@ -405,7 +456,7 @@ def main(): parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--weight_decay", default=0.0, type=float, - help="Weight deay if we apply some.") + help="Weight decay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, @@ -540,7 +591,7 @@ def main(): torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) # Load a trained model and vocabulary that you have fine-tuned - model = model_class.from_pretrained(args.output_dir) + model = model_class.from_pretrained(args.output_dir, force_download=True) tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) model.to(args.device) @@ -548,17 +599,23 @@ def main(): # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory results = {} if args.do_eval and args.local_rank in [-1, 0]: - checkpoints = [args.output_dir] - if args.eval_all_checkpoints: - checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) - logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs + + if args.do_train: + logger.info("Loading checkpoints saved during training for evaluation") + checkpoints = [args.output_dir] + if args.eval_all_checkpoints: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs + else: + logger.info("Loading checkpoint %s for evaluation", args.model_name_or_path) + checkpoints = [args.model_name_or_path] logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: # Reload the model global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" - model = model_class.from_pretrained(checkpoint) + model = model_class.from_pretrained(checkpoint, force_download=True) model.to(args.device) # Evaluate diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py deleted file mode 100644 index f5604c26..00000000 --- a/examples/run_summarization_finetuning.py +++ /dev/null @@ -1,492 +0,0 @@ -# coding=utf-8 -# Copyright 2019 The HuggingFace Inc. team. -# Copyright (c) 2019 The HuggingFace Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Finetuning seq2seq models for sequence generation.""" - -import argparse -import functools -import logging -import os -import random -import sys - -import numpy as np -from tqdm import tqdm, trange -import torch -from torch.optim import Adam -from torch.utils.data import DataLoader, RandomSampler, SequentialSampler - -from transformers import ( - AutoTokenizer, - BertForMaskedLM, - BertConfig, - PreTrainedEncoderDecoder, - Model2Model, -) - -from utils_summarization import ( - CNNDailyMailDataset, - encode_for_summarization, - fit_to_block_size, - build_lm_labels, - build_mask, - compute_token_type_ids, -) - -logger = logging.getLogger(__name__) -logging.basicConfig(stream=sys.stdout, level=logging.INFO) - - -def set_seed(args): - random.seed(args.seed) - np.random.seed(args.seed) - torch.manual_seed(args.seed) - - -# ------------ -# Load dataset -# ------------ - - -def load_and_cache_examples(args, tokenizer): - dataset = CNNDailyMailDataset(tokenizer, data_dir=args.data_dir) - return dataset - - -def collate(data, tokenizer, block_size): - """ List of tuple as an input. """ - # remove the files with empty an story/summary, encode and fit to block - data = filter(lambda x: not (len(x[0]) == 0 or len(x[1]) == 0), data) - data = [ - encode_for_summarization(story, summary, tokenizer) for story, summary in data - ] - data = [ - ( - fit_to_block_size(story, block_size, tokenizer.pad_token_id), - fit_to_block_size(summary, block_size, tokenizer.pad_token_id), - ) - for story, summary in data - ] - - stories = torch.tensor([story for story, summary in data]) - summaries = torch.tensor([summary for story, summary in data]) - encoder_token_type_ids = compute_token_type_ids(stories, tokenizer.cls_token_id) - encoder_mask = build_mask(stories, tokenizer.pad_token_id) - decoder_mask = build_mask(summaries, tokenizer.pad_token_id) - lm_labels = build_lm_labels(summaries, tokenizer.pad_token_id) - - return ( - stories, - summaries, - encoder_token_type_ids, - encoder_mask, - decoder_mask, - lm_labels, - ) - - -# ---------- -# Optimizers -# ---------- - - -class BertSumOptimizer(object): - """ Specific optimizer for BertSum. - - As described in [1], the authors fine-tune BertSum for abstractive - summarization using two Adam Optimizers with different warm-up steps and - learning rate. They also use a custom learning rate scheduler. - - [1] Liu, Yang, and Mirella Lapata. "Text summarization with pretrained encoders." - arXiv preprint arXiv:1908.08345 (2019). - """ - - def __init__(self, model, lr, warmup_steps, beta_1=0.99, beta_2=0.999, eps=1e-8): - self.encoder = model.encoder - self.decoder = model.decoder - self.lr = lr - self.warmup_steps = warmup_steps - - self.optimizers = { - "encoder": Adam( - model.encoder.parameters(), - lr=lr["encoder"], - betas=(beta_1, beta_2), - eps=eps, - ), - "decoder": Adam( - model.decoder.parameters(), - lr=lr["decoder"], - betas=(beta_1, beta_2), - eps=eps, - ), - } - - self._step = 0 - - def _update_rate(self, stack): - return self.lr[stack] * min( - self._step ** (-0.5), self._step * self.warmup_steps[stack] ** (-0.5) - ) - - def zero_grad(self): - self.optimizer_decoder.zero_grad() - self.optimizer_encoder.zero_grad() - - def step(self): - self._step += 1 - for stack, optimizer in self.optimizers.items(): - new_rate = self._update_rate(stack) - for param_group in optimizer.param_groups: - param_group["lr"] = new_rate - optimizer.step() - - -# ------------ -# Train -# ------------ - - -def train(args, model, tokenizer): - """ Fine-tune the pretrained model on the corpus. """ - set_seed(args) - - # Load the data - args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) - train_dataset = load_and_cache_examples(args, tokenizer) - train_sampler = RandomSampler(train_dataset) - model_collate_fn = functools.partial(collate, tokenizer=tokenizer, block_size=512) - train_dataloader = DataLoader( - train_dataset, - sampler=train_sampler, - batch_size=args.train_batch_size, - collate_fn=model_collate_fn, - ) - - # Training schedule - if args.max_steps > 0: - t_total = args.max_steps - args.num_train_epochs = t_total // ( - len(train_dataloader) // args.gradient_accumulation_steps + 1 - ) - else: - t_total = ( - len(train_dataloader) - // args.gradient_accumulation_steps - * args.num_train_epochs - ) - - # Prepare the optimizer - lr = {"encoder": 0.002, "decoder": 0.2} - warmup_steps = {"encoder": 20000, "decoder": 10000} - optimizer = BertSumOptimizer(model, lr, warmup_steps) - - # Train - logger.info("***** Running training *****") - logger.info(" Num examples = %d", len(train_dataset)) - logger.info(" Num Epochs = %d", args.num_train_epochs) - logger.info( - " Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size - ) - logger.info( - " Total train batch size (w. parallel, distributed & accumulation) = %d", - args.train_batch_size * args.gradient_accumulation_steps - # * (torch.distributed.get_world_size() if args.local_rank != -1 else 1), - ) - logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) - logger.info(" Total optimization steps = %d", t_total) - - model.zero_grad() - train_iterator = trange(args.num_train_epochs, desc="Epoch", disable=True) - - global_step = 0 - tr_loss = 0.0 - for _ in train_iterator: - epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=True) - for step, batch in enumerate(epoch_iterator): - source, target, encoder_token_type_ids, encoder_mask, decoder_mask, lm_labels = batch - - source = source.to(args.device) - target = target.to(args.device) - encoder_token_type_ids = encoder_token_type_ids.to(args.device) - encoder_mask = encoder_mask.to(args.device) - decoder_mask = decoder_mask.to(args.device) - lm_labels = lm_labels.to(args.device) - - model.train() - outputs = model( - source, - target, - encoder_token_type_ids=encoder_token_type_ids, - encoder_attention_mask=encoder_mask, - decoder_attention_mask=decoder_mask, - decoder_lm_labels=lm_labels, - ) - - loss = outputs[0] - print(loss) - if args.gradient_accumulation_steps > 1: - loss /= args.gradient_accumulation_steps - - loss.backward() - - tr_loss += loss.item() - if (step + 1) % args.gradient_accumulation_steps == 0: - torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) - optimizer.step() - model.zero_grad() - global_step += 1 - - if args.max_steps > 0 and global_step > args.max_steps: - epoch_iterator.close() - break - - if args.max_steps > 0 and global_step > args.max_steps: - train_iterator.close() - break - - return global_step, tr_loss / global_step - - -# ------------ -# Train -# ------------ - - -def evaluate(args, model, tokenizer, prefix=""): - set_seed(args) - - args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) - eval_dataset = load_and_cache_examples(args, tokenizer, evaluate=True) - eval_sampler = SequentialSampler(eval_dataset) - eval_dataloader = DataLoader( - eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size - ) - - # multi-gpu evaluate - if args.n_gpu > 1: - model = torch.nn.DataParallel(model) - - logger.info("***** Running evaluation {} *****".format(prefix)) - logger.info(" Num examples = %d", len(eval_dataset)) - logger.info(" Batch size = %d", args.eval_batch_size) - eval_loss = 0.0 - nb_eval_steps = 0 - model.eval() - - for batch in tqdm(eval_dataloader, desc="Evaluating"): - source, target, encoder_token_type_ids, encoder_mask, decoder_mask, lm_labels = batch - - source = source.to(args.device) - target = target.to(args.device) - encoder_token_type_ids = encoder_token_type_ids.to(args.device) - encoder_mask = encoder_mask.to(args.device) - decoder_mask = decoder_mask.to(args.device) - lm_labels = lm_labels.to(args.device) - - with torch.no_grad(): - outputs = model( - source, - target, - encoder_token_type_ids=encoder_token_type_ids, - encoder_attention_mask=encoder_mask, - decoder_attention_mask=decoder_mask, - decoder_lm_labels=lm_labels, - ) - lm_loss = outputs[0] - eval_loss += lm_loss.mean().item() - nb_eval_steps += 1 - - eval_loss = eval_loss / nb_eval_steps - perplexity = torch.exp(torch.tensor(eval_loss)) - - result = {"perplexity": perplexity} - - # Save the evaluation's results - output_eval_file = os.path.join(args.output_dir, "eval_results.txt") - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - with open(output_eval_file, "w") as writer: - logger.info("***** Eval results {} *****".format(prefix)) - for key in sorted(result.keys()): - logger.info(" %s = %s", key, str(result[key])) - writer.write("%s = %s\n" % (key, str(result[key]))) - - return result - - -def main(): - parser = argparse.ArgumentParser() - - # Required parameters - parser.add_argument( - "--data_dir", - default=None, - type=str, - required=True, - help="The input training data file (a text file).", - ) - parser.add_argument( - "--output_dir", - default=None, - type=str, - required=True, - help="The output directory where the model predictions and checkpoints will be written.", - ) - - # Optional parameters - parser.add_argument( - "--gradient_accumulation_steps", - type=int, - default=1, - help="Number of updates steps to accumulate before performing a backward/update pass.", - ) - parser.add_argument( - "--do_evaluate", - type=bool, - default=False, - help="Run model evaluation on out-of-sample data.", - ) - parser.add_argument("--do_train", type=bool, default=False, help="Run training.") - parser.add_argument( - "--do_overwrite_output_dir", - type=bool, - default=False, - help="Whether to overwrite the output dir.", - ) - parser.add_argument( - "--model_name_or_path", - default="bert-base-cased", - type=str, - help="The model checkpoint to initialize the encoder and decoder's weights with.", - ) - parser.add_argument( - "--model_type", - default="bert", - type=str, - help="The decoder architecture to be fine-tuned.", - ) - parser.add_argument( - "--max_grad_norm", default=1.0, type=float, help="Max gradient norm." - ) - parser.add_argument( - "--max_steps", - default=-1, - type=int, - help="If > 0: set total number of training steps to perform. Override num_train_epochs.", - ) - parser.add_argument( - "--to_cpu", default=False, type=bool, help="Whether to force training on CPU." - ) - parser.add_argument( - "--num_train_epochs", - default=10, - type=int, - help="Total number of training epochs to perform.", - ) - parser.add_argument( - "--per_gpu_train_batch_size", - default=4, - type=int, - help="Batch size per GPU/CPU for training.", - ) - parser.add_argument("--seed", default=42, type=int) - args = parser.parse_args() - - if ( - os.path.exists(args.output_dir) - and os.listdir(args.output_dir) - and args.do_train - and not args.do_overwrite_output_dir - ): - raise ValueError( - "Output directory ({}) already exists and is not empty. Use --do_overwrite_output_dir to overwrite.".format( - args.output_dir - ) - ) - - # Set up training device - if args.to_cpu or not torch.cuda.is_available(): - args.device = torch.device("cpu") - args.n_gpu = 0 - else: - args.device = torch.device("cuda") - args.n_gpu = torch.cuda.device_count() - - # Load pretrained model and tokenizer. The decoder's weights are randomly initialized. - tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path) - config = BertConfig.from_pretrained(args.model_name_or_path) - decoder_model = BertForMaskedLM(config) - model = Model2Model.from_pretrained( - args.model_name_or_path, decoder_model=decoder_model - ) - - # Setup logging - logging.basicConfig( - format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", - datefmt="%m/%d/%Y %H:%M:%S", - level=logging.INFO, - ) - logger.warning( - "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", - 0, - args.device, - args.n_gpu, - False, - False, - ) - - logger.info("Training/evaluation parameters %s", args) - - # Train the model - model.to(args.device) - if args.do_train: - global_step, tr_loss = train(args, model, tokenizer) - logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) - - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - logger.info("Saving model checkpoint to %s", args.output_dir) - - # Save a trained model, configuration and tokenizer using `save_pretrained()`. - # They can then be reloaded using `from_pretrained()` - model_to_save = ( - model.module if hasattr(model, "module") else model - ) # Take care of distributed/parallel training - model_to_save.save_pretrained(args.output_dir) - tokenizer.save_pretrained(args.output_dir) - torch.save(args, os.path.join(args.output_dir, "training_arguments.bin")) - - # Evaluate the model - results = {} - if args.do_evaluate: - checkpoints = [] - logger.info("Evaluate the following checkpoints: %s", checkpoints) - for checkpoint in checkpoints: - encoder_checkpoint = os.path.join(checkpoint, "encoder") - decoder_checkpoint = os.path.join(checkpoint, "decoder") - model = PreTrainedEncoderDecoder.from_pretrained( - encoder_checkpoint, decoder_checkpoint - ) - model.to(args.device) - results = "placeholder" - - return results - - -if __name__ == "__main__": - main() diff --git a/examples/run_tf_ner.py b/examples/run_tf_ner.py new file mode 100644 index 00000000..eb284f4c --- /dev/null +++ b/examples/run_tf_ner.py @@ -0,0 +1,615 @@ +# coding=utf-8 +import datetime +import os +import math +import glob +import re +import tensorflow as tf +import collections +import numpy as np +from seqeval import metrics +import _pickle as pickle +from absl import logging +from transformers import TF2_WEIGHTS_NAME, BertConfig, BertTokenizer, TFBertForTokenClassification +from transformers import RobertaConfig, RobertaTokenizer, TFRobertaForTokenClassification +from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForTokenClassification +from transformers import create_optimizer, GradientAccumulator +from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file +from fastprogress import master_bar, progress_bar +from absl import flags +from absl import app + + +ALL_MODELS = sum( + (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)), + ()) + +MODEL_CLASSES = { + "bert": (BertConfig, TFBertForTokenClassification, BertTokenizer), + "roberta": (RobertaConfig, TFRobertaForTokenClassification, RobertaTokenizer), + "distilbert": (DistilBertConfig, TFDistilBertForTokenClassification, DistilBertTokenizer) +} + + +flags.DEFINE_string( + "data_dir", None, + "The input data dir. Should contain the .conll files (or other data files) " + "for the task.") + +flags.DEFINE_string( + "model_type", None, + "Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) + +flags.DEFINE_string( + "model_name_or_path", None, + "Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS)) + +flags.DEFINE_string( + "output_dir", None, + "The output directory where the model checkpoints will be written.") + +flags.DEFINE_string( + "labels", "", + "Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.") + +flags.DEFINE_string( + "config_name", "", + "Pretrained config name or path if not the same as model_name") + +flags.DEFINE_string( + "tokenizer_name", "", + "Pretrained tokenizer name or path if not the same as model_name") + +flags.DEFINE_string( + "cache_dir", "", + "Where do you want to store the pre-trained models downloaded from s3") + +flags.DEFINE_integer( + "max_seq_length", 128, + "The maximum total input sentence length after tokenization. " + "Sequences longer than this will be truncated, sequences shorter " + "will be padded.") + +flags.DEFINE_string( + "tpu", None, + "The Cloud TPU to use for training. This should be either the name " + "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " + "url.") + +flags.DEFINE_integer( + "num_tpu_cores", 8, + "Total number of TPU cores to use.") + +flags.DEFINE_boolean( + "do_train", False, + "Whether to run training.") + +flags.DEFINE_boolean( + "do_eval", False, + "Whether to run eval on the dev set.") + +flags.DEFINE_boolean( + "do_predict", False, + "Whether to run predictions on the test set.") + +flags.DEFINE_boolean( + "evaluate_during_training", False, + "Whether to run evaluation during training at each logging step.") + +flags.DEFINE_boolean( + "do_lower_case", False, + "Set this flag if you are using an uncased model.") + +flags.DEFINE_integer( + "per_device_train_batch_size", 8, + "Batch size per GPU/CPU/TPU for training.") + +flags.DEFINE_integer( + "per_device_eval_batch_size", 8, + "Batch size per GPU/CPU/TPU for evaluation.") + +flags.DEFINE_integer( + "gradient_accumulation_steps", 1, + "Number of updates steps to accumulate before performing a backward/update pass.") + +flags.DEFINE_float( + "learning_rate", 5e-5, + "The initial learning rate for Adam.") + +flags.DEFINE_float( + "weight_decay", 0.0, + "Weight decay if we apply some.") + +flags.DEFINE_float( + "adam_epsilon", 1e-8, + "Epsilon for Adam optimizer.") + +flags.DEFINE_float( + "max_grad_norm", 1.0, + "Max gradient norm.") + +flags.DEFINE_integer( + "num_train_epochs", 3, + "Total number of training epochs to perform.") + +flags.DEFINE_integer( + "max_steps", -1, + "If > 0: set total number of training steps to perform. Override num_train_epochs.") + +flags.DEFINE_integer( + "warmup_steps", 0, + "Linear warmup over warmup_steps.") + +flags.DEFINE_integer( + "logging_steps", 50, + "Log every X updates steps.") + +flags.DEFINE_integer( + "save_steps", 50, + "Save checkpoint every X updates steps.") + +flags.DEFINE_boolean( + "eval_all_checkpoints", False, + "Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number") + +flags.DEFINE_boolean( + "no_cuda", False, + "Avoid using CUDA when available") + +flags.DEFINE_boolean( + "overwrite_output_dir", False, + "Overwrite the content of the output directory") + +flags.DEFINE_boolean( + "overwrite_cache", False, + "Overwrite the cached training and evaluation sets") + +flags.DEFINE_integer( + "seed", 42, + "random seed for initialization") + +flags.DEFINE_boolean( + "fp16", False, + "Whether to use 16-bit (mixed) precision instead of 32-bit") + +flags.DEFINE_string( + "gpus", "0", + "Comma separated list of gpus devices. If only one, switch to single " + "gpu strategy, if None takes all the gpus available.") + + +def train(args, strategy, train_dataset, tokenizer, model, num_train_examples, labels, train_batch_size, pad_token_label_id): + if args['max_steps'] > 0: + num_train_steps = args['max_steps'] * args['gradient_accumulation_steps'] + args['num_train_epochs'] = 1 + else: + num_train_steps = math.ceil(num_train_examples / train_batch_size) // args['gradient_accumulation_steps'] * args['num_train_epochs'] + + writer = tf.summary.create_file_writer("/tmp/mylogs") + + with strategy.scope(): + loss_fct = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE) + optimizer = create_optimizer(args['learning_rate'], num_train_steps, args['warmup_steps']) + + if args['fp16']: + optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer(optimizer, 'dynamic') + + loss_metric = tf.keras.metrics.Mean(name='loss', dtype=tf.float32) + gradient_accumulator = GradientAccumulator() + + logging.info("***** Running training *****") + logging.info(" Num examples = %d", num_train_examples) + logging.info(" Num Epochs = %d", args['num_train_epochs']) + logging.info(" Instantaneous batch size per device = %d", args['per_device_train_batch_size']) + logging.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", + train_batch_size * args['gradient_accumulation_steps']) + logging.info(" Gradient Accumulation steps = %d", args['gradient_accumulation_steps']) + logging.info(" Total training steps = %d", num_train_steps) + + model.summary() + + @tf.function + def apply_gradients(): + grads_and_vars = [] + + for gradient, variable in zip(gradient_accumulator.gradients, model.trainable_variables): + if gradient is not None: + scaled_gradient = gradient / (args['n_device'] * args['gradient_accumulation_steps']) + grads_and_vars.append((scaled_gradient, variable)) + else: + grads_and_vars.append((gradient, variable)) + + optimizer.apply_gradients(grads_and_vars, args['max_grad_norm']) + gradient_accumulator.reset() + + @tf.function + def train_step(train_features, train_labels): + def step_fn(train_features, train_labels): + inputs = {'attention_mask': train_features['input_mask'], 'training': True} + + if args['model_type'] != "distilbert": + inputs["token_type_ids"] = train_features['segment_ids'] if args['model_type'] in ["bert", "xlnet"] else None + + with tf.GradientTape() as tape: + logits = model(train_features['input_ids'], **inputs)[0] + logits = tf.reshape(logits, (-1, len(labels) + 1)) + active_loss = tf.reshape(train_features['input_mask'], (-1,)) + active_logits = tf.boolean_mask(logits, active_loss) + train_labels = tf.reshape(train_labels, (-1,)) + active_labels = tf.boolean_mask(train_labels, active_loss) + cross_entropy = loss_fct(active_labels, active_logits) + loss = tf.reduce_sum(cross_entropy) * (1.0 / train_batch_size) + grads = tape.gradient(loss, model.trainable_variables) + + gradient_accumulator(grads) + + return cross_entropy + + per_example_losses = strategy.experimental_run_v2(step_fn, args=(train_features, train_labels)) + mean_loss = strategy.reduce(tf.distribute.ReduceOp.MEAN, per_example_losses, axis=0) + + return mean_loss + + current_time = datetime.datetime.now() + train_iterator = master_bar(range(args['num_train_epochs'])) + global_step = 0 + logging_loss = 0.0 + + for epoch in train_iterator: + epoch_iterator = progress_bar(train_dataset, total=num_train_steps, parent=train_iterator, display=args['n_device'] > 1) + step = 1 + + with strategy.scope(): + for train_features, train_labels in epoch_iterator: + loss = train_step(train_features, train_labels) + + if step % args['gradient_accumulation_steps'] == 0: + strategy.experimental_run_v2(apply_gradients) + + loss_metric(loss) + + global_step += 1 + + if args['logging_steps'] > 0 and global_step % args['logging_steps'] == 0: + # Log metrics + if args['n_device'] == 1 and args['evaluate_during_training']: # Only evaluate when single GPU otherwise metrics may not average well + y_true, y_pred, eval_loss = evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode="dev") + report = metrics.classification_report(y_true, y_pred, digits=4) + + logging.info("Eval at step " + str(global_step) + "\n" + report) + logging.info("eval_loss: " + str(eval_loss)) + + precision = metrics.precision_score(y_true, y_pred) + recall = metrics.recall_score(y_true, y_pred) + f1 = metrics.f1_score(y_true, y_pred) + + with writer.as_default(): + tf.summary.scalar("eval_loss", eval_loss, global_step) + tf.summary.scalar("precision", precision, global_step) + tf.summary.scalar("recall", recall, global_step) + tf.summary.scalar("f1", f1, global_step) + + lr = optimizer.learning_rate + learning_rate = lr(step) + + with writer.as_default(): + tf.summary.scalar("lr", learning_rate, global_step) + tf.summary.scalar("loss", (loss_metric.result() - logging_loss) / args['logging_steps'], global_step) + + logging_loss = loss_metric.result() + + with writer.as_default(): + tf.summary.scalar("loss", loss_metric.result(), step=step) + + if args['save_steps'] > 0 and global_step % args['save_steps'] == 0: + # Save model checkpoint + output_dir = os.path.join(args['output_dir'], "checkpoint-{}".format(global_step)) + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + model.save_pretrained(output_dir) + logging.info("Saving model checkpoint to %s", output_dir) + + train_iterator.child.comment = f'loss : {loss_metric.result()}' + step += 1 + + train_iterator.write(f'loss epoch {epoch + 1}: {loss_metric.result()}') + + loss_metric.reset_states() + + logging.info(" Training took time = {}".format(datetime.datetime.now() - current_time)) + + +def evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode): + eval_batch_size = args['per_device_eval_batch_size'] * args['n_device'] + eval_dataset, size = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, eval_batch_size, mode=mode) + eval_dataset = strategy.experimental_distribute_dataset(eval_dataset) + preds = None + num_eval_steps = math.ceil(size / eval_batch_size) + master = master_bar(range(1)) + eval_iterator = progress_bar(eval_dataset, total=num_eval_steps, parent=master, display=args['n_device'] > 1) + loss_fct = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE) + loss = 0.0 + + logging.info("***** Running evaluation *****") + logging.info(" Num examples = %d", size) + logging.info(" Batch size = %d", eval_batch_size) + + for eval_features, eval_labels in eval_iterator: + inputs = {'attention_mask': eval_features['input_mask'], 'training': False} + + if args['model_type'] != "distilbert": + inputs["token_type_ids"] = eval_features['segment_ids'] if args['model_type'] in ["bert", "xlnet"] else None + + with strategy.scope(): + logits = model(eval_features['input_ids'], **inputs)[0] + tmp_logits = tf.reshape(logits, (-1, len(labels) + 1)) + active_loss = tf.reshape(eval_features['input_mask'], (-1,)) + active_logits = tf.boolean_mask(tmp_logits, active_loss) + tmp_eval_labels = tf.reshape(eval_labels, (-1,)) + active_labels = tf.boolean_mask(tmp_eval_labels, active_loss) + cross_entropy = loss_fct(active_labels, active_logits) + loss += tf.reduce_sum(cross_entropy) * (1.0 / eval_batch_size) + + if preds is None: + preds = logits.numpy() + label_ids = eval_labels.numpy() + else: + preds = np.append(preds, logits.numpy(), axis=0) + label_ids = np.append(label_ids, eval_labels.numpy(), axis=0) + + preds = np.argmax(preds, axis=2) + y_pred = [[] for _ in range(label_ids.shape[0])] + y_true = [[] for _ in range(label_ids.shape[0])] + loss = loss / num_eval_steps + + for i in range(label_ids.shape[0]): + for j in range(label_ids.shape[1]): + if label_ids[i, j] != pad_token_label_id: + y_pred[i].append(labels[preds[i, j] - 1]) + y_true[i].append(labels[label_ids[i, j] - 1]) + + return y_true, y_pred, loss.numpy() + + +def load_cache(cached_file, max_seq_length): + name_to_features = { + "input_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64), + "input_mask": tf.io.FixedLenFeature([max_seq_length], tf.int64), + "segment_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64), + "label_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64), + } + + def _decode_record(record): + example = tf.io.parse_single_example(record, name_to_features) + features = {} + features['input_ids'] = example['input_ids'] + features['input_mask'] = example['input_mask'] + features['segment_ids'] = example['segment_ids'] + + return features, example['label_ids'] + + d = tf.data.TFRecordDataset(cached_file) + d = d.map(_decode_record, num_parallel_calls=4) + count = d.reduce(0, lambda x, _: x + 1) + + return d, count.numpy() + + +def save_cache(features, cached_features_file): + writer = tf.io.TFRecordWriter(cached_features_file) + + for (ex_index, feature) in enumerate(features): + if ex_index % 5000 == 0: + logging.info("Writing example %d of %d" % (ex_index, len(features))) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + record_feature = collections.OrderedDict() + record_feature["input_ids"] = create_int_feature(feature.input_ids) + record_feature["input_mask"] = create_int_feature(feature.input_mask) + record_feature["segment_ids"] = create_int_feature(feature.segment_ids) + record_feature["label_ids"] = create_int_feature(feature.label_ids) + + tf_example = tf.train.Example(features=tf.train.Features(feature=record_feature)) + + writer.write(tf_example.SerializeToString()) + + writer.close() + + +def load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, batch_size, mode): + drop_remainder = True if args['tpu'] or mode == 'train' else False + + # Load data features from cache or dataset file + cached_features_file = os.path.join(args['data_dir'], "cached_{}_{}_{}.tf_record".format(mode, + list(filter(None, args['model_name_or_path'].split("/"))).pop(), + str(args['max_seq_length']))) + if os.path.exists(cached_features_file) and not args['overwrite_cache']: + logging.info("Loading features from cached file %s", cached_features_file) + dataset, size = load_cache(cached_features_file, args['max_seq_length']) + else: + logging.info("Creating features from dataset file at %s", args['data_dir']) + examples = read_examples_from_file(args['data_dir'], mode) + features = convert_examples_to_features(examples, labels, args['max_seq_length'], tokenizer, + cls_token_at_end=bool(args['model_type'] in ["xlnet"]), + # xlnet has a cls token at the end + cls_token=tokenizer.cls_token, + cls_token_segment_id=2 if args['model_type'] in ["xlnet"] else 0, + sep_token=tokenizer.sep_token, + sep_token_extra=bool(args['model_type'] in ["roberta"]), + # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805 + pad_on_left=bool(args['model_type'] in ["xlnet"]), + # pad on the left for xlnet + pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], + pad_token_segment_id=4 if args['model_type'] in ["xlnet"] else 0, + pad_token_label_id=pad_token_label_id + ) + logging.info("Saving features into cached file %s", cached_features_file) + save_cache(features, cached_features_file) + dataset, size = load_cache(cached_features_file, args['max_seq_length']) + + if mode == 'train': + dataset = dataset.repeat() + dataset = dataset.shuffle(buffer_size=8192, seed=args['seed']) + + dataset = dataset.batch(batch_size, drop_remainder) + dataset = dataset.prefetch(buffer_size=batch_size) + + return dataset, size + + +def main(_): + logging.set_verbosity(logging.INFO) + args = flags.FLAGS.flag_values_dict() + + if os.path.exists(args['output_dir']) and os.listdir( + args['output_dir']) and args['do_train'] and not args['overwrite_output_dir']: + raise ValueError( + "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format( + args['output_dir'])) + + if args['fp16']: + tf.config.optimizer.set_experimental_options({"auto_mixed_precision": True}) + + if args['tpu']: + resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=args['tpu']) + tf.config.experimental_connect_to_cluster(resolver) + tf.tpu.experimental.initialize_tpu_system(resolver) + strategy = tf.distribute.experimental.TPUStrategy(resolver) + args['n_device'] = args['num_tpu_cores'] + elif len(args['gpus'].split(',')) > 1: + args['n_device'] = len([f"/gpu:{gpu}" for gpu in args['gpus'].split(',')]) + strategy = tf.distribute.MirroredStrategy(devices=[f"/gpu:{gpu}" for gpu in args['gpus'].split(',')]) + elif args['no_cuda']: + args['n_device'] = 1 + strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0") + else: + args['n_device'] = len(args['gpus'].split(',')) + strategy = tf.distribute.OneDeviceStrategy(device="/gpu:" + args['gpus'].split(',')[0]) + + logging.warning("n_device: %s, distributed training: %s, 16-bits training: %s", + args['n_device'], bool(args['n_device'] > 1), args['fp16']) + + labels = get_labels(args['labels']) + num_labels = len(labels) + 1 + pad_token_label_id = 0 + config_class, model_class, tokenizer_class = MODEL_CLASSES[args['model_type']] + config = config_class.from_pretrained(args['config_name'] if args['config_name'] else args['model_name_or_path'], + num_labels=num_labels, + cache_dir=args['cache_dir'] if args['cache_dir'] else None) + + logging.info("Training/evaluation parameters %s", args) + + # Training + if args['do_train']: + tokenizer = tokenizer_class.from_pretrained(args['tokenizer_name'] if args['tokenizer_name'] else args['model_name_or_path'], + do_lower_case=args['do_lower_case'], + cache_dir=args['cache_dir'] if args['cache_dir'] else None) + + with strategy.scope(): + model = model_class.from_pretrained(args['model_name_or_path'], + from_pt=bool(".bin" in args['model_name_or_path']), + config=config, + cache_dir=args['cache_dir'] if args['cache_dir'] else None) + model.layers[-1].activation = tf.keras.activations.softmax + + train_batch_size = args['per_device_train_batch_size'] * args['n_device'] + train_dataset, num_train_examples = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, train_batch_size, mode="train") + train_dataset = strategy.experimental_distribute_dataset(train_dataset) + train(args, strategy, train_dataset, tokenizer, model, num_train_examples, labels, train_batch_size, pad_token_label_id) + + if not os.path.exists(args['output_dir']): + os.makedirs(args['output_dir']) + + logging.info("Saving model to %s", args['output_dir']) + + model.save_pretrained(args['output_dir']) + tokenizer.save_pretrained(args['output_dir']) + + # Evaluation + if args['do_eval']: + tokenizer = tokenizer_class.from_pretrained(args['output_dir'], do_lower_case=args['do_lower_case']) + checkpoints = [] + results = [] + + if args['eval_all_checkpoints']: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args['output_dir'] + "/**/" + TF2_WEIGHTS_NAME, recursive=True), key=lambda f: int(''.join(filter(str.isdigit, f)) or -1))) + + logging.info("Evaluate the following checkpoints: %s", checkpoints) + + if len(checkpoints) == 0: + checkpoints.append(args['output_dir']) + + for checkpoint in checkpoints: + global_step = checkpoint.split("-")[-1] if re.match(".*checkpoint-[0-9]", checkpoint) else "final" + + with strategy.scope(): + model = model_class.from_pretrained(checkpoint) + + y_true, y_pred, eval_loss = evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode="dev") + report = metrics.classification_report(y_true, y_pred, digits=4) + + if global_step: + results.append({global_step + "_report": report, global_step + "_loss": eval_loss}) + + output_eval_file = os.path.join(args['output_dir'], "eval_results.txt") + + with tf.io.gfile.GFile(output_eval_file, "w") as writer: + for res in results: + for key, val in res.items(): + if "loss" in key: + logging.info(key + " = " + str(val)) + writer.write(key + " = " + str(val)) + writer.write("\n") + else: + logging.info(key) + logging.info("\n" + report) + writer.write(key + "\n") + writer.write(report) + writer.write("\n") + + if args['do_predict']: + tokenizer = tokenizer_class.from_pretrained(args['output_dir'], do_lower_case=args['do_lower_case']) + model = model_class.from_pretrained(args['output_dir']) + eval_batch_size = args['per_device_eval_batch_size'] * args['n_device'] + predict_dataset, _ = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, eval_batch_size, mode="test") + y_true, y_pred, pred_loss = evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode="test") + output_test_results_file = os.path.join(args['output_dir'], "test_results.txt") + output_test_predictions_file = os.path.join(args['output_dir'], "test_predictions.txt") + report = metrics.classification_report(y_true, y_pred, digits=4) + + with tf.io.gfile.GFile(output_test_results_file, "w") as writer: + report = metrics.classification_report(y_true, y_pred, digits=4) + + logging.info("\n" + report) + + writer.write(report) + writer.write("\n\nloss = " + str(pred_loss)) + + with tf.io.gfile.GFile(output_test_predictions_file, "w") as writer: + with tf.io.gfile.GFile(os.path.join(args['data_dir'], "test.txt"), "r") as f: + example_id = 0 + + for line in f: + if line.startswith("-DOCSTART-") or line == "" or line == "\n": + writer.write(line) + + if not y_pred[example_id]: + example_id += 1 + elif y_pred[example_id]: + output_line = line.split()[0] + " " + y_pred[example_id].pop(0) + "\n" + writer.write(output_line) + else: + logging.warning("Maximum sequence length exceeded: No prediction for '%s'.", line.split()[0]) + + +if __name__ == "__main__": + flags.mark_flag_as_required("data_dir") + flags.mark_flag_as_required("output_dir") + flags.mark_flag_as_required("model_name_or_path") + flags.mark_flag_as_required("model_type") + app.run(main) diff --git a/examples/run_xnli.py b/examples/run_xnli.py new file mode 100644 index 00000000..42d134a4 --- /dev/null +++ b/examples/run_xnli.py @@ -0,0 +1,515 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Finetuning multi-lingual models on XNLI (Bert, DistilBERT, XLM). + Adapted from `examples/run_glue.py`""" + +from __future__ import absolute_import, division, print_function + +import argparse +import glob +import logging +import os +import random + +import numpy as np +import torch +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, + TensorDataset) +from torch.utils.data.distributed import DistributedSampler + +try: + from torch.utils.tensorboard import SummaryWriter +except: + from tensorboardX import SummaryWriter + +from tqdm import tqdm, trange + +from transformers import (WEIGHTS_NAME, + BertConfig, BertForSequenceClassification, BertTokenizer, + XLMConfig, XLMForSequenceClassification, XLMTokenizer, + DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + +from transformers import AdamW, get_linear_schedule_with_warmup + +from transformers import xnli_compute_metrics as compute_metrics +from transformers import xnli_output_modes as output_modes +from transformers import xnli_processors as processors + +from transformers import glue_convert_examples_to_features as convert_examples_to_features + +logger = logging.getLogger(__name__) + +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, DistilBertConfig, XLMConfig)), ()) + +MODEL_CLASSES = { + 'bert': (BertConfig, BertForSequenceClassification, BertTokenizer), + 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) +} + + +def set_seed(args): + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if args.n_gpu > 0: + torch.cuda.manual_seed_all(args.seed) + + +def train(args, train_dataset, model, tokenizer): + """ Train the model """ + if args.local_rank in [-1, 0]: + tb_writer = SummaryWriter() + + args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) + train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset) + train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size) + + if args.max_steps > 0: + t_total = args.max_steps + args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1 + else: + t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs + + # Prepare optimizer and schedule (linear warmup and decay) + no_decay = ['bias', 'LayerNorm.weight'] + optimizer_grouped_parameters = [ + {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, + {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} + ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) + if args.fp16: + try: + from apex import amp + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) + + # multi-gpu training (should be after apex fp16 initialization) + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + + # Distributed training (should be after apex fp16 initialization) + if args.local_rank != -1: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], + output_device=args.local_rank, + find_unused_parameters=True) + + # Train! + logger.info("***** Running training *****") + logger.info(" Num examples = %d", len(train_dataset)) + logger.info(" Num Epochs = %d", args.num_train_epochs) + logger.info(" Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size) + logger.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", + args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1)) + logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) + logger.info(" Total optimization steps = %d", t_total) + + global_step = 0 + tr_loss, logging_loss = 0.0, 0.0 + model.zero_grad() + train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) + set_seed(args) # Added here for reproductibility (even between python 2 and 3) + for _ in train_iterator: + epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) + for step, batch in enumerate(epoch_iterator): + model.train() + batch = tuple(t.to(args.device) for t in batch) + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None # XLM and DistilBERT don't use segment_ids + outputs = model(**inputs) + loss = outputs[0] # model outputs are always tuple in transformers (see doc) + + if args.n_gpu > 1: + loss = loss.mean() # mean() to average on multi-gpu parallel training + if args.gradient_accumulation_steps > 1: + loss = loss / args.gradient_accumulation_steps + + if args.fp16: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + tr_loss += loss.item() + if (step + 1) % args.gradient_accumulation_steps == 0: + if args.fp16: + torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) + else: + torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) + + optimizer.step() + scheduler.step() # Update learning rate schedule + model.zero_grad() + global_step += 1 + + if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: + # Log metrics + if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well + results = evaluate(args, model, tokenizer) + for key, value in results.items(): + tb_writer.add_scalar('eval_{}'.format(key), value, global_step) + tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step) + tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) + logging_loss = tr_loss + + if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: + # Save model checkpoint + output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(output_dir) + torch.save(args, os.path.join(output_dir, 'training_args.bin')) + logger.info("Saving model checkpoint to %s", output_dir) + + if args.max_steps > 0 and global_step > args.max_steps: + epoch_iterator.close() + break + if args.max_steps > 0 and global_step > args.max_steps: + train_iterator.close() + break + + if args.local_rank in [-1, 0]: + tb_writer.close() + + return global_step, tr_loss / global_step + + +def evaluate(args, model, tokenizer, prefix=""): + eval_task_names = (args.task_name,) + eval_outputs_dirs = (args.output_dir,) + + results = {} + for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): + eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True) + + if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: + os.makedirs(eval_output_dir) + + args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) + # Note that DistributedSampler samples randomly + eval_sampler = SequentialSampler(eval_dataset) + eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + + # multi-gpu eval + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + + # Eval! + logger.info("***** Running evaluation {} *****".format(prefix)) + logger.info(" Num examples = %d", len(eval_dataset)) + logger.info(" Batch size = %d", args.eval_batch_size) + eval_loss = 0.0 + nb_eval_steps = 0 + preds = None + out_label_ids = None + for batch in tqdm(eval_dataloader, desc="Evaluating"): + model.eval() + batch = tuple(t.to(args.device) for t in batch) + + with torch.no_grad(): + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None # XLM and DistilBERT don't use segment_ids + outputs = model(**inputs) + tmp_eval_loss, logits = outputs[:2] + + eval_loss += tmp_eval_loss.mean().item() + nb_eval_steps += 1 + if preds is None: + preds = logits.detach().cpu().numpy() + out_label_ids = inputs['labels'].detach().cpu().numpy() + else: + preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) + out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0) + + eval_loss = eval_loss / nb_eval_steps + if args.output_mode == "classification": + preds = np.argmax(preds, axis=1) + else: + raise ValueError('No other `output_mode` for XNLI.') + result = compute_metrics(eval_task, preds, out_label_ids) + results.update(result) + + output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt") + with open(output_eval_file, "w") as writer: + logger.info("***** Eval results {} *****".format(prefix)) + for key in sorted(result.keys()): + logger.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + + return results + + +def load_and_cache_examples(args, task, tokenizer, evaluate=False): + if args.local_rank not in [-1, 0] and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + processor = processors[task](language=args.language, train_language=args.train_language) + output_mode = output_modes[task] + # Load data features from cache or dataset file + cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}_{}'.format( + 'test' if evaluate else 'train', + list(filter(None, args.model_name_or_path.split('/'))).pop(), + str(args.max_seq_length), + str(task), + str(args.train_language if (not evaluate and args.train_language is not None) else args.language))) + if os.path.exists(cached_features_file) and not args.overwrite_cache: + logger.info("Loading features from cached file %s", cached_features_file) + features = torch.load(cached_features_file) + else: + logger.info("Creating features from dataset file at %s", args.data_dir) + label_list = processor.get_labels() + examples = processor.get_test_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) + features = convert_examples_to_features(examples, + tokenizer, + label_list=label_list, + max_length=args.max_seq_length, + output_mode=output_mode, + pad_on_left=False, + pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], + pad_token_segment_id=0, + ) + if args.local_rank in [-1, 0]: + logger.info("Saving features into cached file %s", cached_features_file) + torch.save(features, cached_features_file) + + if args.local_rank == 0 and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + # Convert to Tensors and build dataset + all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) + all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) + if output_mode == "classification": + all_labels = torch.tensor([f.label for f in features], dtype=torch.long) + else: + raise ValueError('No other `output_mode` for XNLI.') + + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) + return dataset + + +def main(): + parser = argparse.ArgumentParser() + + ## Required parameters + parser.add_argument("--data_dir", default=None, type=str, required=True, + help="The input data dir. Should contain the .tsv files (or other data files) for the task.") + parser.add_argument("--model_type", default=None, type=str, required=True, + help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) + parser.add_argument("--model_name_or_path", default=None, type=str, required=True, + help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS)) + parser.add_argument("--language", default=None, type=str, required=True, + help="Evaluation language. Also train language if `train_language` is set to None.") + parser.add_argument("--train_language", default=None, type=str, + help="Train language if is different of the evaluation language.") + parser.add_argument("--output_dir", default=None, type=str, required=True, + help="The output directory where the model predictions and checkpoints will be written.") + + ## Other parameters + parser.add_argument("--config_name", default="", type=str, + help="Pretrained config name or path if not the same as model_name") + parser.add_argument("--tokenizer_name", default="", type=str, + help="Pretrained tokenizer name or path if not the same as model_name") + parser.add_argument("--cache_dir", default="", type=str, + help="Where do you want to store the pre-trained models downloaded from s3") + parser.add_argument("--max_seq_length", default=128, type=int, + help="The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded.") + parser.add_argument("--do_train", action='store_true', + help="Whether to run training.") + parser.add_argument("--do_eval", action='store_true', + help="Whether to run eval on the test set.") + parser.add_argument("--evaluate_during_training", action='store_true', + help="Rul evaluation during training at each logging step.") + parser.add_argument("--do_lower_case", action='store_true', + help="Set this flag if you are using an uncased model.") + + parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for training.") + parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for evaluation.") + parser.add_argument('--gradient_accumulation_steps', type=int, default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.") + parser.add_argument("--learning_rate", default=5e-5, type=float, + help="The initial learning rate for Adam.") + parser.add_argument("--weight_decay", default=0.0, type=float, + help="Weight deay if we apply some.") + parser.add_argument("--adam_epsilon", default=1e-8, type=float, + help="Epsilon for Adam optimizer.") + parser.add_argument("--max_grad_norm", default=1.0, type=float, + help="Max gradient norm.") + parser.add_argument("--num_train_epochs", default=3.0, type=float, + help="Total number of training epochs to perform.") + parser.add_argument("--max_steps", default=-1, type=int, + help="If > 0: set total number of training steps to perform. Override num_train_epochs.") + parser.add_argument("--warmup_steps", default=0, type=int, + help="Linear warmup over warmup_steps.") + + parser.add_argument('--logging_steps', type=int, default=50, + help="Log every X updates steps.") + parser.add_argument('--save_steps', type=int, default=50, + help="Save checkpoint every X updates steps.") + parser.add_argument("--eval_all_checkpoints", action='store_true', + help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number") + parser.add_argument("--no_cuda", action='store_true', + help="Avoid using CUDA when available") + parser.add_argument('--overwrite_output_dir', action='store_true', + help="Overwrite the content of the output directory") + parser.add_argument('--overwrite_cache', action='store_true', + help="Overwrite the cached training and evaluation sets") + parser.add_argument('--seed', type=int, default=42, + help="random seed for initialization") + + parser.add_argument('--fp16', action='store_true', + help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") + parser.add_argument('--fp16_opt_level', type=str, default='O1', + help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." + "See details at https://nvidia.github.io/apex/amp.html") + parser.add_argument("--local_rank", type=int, default=-1, + help="For distributed training: local_rank") + parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.") + parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") + args = parser.parse_args() + + if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir: + raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir)) + + # Setup distant debugging if needed + if args.server_ip and args.server_port: + # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script + import ptvsd + print("Waiting for debugger attach") + ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) + ptvsd.wait_for_attach() + + # Setup CUDA, GPU & distributed training + if args.local_rank == -1 or args.no_cuda: + device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + args.n_gpu = torch.cuda.device_count() + else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs + torch.cuda.set_device(args.local_rank) + device = torch.device("cuda", args.local_rank) + torch.distributed.init_process_group(backend='nccl') + args.n_gpu = 1 + args.device = device + + # Setup logging + logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt = '%m/%d/%Y %H:%M:%S', + level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN) + logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", + args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) + + # Set seed + set_seed(args) + + # Prepare XNLI task + args.task_name = 'xnli' + if args.task_name not in processors: + raise ValueError("Task not found: %s" % (args.task_name)) + processor = processors[args.task_name](language=args.language, train_language=args.train_language) + args.output_mode = output_modes[args.task_name] + label_list = processor.get_labels() + num_labels = len(label_list) + + # Load pretrained model and tokenizer + if args.local_rank not in [-1, 0]: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + args.model_type = args.model_type.lower() + config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, + num_labels=num_labels, + finetuning_task=args.task_name, + cache_dir=args.cache_dir if args.cache_dir else None) + tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, + do_lower_case=args.do_lower_case, + cache_dir=args.cache_dir if args.cache_dir else None) + model = model_class.from_pretrained(args.model_name_or_path, + from_tf=bool('.ckpt' in args.model_name_or_path), + config=config, + cache_dir=args.cache_dir if args.cache_dir else None) + + if args.local_rank == 0: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + model.to(args.device) + + logger.info("Training/evaluation parameters %s", args) + + + # Training + if args.do_train: + train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False) + global_step, tr_loss = train(args, train_dataset, model, tokenizer) + logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) + + + # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): + # Create output directory if needed + if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: + os.makedirs(args.output_dir) + + logger.info("Saving model checkpoint to %s", args.output_dir) + # Save a trained model, configuration and tokenizer using `save_pretrained()`. + # They can then be reloaded using `from_pretrained()` + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(args.output_dir) + tokenizer.save_pretrained(args.output_dir) + + # Good practice: save your training arguments together with the trained model + torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) + + # Load a trained model and vocabulary that you have fine-tuned + model = model_class.from_pretrained(args.output_dir) + tokenizer = tokenizer_class.from_pretrained(args.output_dir) + model.to(args.device) + + + # Evaluation + results = {} + if args.do_eval and args.local_rank in [-1, 0]: + tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + checkpoints = [args.output_dir] + if args.eval_all_checkpoints: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + logger.info("Evaluate the following checkpoints: %s", checkpoints) + for checkpoint in checkpoints: + global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" + prefix = checkpoint.split('/')[-1] if checkpoint.find('checkpoint') != -1 else "" + + model = model_class.from_pretrained(checkpoint) + model.to(args.device) + result = evaluate(args, model, tokenizer, prefix=prefix) + result = dict((k + '_{}'.format(global_step), v) for k, v in result.items()) + results.update(result) + + return results + + +if __name__ == "__main__": + main() diff --git a/examples/summarization/README.md b/examples/summarization/README.md new file mode 100644 index 00000000..b98581e8 --- /dev/null +++ b/examples/summarization/README.md @@ -0,0 +1,61 @@ +# Text Summarization with Pretrained Encoders + +This folder contains part of the code necessary to reproduce the results on abstractive summarization from the article [Text Summarization with Pretrained Encoders](https://arxiv.org/pdf/1908.08345.pdf) by [Yang Liu](https://nlp-yang.github.io/) and [Mirella Lapata](https://homepages.inf.ed.ac.uk/mlap/). It can also be used to summarize any document. + +The original code can be found on the Yang Liu's [github repository](https://github.com/nlpyang/PreSumm). + +The model is loaded with the pre-trained weights for the abstractive summarization model trained on the CNN/Daily Mail dataset with an extractive and then abstractive tasks. + +## Setup + +``` +git clone https://github.com/huggingface/transformers && cd transformers +pip install [--editable] . +pip install nltk py-rouge +cd examples/summarization +``` + +## Reproduce the authors' results on ROUGE + +To be able to reproduce the authors' results on the CNN/Daily Mail dataset you first need to download both CNN and Daily Mail datasets [from Kyunghyun Cho's website](https://cs.nyu.edu/~kcho/DMQA/) (the links next to "Stories") in the same folder. Then uncompress the archives by running: + +```bash +tar -xvf cnn_stories.tgz && tar -xvf dailymail_stories.tgz +``` + +And move all the stories to the same folder. We will refer as `$DATA_PATH` the path to where you uncompressed both archive. Then run the following in the same folder as `run_summarization.py`: + +```bash +python run_summarization.py \ + --documents_dir $DATA_PATH \ + --summaries_output_dir $SUMMARIES_PATH \ # optional + --no_cuda false \ + --batch_size 4 \ + --min_length 50 \ + --max_length 200 \ + --beam_size 5 \ + --alpha 0.95 \ + --block_trigram true \ + --compute_rouge true +``` + +The scripts executes on GPU if one is available and if `no_cuda` is not set to `true`. Inference on multiple GPUs is not suported yet. The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. The script takes 30 hours to compute with a single Tesla V100 GPU and a batch size of 10 (300,000 texts to summarize). + +## Summarize any text + +Put the documents that you would like to summarize in a folder (the path to which is referred to as `$DATA_PATH` below) and run the following in the same folder as `run_summarization.py`: + +```bash +python run_summarization.py \ + --documents_dir $DATA_PATH \ + --summaries_output_dir $SUMMARIES_PATH \ # optional + --no_cuda false \ + --batch_size 4 \ + --min_length 50 \ + --max_length 200 \ + --beam_size 5 \ + --alpha 0.95 \ + --block_trigram true \ +``` + +You may want to play around with `min_length`, `max_length` and `alpha` to suit your use case. If you want to compute ROUGE on another dataset you will need to tweak the stories/summaries import in `utils_summarization.py` and tell it where to fetch the reference summaries. diff --git a/examples/summarization/configuration_bertabs.py b/examples/summarization/configuration_bertabs.py new file mode 100644 index 00000000..b862d58d --- /dev/null +++ b/examples/summarization/configuration_bertabs.py @@ -0,0 +1,99 @@ +# coding=utf-8 +# Copyright 2019 The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BertAbs configuration """ +import json +import logging +import sys + +from transformers import PretrainedConfig + + +logger = logging.getLogger(__name__) + + +BERTABS_FINETUNED_CONFIG_MAP = { + "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-config.json", +} + + +class BertAbsConfig(PretrainedConfig): + r""" Class to store the configuration of the BertAbs model. + + Arguments: + vocab_size: int + Number of tokens in the vocabulary. + max_pos: int + The maximum sequence length that this model will be used with. + enc_layer: int + The numner of hidden layers in the Transformer encoder. + enc_hidden_size: int + The size of the encoder's layers. + enc_heads: int + The number of attention heads for each attention layer in the encoder. + enc_ff_size: int + The size of the encoder's feed-forward layers. + enc_dropout: int + The dropout probabilitiy for all fully connected layers in the + embeddings, layers, pooler and also the attention probabilities in + the encoder. + dec_layer: int + The numner of hidden layers in the decoder. + dec_hidden_size: int + The size of the decoder's layers. + dec_heads: int + The number of attention heads for each attention layer in the decoder. + dec_ff_size: int + The size of the decoder's feed-forward layers. + dec_dropout: int + The dropout probabilitiy for all fully connected layers in the + embeddings, layers, pooler and also the attention probabilities in + the decoder. + """ + + pretrained_config_archive_map = BERTABS_FINETUNED_CONFIG_MAP + + def __init__( + self, + vocab_size=30522, + max_pos=512, + enc_layers=6, + enc_hidden_size=512, + enc_heads=8, + enc_ff_size=512, + enc_dropout=0.2, + dec_layers=6, + dec_hidden_size=768, + dec_heads=8, + dec_ff_size=2048, + dec_dropout=0.2, + **kwargs, + ): + super(BertAbsConfig, self).__init__(**kwargs) + + self.vocab_size = vocab_size + self.max_pos = max_pos + + self.enc_layers = enc_layers + self.enc_hidden_size = enc_hidden_size + self.enc_heads = enc_heads + self.enc_ff_size = enc_ff_size + self.enc_dropout = enc_dropout + + self.dec_layers = dec_layers + self.dec_hidden_size = dec_hidden_size + self.dec_heads = dec_heads + self.dec_ff_size = dec_ff_size + self.dec_dropout = dec_dropout diff --git a/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py b/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py new file mode 100644 index 00000000..33b17bfb --- /dev/null +++ b/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py @@ -0,0 +1,163 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Convert BertExtAbs's checkpoints. + +The script looks like it is doing something trivial but it is not. The "weights" +proposed by the authors are actually the entire model pickled. We need to load +the model within the original codebase to be able to only save its `state_dict`. +""" + +import argparse +from collections import namedtuple +import logging +import torch + +from models.model_builder import AbsSummarizer # The authors' implementation +from model_bertabs import BertAbsSummarizer + +from transformers import BertTokenizer + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +SAMPLE_TEXT = 'Hello world! cécé herlolip' + + +BertAbsConfig = namedtuple( + "BertAbsConfig", + ["temp_dir", "large", "use_bert_emb", "finetune_bert", "encoder", "share_emb", "max_pos", "enc_layers", "enc_hidden_size", "enc_heads", "enc_ff_size", "enc_dropout", "dec_layers", "dec_hidden_size", "dec_heads", "dec_ff_size", "dec_dropout"], +) + + +def convert_bertabs_checkpoints(path_to_checkpoints, dump_path): + """ Copy/paste and tweak the pre-trained weights provided by the creators + of BertAbs for the internal architecture. + """ + + # Instantiate the authors' model with the pre-trained weights + config = BertAbsConfig( + temp_dir=".", + finetune_bert=False, + large=False, + share_emb=True, + use_bert_emb=False, + encoder="bert", + max_pos=512, + enc_layers=6, + enc_hidden_size=512, + enc_heads=8, + enc_ff_size=512, + enc_dropout=0.2, + dec_layers=6, + dec_hidden_size=768, + dec_heads=8, + dec_ff_size=2048, + dec_dropout=0.2, + ) + checkpoints = torch.load(path_to_checkpoints, lambda storage, loc: storage) + original = AbsSummarizer(config, torch.device("cpu"), checkpoints) + original.eval() + + new_model = BertAbsSummarizer(config, torch.device("cpu")) + new_model.eval() + + # ------------------- + # Convert the weights + # ------------------- + + logging.info("convert the model") + new_model.bert.load_state_dict(original.bert.state_dict()) + new_model.decoder.load_state_dict(original.decoder.state_dict()) + new_model.generator.load_state_dict(original.generator.state_dict()) + + # ---------------------------------- + # Make sure the outpus are identical + # ---------------------------------- + + logging.info("Make sure that the models' outputs are identical") + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + + # prepare the model inputs + encoder_input_ids = tokenizer.encode("This is sample éàalj'-.") + encoder_input_ids.extend([tokenizer.pad_token_id] * (512 - len(encoder_input_ids))) + encoder_input_ids = torch.tensor(encoder_input_ids).unsqueeze(0) + decoder_input_ids = tokenizer.encode("This is sample 3 éàalj'-.") + decoder_input_ids.extend([tokenizer.pad_token_id] * (512 - len(decoder_input_ids))) + decoder_input_ids = torch.tensor(decoder_input_ids).unsqueeze(0) + + # failsafe to make sure the weights reset does not affect the + # loaded weights. + assert torch.max(torch.abs(original.generator[0].weight - new_model.generator[0].weight)) == 0 + + # forward pass + src = encoder_input_ids + tgt = decoder_input_ids + segs = token_type_ids = None + clss = None + mask_src = encoder_attention_mask = None + mask_tgt = decoder_attention_mask = None + mask_cls = None + + # The original model does not apply the geneator layer immediatly but rather in + # the beam search (where it combines softmax + linear layer). Since we already + # apply the softmax in our generation process we only apply the linear layer here. + # We make sure that the outputs of the full stack are identical + output_original_model = original(src, tgt, segs, clss, mask_src, mask_tgt, mask_cls)[0] + output_original_generator = original.generator(output_original_model) + + output_converted_model = new_model(encoder_input_ids, decoder_input_ids, token_type_ids, encoder_attention_mask, decoder_attention_mask)[0] + output_converted_generator = new_model.generator(output_converted_model) + + maximum_absolute_difference = torch.max(torch.abs(output_converted_model - output_original_model)).item() + print("Maximum absolute difference beween weights: {:.2f}".format(maximum_absolute_difference)) + maximum_absolute_difference = torch.max(torch.abs(output_converted_generator - output_original_generator)).item() + print("Maximum absolute difference beween weights: {:.2f}".format(maximum_absolute_difference)) + + are_identical = torch.allclose(output_converted_model, output_original_model, atol=1e-3) + if are_identical: + logging.info("all weights are equal up to 1e-3") + else: + raise ValueError("the weights are different. The new model is likely different from the original one.") + + # The model has been saved with torch.save(model) and this is bound to the exact + # directory structure. We save the state_dict instead. + logging.info("saving the model's state dictionary") + torch.save(new_model.state_dict(), "bertabs-finetuned-cnndm-extractive-abstractive-summarization-pytorch_model.bin") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--bertabs_checkpoint_path", + default=None, + type=str, + required=True, + help="Path the official PyTorch dump.", + ) + parser.add_argument( + "--pytorch_dump_folder_path", + default=None, + type=str, + required=True, + help="Path to the output PyTorch model.", + ) + args = parser.parse_args() + + convert_bertabs_checkpoints( + args.bertabs_checkpoint_path, + args.pytorch_dump_folder_path, + ) diff --git a/examples/summarization/modeling_bertabs.py b/examples/summarization/modeling_bertabs.py new file mode 100644 index 00000000..5bf1599a --- /dev/null +++ b/examples/summarization/modeling_bertabs.py @@ -0,0 +1,1161 @@ +# MIT License + +# Copyright (c) 2019 Yang Liu and the HuggingFace team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +import copy +import math + +import numpy as np +import torch +from torch import nn +from torch.nn.init import xavier_uniform_ + +from transformers import BertModel, BertConfig, PreTrainedModel + +from configuration_bertabs import BertAbsConfig + + +MAX_SIZE = 5000 + +BERTABS_FINETUNED_MODEL_MAP = { + "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-pytorch_model.bin", +} + + +class BertAbsPreTrainedModel(PreTrainedModel): + config_class = BertAbsConfig + pretrained_model_archive_map = BERTABS_FINETUNED_MODEL_MAP + load_tf_weights = False + base_model_prefix = "bert" + + +class BertAbs(BertAbsPreTrainedModel): + def __init__(self, args, checkpoint=None, bert_extractive_checkpoint=None): + super(BertAbs, self).__init__(args) + self.args = args + self.bert = Bert() + + # If pre-trained weights are passed for Bert, load these. + load_bert_pretrained_extractive = True if bert_extractive_checkpoint else False + if load_bert_pretrained_extractive: + self.bert.model.load_state_dict( + dict( + [ + (n[11:], p) + for n, p in bert_extractive_checkpoint.items() + if n.startswith("bert.model") + ] + ), + strict=True, + ) + + self.vocab_size = self.bert.model.config.vocab_size + + if args.max_pos > 512: + my_pos_embeddings = nn.Embedding( + args.max_pos, self.bert.model.config.hidden_size + ) + my_pos_embeddings.weight.data[ + :512 + ] = self.bert.model.embeddings.position_embeddings.weight.data + my_pos_embeddings.weight.data[ + 512: + ] = self.bert.model.embeddings.position_embeddings.weight.data[-1][ + None, : + ].repeat( + args.max_pos - 512, 1 + ) + self.bert.model.embeddings.position_embeddings = my_pos_embeddings + tgt_embeddings = nn.Embedding( + self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0 + ) + + tgt_embeddings.weight = copy.deepcopy( + self.bert.model.embeddings.word_embeddings.weight + ) + + self.decoder = TransformerDecoder( + self.args.dec_layers, + self.args.dec_hidden_size, + heads=self.args.dec_heads, + d_ff=self.args.dec_ff_size, + dropout=self.args.dec_dropout, + embeddings=tgt_embeddings, + vocab_size=self.vocab_size, + ) + + gen_func = nn.LogSoftmax(dim=-1) + self.generator = nn.Sequential( + nn.Linear(args.dec_hidden_size, args.vocab_size), gen_func + ) + self.generator[0].weight = self.decoder.embeddings.weight + + load_from_checkpoints = False if checkpoint is None else True + if load_from_checkpoints: + self.load_state_dict(checkpoint) + + def init_weights(self): + for module in self.decoder.modules(): + if isinstance(module, (nn.Linear, nn.Embedding)): + module.weight.data.normal_(mean=0.0, std=0.02) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + for p in self.generator.parameters(): + if p.dim() > 1: + xavier_uniform_(p) + else: + p.data.zero_() + + def forward( + self, + encoder_input_ids, + decoder_input_ids, + token_type_ids, + encoder_attention_mask, + decoder_attention_mask, + ): + encoder_output = self.bert( + input_ids=encoder_input_ids, + token_type_ids=token_type_ids, + attention_mask=encoder_attention_mask, + ) + encoder_hidden_states = encoder_output[0] + dec_state = self.decoder.init_decoder_state( + encoder_input_ids, encoder_hidden_states + ) + decoder_outputs, _ = self.decoder( + decoder_input_ids[:, :-1], encoder_hidden_states, dec_state + ) + return decoder_outputs + + +class Bert(nn.Module): + """ This class is not really necessary and should probably disappear. + """ + + def __init__(self): + super(Bert, self).__init__() + config = BertConfig.from_pretrained("bert-base-uncased") + self.model = BertModel(config) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, **kwargs): + self.eval() + with torch.no_grad(): + encoder_outputs, _ = self.model( + input_ids, + token_type_ids=token_type_ids, + attention_mask=attention_mask, + **kwargs + ) + return encoder_outputs + + +class TransformerDecoder(nn.Module): + """ + The Transformer decoder from "Attention is All You Need". + + Args: + num_layers (int): number of encoder layers. + d_model (int): size of the model + heads (int): number of heads + d_ff (int): size of the inner FF layer + dropout (float): dropout parameters + embeddings (:obj:`onmt.modules.Embeddings`): + embeddings to use, should have positional encodings + attn_type (str): if using a seperate copy attention + """ + + def __init__(self, num_layers, d_model, heads, d_ff, dropout, embeddings, vocab_size): + super(TransformerDecoder, self).__init__() + + # Basic attributes. + self.decoder_type = "transformer" + self.num_layers = num_layers + self.embeddings = embeddings + self.pos_emb = PositionalEncoding(dropout, self.embeddings.embedding_dim) + + # Build TransformerDecoder. + self.transformer_layers = nn.ModuleList( + [ + TransformerDecoderLayer(d_model, heads, d_ff, dropout) + for _ in range(num_layers) + ] + ) + + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + + # forward(input_ids, attention_mask, encoder_hidden_states, encoder_attention_mask) + # def forward(self, input_ids, state, attention_mask=None, memory_lengths=None, + # step=None, cache=None, encoder_attention_mask=None, encoder_hidden_states=None, memory_masks=None): + def forward( + self, + input_ids, + encoder_hidden_states=None, + state=None, + attention_mask=None, + memory_lengths=None, + step=None, + cache=None, + encoder_attention_mask=None, + ): + """ + See :obj:`onmt.modules.RNNDecoderBase.forward()` + memory_bank = encoder_hidden_states + """ + # Name conversion + tgt = input_ids + memory_bank = encoder_hidden_states + memory_mask = encoder_attention_mask + + # src_words = state.src + src_words = state.src + src_batch, src_len = src_words.size() + + padding_idx = self.embeddings.padding_idx + + # Decoder padding mask + tgt_words = tgt + tgt_batch, tgt_len = tgt_words.size() + tgt_pad_mask = ( + tgt_words.data.eq(padding_idx).unsqueeze(1).expand(tgt_batch, tgt_len, tgt_len) + ) + + # Encoder padding mask + if memory_mask is not None: + src_len = memory_mask.size(-1) + src_pad_mask = memory_mask.expand(src_batch, tgt_len, src_len) + else: + src_pad_mask = ( + src_words.data.eq(padding_idx) + .unsqueeze(1) + .expand(src_batch, tgt_len, src_len) + ) + + # Pass through the embeddings + emb = self.embeddings(input_ids) + output = self.pos_emb(emb, step) + assert emb.dim() == 3 # len x batch x embedding_dim + + if state.cache is None: + saved_inputs = [] + + for i in range(self.num_layers): + prev_layer_input = None + if state.cache is None: + if state.previous_input is not None: + prev_layer_input = state.previous_layer_inputs[i] + + output, all_input = self.transformer_layers[i]( + output, + memory_bank, + src_pad_mask, + tgt_pad_mask, + previous_input=prev_layer_input, + layer_cache=state.cache["layer_{}".format(i)] + if state.cache is not None + else None, + step=step, + ) + if state.cache is None: + saved_inputs.append(all_input) + + if state.cache is None: + saved_inputs = torch.stack(saved_inputs) + + output = self.layer_norm(output) + + if state.cache is None: + state = state.update_state(tgt, saved_inputs) + + # Decoders in transformers return a tuple. Beam search will fail + # if we don't follow this convention. + return output, state # , state + + def init_decoder_state(self, src, memory_bank, with_cache=False): + """ Init decoder state """ + state = TransformerDecoderState(src) + if with_cache: + state._init_cache(memory_bank, self.num_layers) + return state + + +class PositionalEncoding(nn.Module): + def __init__(self, dropout, dim, max_len=5000): + pe = torch.zeros(max_len, dim) + position = torch.arange(0, max_len).unsqueeze(1) + div_term = torch.exp( + (torch.arange(0, dim, 2, dtype=torch.float) * -(math.log(10000.0) / dim)) + ) + pe[:, 0::2] = torch.sin(position.float() * div_term) + pe[:, 1::2] = torch.cos(position.float() * div_term) + pe = pe.unsqueeze(0) + super(PositionalEncoding, self).__init__() + self.register_buffer("pe", pe) + self.dropout = nn.Dropout(p=dropout) + self.dim = dim + + def forward(self, emb, step=None): + emb = emb * math.sqrt(self.dim) + if step: + emb = emb + self.pe[:, step][:, None, :] + + else: + emb = emb + self.pe[:, : emb.size(1)] + emb = self.dropout(emb) + return emb + + def get_emb(self, emb): + return self.pe[:, : emb.size(1)] + + +class TransformerDecoderLayer(nn.Module): + """ + Args: + d_model (int): the dimension of keys/values/queries in + MultiHeadedAttention, also the input size of + the first-layer of the PositionwiseFeedForward. + heads (int): the number of heads for MultiHeadedAttention. + d_ff (int): the second-layer of the PositionwiseFeedForward. + dropout (float): dropout probability(0-1.0). + self_attn_type (string): type of self-attention scaled-dot, average + """ + + def __init__(self, d_model, heads, d_ff, dropout): + super(TransformerDecoderLayer, self).__init__() + + self.self_attn = MultiHeadedAttention(heads, d_model, dropout=dropout) + + self.context_attn = MultiHeadedAttention(heads, d_model, dropout=dropout) + self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout) + self.layer_norm_1 = nn.LayerNorm(d_model, eps=1e-6) + self.layer_norm_2 = nn.LayerNorm(d_model, eps=1e-6) + self.drop = nn.Dropout(dropout) + mask = self._get_attn_subsequent_mask(MAX_SIZE) + # Register self.mask as a buffer in TransformerDecoderLayer, so + # it gets TransformerDecoderLayer's cuda behavior automatically. + self.register_buffer("mask", mask) + + def forward( + self, + inputs, + memory_bank, + src_pad_mask, + tgt_pad_mask, + previous_input=None, + layer_cache=None, + step=None, + ): + """ + Args: + inputs (`FloatTensor`): `[batch_size x 1 x model_dim]` + memory_bank (`FloatTensor`): `[batch_size x src_len x model_dim]` + src_pad_mask (`LongTensor`): `[batch_size x 1 x src_len]` + tgt_pad_mask (`LongTensor`): `[batch_size x 1 x 1]` + + Returns: + (`FloatTensor`, `FloatTensor`, `FloatTensor`): + + * output `[batch_size x 1 x model_dim]` + * attn `[batch_size x 1 x src_len]` + * all_input `[batch_size x current_step x model_dim]` + + """ + dec_mask = torch.gt( + tgt_pad_mask + self.mask[:, : tgt_pad_mask.size(1), : tgt_pad_mask.size(1)], 0 + ) + input_norm = self.layer_norm_1(inputs) + all_input = input_norm + if previous_input is not None: + all_input = torch.cat((previous_input, input_norm), dim=1) + dec_mask = None + + query = self.self_attn( + all_input, + all_input, + input_norm, + mask=dec_mask, + layer_cache=layer_cache, + type="self", + ) + + query = self.drop(query) + inputs + + query_norm = self.layer_norm_2(query) + mid = self.context_attn( + memory_bank, + memory_bank, + query_norm, + mask=src_pad_mask, + layer_cache=layer_cache, + type="context", + ) + output = self.feed_forward(self.drop(mid) + query) + + return output, all_input + # return output + + def _get_attn_subsequent_mask(self, size): + """ + Get an attention mask to avoid using the subsequent info. + + Args: + size: int + + Returns: + (`LongTensor`): + + * subsequent_mask `[1 x size x size]` + """ + attn_shape = (1, size, size) + subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype("uint8") + subsequent_mask = torch.from_numpy(subsequent_mask) + return subsequent_mask + + +class MultiHeadedAttention(nn.Module): + """ + Multi-Head Attention module from + "Attention is All You Need" + :cite:`DBLP:journals/corr/VaswaniSPUJGKP17`. + + Similar to standard `dot` attention but uses + multiple attention distributions simulataneously + to select relevant items. + + .. mermaid:: + + graph BT + A[key] + B[value] + C[query] + O[output] + subgraph Attn + D[Attn 1] + E[Attn 2] + F[Attn N] + end + A --> D + C --> D + A --> E + C --> E + A --> F + C --> F + D --> O + E --> O + F --> O + B --> O + + Also includes several additional tricks. + + Args: + head_count (int): number of parallel heads + model_dim (int): the dimension of keys/values/queries, + must be divisible by head_count + dropout (float): dropout parameter + """ + + def __init__(self, head_count, model_dim, dropout=0.1, use_final_linear=True): + assert model_dim % head_count == 0 + self.dim_per_head = model_dim // head_count + self.model_dim = model_dim + + super(MultiHeadedAttention, self).__init__() + self.head_count = head_count + + self.linear_keys = nn.Linear(model_dim, head_count * self.dim_per_head) + self.linear_values = nn.Linear(model_dim, head_count * self.dim_per_head) + self.linear_query = nn.Linear(model_dim, head_count * self.dim_per_head) + self.softmax = nn.Softmax(dim=-1) + self.dropout = nn.Dropout(dropout) + self.use_final_linear = use_final_linear + if self.use_final_linear: + self.final_linear = nn.Linear(model_dim, model_dim) + + def forward( + self, + key, + value, + query, + mask=None, + layer_cache=None, + type=None, + predefined_graph_1=None, + ): + """ + Compute the context vector and the attention vectors. + + Args: + key (`FloatTensor`): set of `key_len` + key vectors `[batch, key_len, dim]` + value (`FloatTensor`): set of `key_len` + value vectors `[batch, key_len, dim]` + query (`FloatTensor`): set of `query_len` + query vectors `[batch, query_len, dim]` + mask: binary mask indicating which keys have + non-zero attention `[batch, query_len, key_len]` + Returns: + (`FloatTensor`, `FloatTensor`) : + + * output context vectors `[batch, query_len, dim]` + * one of the attention vectors `[batch, query_len, key_len]` + """ + batch_size = key.size(0) + dim_per_head = self.dim_per_head + head_count = self.head_count + key_len = key.size(1) + query_len = query.size(1) + + def shape(x): + """ projection """ + return x.view(batch_size, -1, head_count, dim_per_head).transpose(1, 2) + + def unshape(x): + """ compute context """ + return ( + x.transpose(1, 2) + .contiguous() + .view(batch_size, -1, head_count * dim_per_head) + ) + + # 1) Project key, value, and query. + if layer_cache is not None: + if type == "self": + query, key, value = ( + self.linear_query(query), + self.linear_keys(query), + self.linear_values(query), + ) + + key = shape(key) + value = shape(value) + + if layer_cache is not None: + device = key.device + if layer_cache["self_keys"] is not None: + key = torch.cat((layer_cache["self_keys"].to(device), key), dim=2) + if layer_cache["self_values"] is not None: + value = torch.cat( + (layer_cache["self_values"].to(device), value), dim=2 + ) + layer_cache["self_keys"] = key + layer_cache["self_values"] = value + elif type == "context": + query = self.linear_query(query) + if layer_cache is not None: + if layer_cache["memory_keys"] is None: + key, value = self.linear_keys(key), self.linear_values(value) + key = shape(key) + value = shape(value) + else: + key, value = ( + layer_cache["memory_keys"], + layer_cache["memory_values"], + ) + layer_cache["memory_keys"] = key + layer_cache["memory_values"] = value + else: + key, value = self.linear_keys(key), self.linear_values(value) + key = shape(key) + value = shape(value) + else: + key = self.linear_keys(key) + value = self.linear_values(value) + query = self.linear_query(query) + key = shape(key) + value = shape(value) + + query = shape(query) + + key_len = key.size(2) + query_len = query.size(2) + + # 2) Calculate and scale scores. + query = query / math.sqrt(dim_per_head) + scores = torch.matmul(query, key.transpose(2, 3)) + + if mask is not None: + mask = mask.unsqueeze(1).expand_as(scores) + scores = scores.masked_fill(mask, -1e18) + + # 3) Apply attention dropout and compute context vectors. + + attn = self.softmax(scores) + + if not predefined_graph_1 is None: + attn_masked = attn[:, -1] * predefined_graph_1 + attn_masked = attn_masked / (torch.sum(attn_masked, 2).unsqueeze(2) + 1e-9) + + attn = torch.cat([attn[:, :-1], attn_masked.unsqueeze(1)], 1) + + drop_attn = self.dropout(attn) + if self.use_final_linear: + context = unshape(torch.matmul(drop_attn, value)) + output = self.final_linear(context) + return output + else: + context = torch.matmul(drop_attn, value) + return context + + +class DecoderState(object): + """Interface for grouping together the current state of a recurrent + decoder. In the simplest case just represents the hidden state of + the model. But can also be used for implementing various forms of + input_feeding and non-recurrent models. + + Modules need to implement this to utilize beam search decoding. + """ + + def detach(self): + """ Need to document this """ + self.hidden = tuple([_.detach() for _ in self.hidden]) + self.input_feed = self.input_feed.detach() + + def beam_update(self, idx, positions, beam_size): + """ Need to document this """ + for e in self._all: + sizes = e.size() + br = sizes[1] + if len(sizes) == 3: + sent_states = e.view(sizes[0], beam_size, br // beam_size, sizes[2])[ + :, :, idx + ] + else: + sent_states = e.view( + sizes[0], beam_size, br // beam_size, sizes[2], sizes[3] + )[:, :, idx] + + sent_states.data.copy_(sent_states.data.index_select(1, positions)) + + def map_batch_fn(self, fn): + raise NotImplementedError() + + +class TransformerDecoderState(DecoderState): + """ Transformer Decoder state base class """ + + def __init__(self, src): + """ + Args: + src (FloatTensor): a sequence of source words tensors + with optional feature tensors, of size (len x batch). + """ + self.src = src + self.previous_input = None + self.previous_layer_inputs = None + self.cache = None + + @property + def _all(self): + """ + Contains attributes that need to be updated in self.beam_update(). + """ + if self.previous_input is not None and self.previous_layer_inputs is not None: + return (self.previous_input, self.previous_layer_inputs, self.src) + else: + return (self.src,) + + def detach(self): + if self.previous_input is not None: + self.previous_input = self.previous_input.detach() + if self.previous_layer_inputs is not None: + self.previous_layer_inputs = self.previous_layer_inputs.detach() + self.src = self.src.detach() + + def update_state(self, new_input, previous_layer_inputs): + state = TransformerDecoderState(self.src) + state.previous_input = new_input + state.previous_layer_inputs = previous_layer_inputs + return state + + def _init_cache(self, memory_bank, num_layers): + self.cache = {} + + for l in range(num_layers): + layer_cache = {"memory_keys": None, "memory_values": None} + layer_cache["self_keys"] = None + layer_cache["self_values"] = None + self.cache["layer_{}".format(l)] = layer_cache + + def repeat_beam_size_times(self, beam_size): + """ Repeat beam_size times along batch dimension. """ + self.src = self.src.data.repeat(1, beam_size, 1) + + def map_batch_fn(self, fn): + def _recursive_map(struct, batch_dim=0): + for k, v in struct.items(): + if v is not None: + if isinstance(v, dict): + _recursive_map(v) + else: + struct[k] = fn(v, batch_dim) + + self.src = fn(self.src, 0) + if self.cache is not None: + _recursive_map(self.cache) + + +def gelu(x): + return ( + 0.5 + * x + * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + ) + + +class PositionwiseFeedForward(nn.Module): + """ A two-layer Feed-Forward-Network with residual layer norm. + + Args: + d_model (int): the size of input for the first-layer of the FFN. + d_ff (int): the hidden layer size of the second-layer + of the FNN. + dropout (float): dropout probability in :math:`[0, 1)`. + """ + + def __init__(self, d_model, d_ff, dropout=0.1): + super(PositionwiseFeedForward, self).__init__() + self.w_1 = nn.Linear(d_model, d_ff) + self.w_2 = nn.Linear(d_ff, d_model) + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.actv = gelu + self.dropout_1 = nn.Dropout(dropout) + self.dropout_2 = nn.Dropout(dropout) + + def forward(self, x): + inter = self.dropout_1(self.actv(self.w_1(self.layer_norm(x)))) + output = self.dropout_2(self.w_2(inter)) + return output + x + + +# +# TRANSLATOR +# The following code is used to generate summaries using the +# pre-trained weights and beam search. +# + + +def build_predictor(args, tokenizer, symbols, model, logger=None): + # we should be able to refactor the global scorer a lot + scorer = GNMTGlobalScorer(args.alpha, length_penalty="wu") + translator = Translator( + args, model, tokenizer, symbols, global_scorer=scorer, logger=logger + ) + return translator + + +class GNMTGlobalScorer(object): + """ + NMT re-ranking score from + "Google's Neural Machine Translation System" :cite:`wu2016google` + + Args: + alpha (float): length parameter + beta (float): coverage parameter + """ + + def __init__(self, alpha, length_penalty): + self.alpha = alpha + penalty_builder = PenaltyBuilder(length_penalty) + self.length_penalty = penalty_builder.length_penalty() + + def score(self, beam, logprobs): + """ + Rescores a prediction based on penalty functions + """ + normalized_probs = self.length_penalty(beam, logprobs, self.alpha) + return normalized_probs + + +class PenaltyBuilder(object): + """ + Returns the Length and Coverage Penalty function for Beam Search. + + Args: + length_pen (str): option name of length pen + cov_pen (str): option name of cov pen + """ + + def __init__(self, length_pen): + self.length_pen = length_pen + + def length_penalty(self): + if self.length_pen == "wu": + return self.length_wu + elif self.length_pen == "avg": + return self.length_average + else: + return self.length_none + + """ + Below are all the different penalty terms implemented so far + """ + + def length_wu(self, beam, logprobs, alpha=0.0): + """ + NMT length re-ranking score from + "Google's Neural Machine Translation System" :cite:`wu2016google`. + """ + + modifier = ((5 + len(beam.next_ys)) ** alpha) / ((5 + 1) ** alpha) + return logprobs / modifier + + def length_average(self, beam, logprobs, alpha=0.0): + """ + Returns the average probability of tokens in a sequence. + """ + return logprobs / len(beam.next_ys) + + def length_none(self, beam, logprobs, alpha=0.0, beta=0.0): + """ + Returns unmodified scores. + """ + return logprobs + + +class Translator(object): + """ + Uses a model to translate a batch of sentences. + + Args: + model (:obj:`onmt.modules.NMTModel`): + NMT model to use for translation + fields (dict of Fields): data fields + beam_size (int): size of beam to use + n_best (int): number of translations produced + max_length (int): maximum length output to produce + global_scores (:obj:`GlobalScorer`): + object to rescore final translations + copy_attn (bool): use copy attention during translation + beam_trace (bool): trace beam search for debugging + logger(logging.Logger): logger. + """ + + def __init__(self, args, model, vocab, symbols, global_scorer=None, logger=None): + self.logger = logger + + self.args = args + self.model = model + self.generator = self.model.generator + self.vocab = vocab + self.symbols = symbols + self.start_token = symbols["BOS"] + self.end_token = symbols["EOS"] + + self.global_scorer = global_scorer + self.beam_size = args.beam_size + self.min_length = args.min_length + self.max_length = args.max_length + + def translate(self, batch, step, attn_debug=False): + """ Generates summaries from one batch of data. + """ + self.model.eval() + with torch.no_grad(): + batch_data = self.translate_batch(batch) + translations = self.from_batch(batch_data) + return translations + + def translate_batch(self, batch, fast=False): + """ + Translate a batch of sentences. + + Mostly a wrapper around :obj:`Beam`. + + Args: + batch (:obj:`Batch`): a batch from a dataset object + data (:obj:`Dataset`): the dataset object + fast (bool): enables fast beam search (may not support all features) + + Todo: + Shouldn't need the original dataset. + """ + with torch.no_grad(): + return self._fast_translate_batch( + batch, self.max_length, min_length=self.min_length + ) + + # Where the beam search lives + # I have no idea why it is being called from the method above + def _fast_translate_batch(self, batch, max_length, min_length=0): + """ Beam Search using the encoder inputs contained in `batch`. + """ + + # The batch object is funny + # Instead of just looking at the size of the arguments we encapsulate + # a size argument. + # Where is it defined? + beam_size = self.beam_size + batch_size = batch.batch_size + src = batch.src + segs = batch.segs + mask_src = batch.mask_src + + src_features = self.model.bert(src, segs, mask_src) + dec_states = self.model.decoder.init_decoder_state( + src, src_features, with_cache=True + ) + device = src_features.device + + # Tile states and memory beam_size times. + dec_states.map_batch_fn(lambda state, dim: tile(state, beam_size, dim=dim)) + src_features = tile(src_features, beam_size, dim=0) + batch_offset = torch.arange(batch_size, dtype=torch.long, device=device) + beam_offset = torch.arange( + 0, batch_size * beam_size, step=beam_size, dtype=torch.long, device=device + ) + alive_seq = torch.full( + [batch_size * beam_size, 1], self.start_token, dtype=torch.long, device=device + ) + + # Give full probability to the first beam on the first step. + topk_log_probs = torch.tensor( + [0.0] + [float("-inf")] * (beam_size - 1), device=device + ).repeat(batch_size) + + # Structure that holds finished hypotheses. + hypotheses = [[] for _ in range(batch_size)] # noqa: F812 + + results = {} + results["predictions"] = [[] for _ in range(batch_size)] # noqa: F812 + results["scores"] = [[] for _ in range(batch_size)] # noqa: F812 + results["gold_score"] = [0] * batch_size + results["batch"] = batch + + for step in range(max_length): + decoder_input = alive_seq[:, -1].view(1, -1) + + # Decoder forward. + decoder_input = decoder_input.transpose(0, 1) + + dec_out, dec_states = self.model.decoder( + decoder_input, src_features, dec_states, step=step + ) + + # Generator forward. + log_probs = self.generator.forward(dec_out.transpose(0, 1).squeeze(0)) + vocab_size = log_probs.size(-1) + + if step < min_length: + log_probs[:, self.end_token] = -1e20 + + # Multiply probs by the beam probability. + log_probs += topk_log_probs.view(-1).unsqueeze(1) + + alpha = self.global_scorer.alpha + length_penalty = ((5.0 + (step + 1)) / 6.0) ** alpha + + # Flatten probs into a list of possibilities. + curr_scores = log_probs / length_penalty + + if self.args.block_trigram: + cur_len = alive_seq.size(1) + if cur_len > 3: + for i in range(alive_seq.size(0)): + fail = False + words = [int(w) for w in alive_seq[i]] + words = [self.vocab.ids_to_tokens[w] for w in words] + words = " ".join(words).replace(" ##", "").split() + if len(words) <= 3: + continue + trigrams = [ + (words[i - 1], words[i], words[i + 1]) + for i in range(1, len(words) - 1) + ] + trigram = tuple(trigrams[-1]) + if trigram in trigrams[:-1]: + fail = True + if fail: + curr_scores[i] = -10e20 + + curr_scores = curr_scores.reshape(-1, beam_size * vocab_size) + topk_scores, topk_ids = curr_scores.topk(beam_size, dim=-1) + + # Recover log probs. + topk_log_probs = topk_scores * length_penalty + + # Resolve beam origin and true word ids. + topk_beam_index = topk_ids.div(vocab_size) + topk_ids = topk_ids.fmod(vocab_size) + + # Map beam_index to batch_index in the flat representation. + batch_index = topk_beam_index + beam_offset[ + : topk_beam_index.size(0) + ].unsqueeze(1) + select_indices = batch_index.view(-1) + + # Append last prediction. + alive_seq = torch.cat( + [alive_seq.index_select(0, select_indices), topk_ids.view(-1, 1)], -1 + ) + + is_finished = topk_ids.eq(self.end_token) + if step + 1 == max_length: + is_finished.fill_(1) + # End condition is top beam is finished. + end_condition = is_finished[:, 0].eq(1) + # Save finished hypotheses. + if is_finished.any(): + predictions = alive_seq.view(-1, beam_size, alive_seq.size(-1)) + for i in range(is_finished.size(0)): + b = batch_offset[i] + if end_condition[i]: + is_finished[i].fill_(1) + finished_hyp = is_finished[i].nonzero().view(-1) + # Store finished hypotheses for this batch. + for j in finished_hyp: + hypotheses[b].append((topk_scores[i, j], predictions[i, j, 1:])) + # If the batch reached the end, save the n_best hypotheses. + if end_condition[i]: + best_hyp = sorted(hypotheses[b], key=lambda x: x[0], reverse=True) + score, pred = best_hyp[0] + + results["scores"][b].append(score) + results["predictions"][b].append(pred) + non_finished = end_condition.eq(0).nonzero().view(-1) + # If all sentences are translated, no need to go further. + if len(non_finished) == 0: + break + # Remove finished batches for the next step. + topk_log_probs = topk_log_probs.index_select(0, non_finished) + batch_index = batch_index.index_select(0, non_finished) + batch_offset = batch_offset.index_select(0, non_finished) + alive_seq = predictions.index_select(0, non_finished).view( + -1, alive_seq.size(-1) + ) + # Reorder states. + select_indices = batch_index.view(-1) + src_features = src_features.index_select(0, select_indices) + dec_states.map_batch_fn( + lambda state, dim: state.index_select(dim, select_indices) + ) + + return results + + def from_batch(self, translation_batch): + batch = translation_batch["batch"] + assert len(translation_batch["gold_score"]) == len(translation_batch["predictions"]) + batch_size = batch.batch_size + + preds, _, _, tgt_str, src = ( + translation_batch["predictions"], + translation_batch["scores"], + translation_batch["gold_score"], + batch.tgt_str, + batch.src, + ) + + translations = [] + for b in range(batch_size): + pred_sents = self.vocab.convert_ids_to_tokens([int(n) for n in preds[b][0]]) + pred_sents = " ".join(pred_sents).replace(" ##", "") + gold_sent = " ".join(tgt_str[b].split()) + raw_src = [self.vocab.ids_to_tokens[int(t)] for t in src[b]][:500] + raw_src = " ".join(raw_src) + translation = (pred_sents, gold_sent, raw_src) + translations.append(translation) + + return translations + + +def tile(x, count, dim=0): + """ + Tiles x on dimension dim count times. + """ + perm = list(range(len(x.size()))) + if dim != 0: + perm[0], perm[dim] = perm[dim], perm[0] + x = x.permute(perm).contiguous() + out_size = list(x.size()) + out_size[0] *= count + batch = x.size(0) + x = ( + x.view(batch, -1) + .transpose(0, 1) + .repeat(count, 1) + .transpose(0, 1) + .contiguous() + .view(*out_size) + ) + if dim != 0: + x = x.permute(perm).contiguous() + return x + + +# +# Optimizer for training. We keep this here in case we want to add +# a finetuning script. +# + +class BertSumOptimizer(object): + """ Specific optimizer for BertSum. + + As described in [1], the authors fine-tune BertSum for abstractive + summarization using two Adam Optimizers with different warm-up steps and + learning rate. They also use a custom learning rate scheduler. + + [1] Liu, Yang, and Mirella Lapata. "Text summarization with pretrained encoders." + arXiv preprint arXiv:1908.08345 (2019). + """ + + def __init__(self, model, lr, warmup_steps, beta_1=0.99, beta_2=0.999, eps=1e-8): + self.encoder = model.encoder + self.decoder = model.decoder + self.lr = lr + self.warmup_steps = warmup_steps + + self.optimizers = { + "encoder": torch.optim.Adam( + model.encoder.parameters(), + lr=lr["encoder"], + betas=(beta_1, beta_2), + eps=eps, + ), + "decoder": torch.optim.Adam( + model.decoder.parameters(), + lr=lr["decoder"], + betas=(beta_1, beta_2), + eps=eps, + ), + } + + self._step = 0 + self.current_learning_rates = {} + + def _update_rate(self, stack): + return self.lr[stack] * min( + self._step ** (-0.5), self._step * self.warmup_steps[stack] ** (-1.5) + ) + + def zero_grad(self): + self.optimizer_decoder.zero_grad() + self.optimizer_encoder.zero_grad() + + def step(self): + self._step += 1 + for stack, optimizer in self.optimizers.items(): + new_rate = self._update_rate(stack) + for param_group in optimizer.param_groups: + param_group["lr"] = new_rate + optimizer.step() + self.current_learning_rates[stack] = new_rate diff --git a/examples/summarization/requirements.txt b/examples/summarization/requirements.txt new file mode 100644 index 00000000..36d75a5e --- /dev/null +++ b/examples/summarization/requirements.txt @@ -0,0 +1,9 @@ +# progress bars in model download and training scripts +tqdm +# Accessing files from S3 directly. +boto3 +# Used for downloading models over HTTP +requests +# For ROUGE +nltk +py-rouge diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py new file mode 100644 index 00000000..3c339d0c --- /dev/null +++ b/examples/summarization/run_summarization.py @@ -0,0 +1,344 @@ +#! /usr/bin/python3 +import argparse +from collections import namedtuple +import logging +import os +import sys + +import torch +from torch.utils.data import DataLoader, SequentialSampler +from tqdm import tqdm + +from transformers import BertTokenizer + +from modeling_bertabs import BertAbs, build_predictor + +from utils_summarization import ( + SummarizationDataset, + encode_for_summarization, + build_mask, + fit_to_block_size, + compute_token_type_ids, +) + +logger = logging.getLogger(__name__) +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + +Batch = namedtuple( + "Batch", ["document_names", "batch_size", "src", "segs", "mask_src", "tgt_str"] +) + + +def evaluate(args): + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True) + model = BertAbs.from_pretrained("bertabs-finetuned-cnndm") + model.to(args.device) + model.eval() + + symbols = { + "BOS": tokenizer.vocab["[unused0]"], + "EOS": tokenizer.vocab["[unused1]"], + "PAD": tokenizer.vocab["[PAD]"], + } + + if args.compute_rouge: + reference_summaries = [] + generated_summaries = [] + + import rouge + import nltk + nltk.download('punkt') + rouge_evaluator = rouge.Rouge( + metrics=['rouge-n', 'rouge-l'], + max_n=2, + limit_length=True, + length_limit=args.beam_size, + length_limit_type='words', + apply_avg=True, + apply_best=False, + alpha=0.5, # Default F1_score + weight_factor=1.2, + stemming=True, + ) + + # these (unused) arguments are defined to keep the compatibility + # with the legacy code and will be deleted in a next iteration. + args.result_path = "" + args.temp_dir = "" + + data_iterator = build_data_iterator(args, tokenizer) + predictor = build_predictor(args, tokenizer, symbols, model) + + logger.info("***** Running evaluation *****") + logger.info(" Number examples = %d", len(data_iterator.dataset)) + logger.info(" Batch size = %d", args.batch_size) + logger.info("") + logger.info("***** Beam Search parameters *****") + logger.info(" Beam size = %d", args.beam_size) + logger.info(" Minimum length = %d", args.min_length) + logger.info(" Maximum length = %d", args.max_length) + logger.info(" Alpha (length penalty) = %.2f", args.alpha) + logger.info(" Trigrams %s be blocked", ("will" if args.block_trigram else "will NOT")) + + for batch in tqdm(data_iterator): + batch_data = predictor.translate_batch(batch) + translations = predictor.from_batch(batch_data) + summaries = [format_summary(t) for t in translations] + save_summaries(summaries, args.summaries_output_dir, batch.document_names) + + if args.compute_rouge: + reference_summaries += batch.tgt_str + generated_summaries += summaries + + if args.compute_rouge: + scores = rouge_evaluator.get_scores(generated_summaries, reference_summaries) + str_scores = format_rouge_scores(scores) + save_rouge_scores(str_scores) + print(str_scores) + + +def save_summaries(summaries, path, original_document_name): + """ Write the summaries in fies that are prefixed by the original + files' name with the `_summary` appended. + + Attributes: + original_document_names: List[string] + Name of the document that was summarized. + path: string + Path were the summaries will be written + summaries: List[string] + The summaries that we produced. + """ + for summary, document_name in zip(summaries, original_document_name): + # Prepare the summary file's name + if "." in document_name: + bare_document_name = ".".join(document_name.split(".")[:-1]) + extension = document_name.split(".")[-1] + name = bare_document_name + "_summary." + extension + else: + name = document_name + "_summary" + + file_path = os.path.join(path, name) + with open(file_path, "w") as output: + output.write(summary) + + +def format_summary(translation): + """ Transforms the output of the `from_batch` function + into nicely formatted summaries. + """ + raw_summary, _, _ = translation + summary = ( + raw_summary.replace("[unused0]", "") + .replace("[unused3]", "") + .replace("[PAD]", "") + .replace("[unused1]", "") + .replace(r" +", " ") + .replace(" [unused2] ", ". ") + .replace("[unused2]", "") + .strip() + ) + + return summary + + +def format_rouge_scores(scores): + return """\n +****** ROUGE SCORES ****** + +** ROUGE 1 +F1 >> {:.3f} +Precision >> {:.3f} +Recall >> {:.3f} + +** ROUGE 2 +F1 >> {:.3f} +Precision >> {:.3f} +Recall >> {:.3f} + +** ROUGE L +F1 >> {:.3f} +Precision >> {:.3f} +Recall >> {:.3f}""".format( + scores['rouge-1']['f'], + scores['rouge-1']['p'], + scores['rouge-1']['r'], + scores['rouge-2']['f'], + scores['rouge-2']['p'], + scores['rouge-2']['r'], + scores['rouge-l']['f'], + scores['rouge-l']['p'], + scores['rouge-l']['r'], + ) + + +def save_rouge_scores(str_scores): + with open("rouge_scores.txt", "w") as output: + output.write(str_scores) + + +# +# LOAD the dataset +# + + +def build_data_iterator(args, tokenizer): + dataset = load_and_cache_examples(args, tokenizer) + sampler = SequentialSampler(dataset) + collate_fn = lambda data: collate(data, tokenizer, block_size=512, device=args.device) + iterator = DataLoader( + dataset, sampler=sampler, batch_size=args.batch_size, collate_fn=collate_fn, + ) + + return iterator + + +def load_and_cache_examples(args, tokenizer): + dataset = SummarizationDataset(args.documents_dir) + return dataset + + +def collate(data, tokenizer, block_size, device): + """ Collate formats the data passed to the data loader. + + In particular we tokenize the data batch after batch to avoid keeping them + all in memory. We output the data as a namedtuple to fit the original BertAbs's + API. + """ + data = [x for x in data if not len(x[1]) == 0] # remove empty_files + names = [name for name, _, _ in data] + summaries = [" ".join(summary_list) for _, _, summary_list in data] + + encoded_text = [ + encode_for_summarization(story, summary, tokenizer) for _, story, summary in data + ] + encoded_stories = torch.tensor( + [ + fit_to_block_size(story, block_size, tokenizer.pad_token_id) + for story, _ in encoded_text + ] + ) + encoder_token_type_ids = compute_token_type_ids(encoded_stories, tokenizer.cls_token_id) + encoder_mask = build_mask(encoded_stories, tokenizer.pad_token_id) + + batch = Batch( + document_names=names, + batch_size=len(encoded_stories), + src=encoded_stories.to(device), + segs=encoder_token_type_ids.to(device), + mask_src=encoder_mask.to(device), + tgt_str=summaries, + ) + + return batch + + +def decode_summary(summary_tokens, tokenizer): + """ Decode the summary and return it in a format + suitable for evaluation. + """ + summary_tokens = summary_tokens.to("cpu").numpy() + summary = tokenizer.decode(summary_tokens) + sentences = summary.split(".") + sentences = [s + "." for s in sentences] + return sentences + + +def main(): + """ The main function defines the interface with the users. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--documents_dir", + default=None, + type=str, + required=True, + help="The folder where the documents to summarize are located.", + ) + parser.add_argument( + "--summaries_output_dir", + default=None, + type=str, + required=False, + help="The folder in wich the summaries should be written. Defaults to the folder where the documents are", + ) + parser.add_argument( + "--compute_rouge", + default=False, + type=bool, + required=False, + help="Compute the ROUGE metrics during evaluation. Only available for the CNN/DailyMail dataset.", + ) + # EVALUATION options + parser.add_argument( + "--no_cuda", + default=False, + type=bool, + help="Whether to force the execution on CPU.", + ) + parser.add_argument( + "--batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.", + ) + # BEAM SEARCH arguments + parser.add_argument( + "--min_length", + default=50, + type=int, + help="Minimum number of tokens for the summaries.", + ) + parser.add_argument( + "--max_length", + default=200, + type=int, + help="Maixmum number of tokens for the summaries.", + ) + parser.add_argument( + "--beam_size", + default=5, + type=int, + help="The number of beams to start with for each example.", + ) + parser.add_argument( + "--alpha", + default=0.95, + type=float, + help="The value of alpha for the length penalty in the beam search.", + ) + parser.add_argument( + "--block_trigram", + default=True, + type=bool, + help="Whether to block the existence of repeating trigrams in the text generated by beam search.", + ) + args = parser.parse_args() + + # Select device (distibuted not available) + args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + + # Check the existence of directories + if not args.summaries_output_dir: + args.summaries_output_dir = args.documents_dir + + if not documents_dir_is_valid(args.documents_dir): + raise FileNotFoundError( + "We could not find the directory you specified for the documents to summarize, or it was empty. Please specify a valid path." + ) + os.makedirs(args.summaries_output_dir, exist_ok=True) + + evaluate(args) + + +def documents_dir_is_valid(path): + if not os.path.exists(path): + return False + + file_list = os.listdir(path) + if len(file_list) == 0: + return False + + return True + + +if __name__ == "__main__": + main() diff --git a/examples/utils_summarization.py b/examples/summarization/utils_summarization.py similarity index 66% rename from examples/utils_summarization.py rename to examples/summarization/utils_summarization.py index 327ca8cc..1d8c436a 100644 --- a/examples/utils_summarization.py +++ b/examples/summarization/utils_summarization.py @@ -10,9 +10,14 @@ from torch.utils.data import Dataset # ------------ -class CNNDailyMailDataset(Dataset): +class SummarizationDataset(Dataset): """ Abstracts the dataset used to train seq2seq models. + The class will process the documents that are located in the specified + folder. The preprocessing will work on any document that is reasonably + formatted. On the CNN/DailyMail dataset it will extract both the story + and the summary. + CNN/Daily News: The CNN/Daily News raw datasets are downloaded from [1]. The stories are @@ -25,33 +30,33 @@ class CNNDailyMailDataset(Dataset): [2] https://github.com/abisee/cnn-dailymail/ """ - def __init__(self, tokenizer, prefix="train", data_dir=""): - assert os.path.isdir(data_dir) - self.tokenizer = tokenizer + def __init__(self, path="", prefix="train"): + """ We initialize the class by listing all the documents to summarize. + Files are not read in memory due to the size of some datasets (like CNN/DailyMail). + """ + assert os.path.isdir(path) - # We initialize the class by listing all the files that contain - # stories and summaries. Files are not read in memory given - # the size of the corpus. - self.stories_path = [] - datasets = ("cnn", "dailymail") - for dataset in datasets: - path_to_stories = os.path.join(data_dir, dataset, "stories") - story_filenames_list = os.listdir(path_to_stories) - for story_filename in story_filenames_list: - path_to_story = os.path.join(path_to_stories, story_filename) - if not os.path.isfile(path_to_story): - continue - self.stories_path.append(path_to_story) + self.documents = [] + story_filenames_list = os.listdir(path) + for story_filename in story_filenames_list: + if "summary" in story_filename: + continue + path_to_story = os.path.join(path, story_filename) + if not os.path.isfile(path_to_story): + continue + self.documents.append(path_to_story) def __len__(self): - return len(self.stories_path) + """ Returns the number of documents. """ + return len(self.documents) def __getitem__(self, idx): - story_path = self.stories_path[idx] - with open(story_path, encoding="utf-8") as source: + document_path = self.documents[idx] + document_name = document_path.split("/")[-1] + with open(document_path, encoding="utf-8") as source: raw_story = source.read() story_lines, summary_lines = process_story(raw_story) - return story_lines, summary_lines + return document_name, story_lines, summary_lines def process_story(raw_story): @@ -81,7 +86,7 @@ def process_story(raw_story): story_lines.append(element) except IndexError: # if "@highlight" is absent from the file we pop - # all elements until there is None. + # all elements until there is None, raising an exception. return story_lines, [] # gather summary lines @@ -104,31 +109,22 @@ def _add_missing_period(line): # -------------------------- -def fit_to_block_size(sequence, block_size, pad_token): +def fit_to_block_size(sequence, block_size, pad_token_id): """ Adapt the source and target sequences' lengths to the block size. - If the sequence is shorter than the block size we pad it with -1 ids - which correspond to padding tokens. + If the sequence is shorter we append padding token to the right of the sequence. """ if len(sequence) > block_size: return sequence[:block_size] else: - sequence.extend([pad_token] * (block_size - len(sequence))) + sequence.extend([pad_token_id] * (block_size - len(sequence))) return sequence -def build_lm_labels(sequence, pad_token): - """ Padding token, encoded as 0, are represented by the value -1 so they - are not taken into account in the loss computation. """ - padded = sequence.clone() - padded[padded == pad_token] = -1 - return padded - - -def build_mask(sequence, pad_token): +def build_mask(sequence, pad_token_id): """ Builds the mask. The attention mechanism will only attend to positions with value 1. """ mask = torch.ones_like(sequence) - idx_pad_tokens = sequence == pad_token + idx_pad_tokens = sequence == pad_token_id mask[idx_pad_tokens] = 0 return mask @@ -138,18 +134,11 @@ def encode_for_summarization(story_lines, summary_lines, tokenizer): as specified in [1] by using `[SEP] [CLS]` tokens to separate sentences. """ - story_lines_token_ids = [ - tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line)) - for line in story_lines - ] - summary_lines_token_ids = [ - tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line)) - for line in summary_lines - ] - + story_lines_token_ids = [tokenizer.encode(line) for line in story_lines] story_token_ids = [ token for sentence in story_lines_token_ids for token in sentence ] + summary_lines_token_ids = [tokenizer.encode(line) for line in summary_lines] summary_token_ids = [ token for sentence in summary_lines_token_ids for token in sentence ] @@ -174,7 +163,7 @@ def compute_token_type_ids(batch, separator_token_id): """ batch_embeddings = [] for sequence in batch: - sentence_num = 0 + sentence_num = -1 embeddings = [] for s in sequence: if s == separator_token_id: diff --git a/examples/utils_summarization_test.py b/examples/summarization/utils_summarization_test.py similarity index 88% rename from examples/utils_summarization_test.py rename to examples/summarization/utils_summarization_test.py index 1d56ff08..8bfbf6ab 100644 --- a/examples/utils_summarization_test.py +++ b/examples/summarization/utils_summarization_test.py @@ -21,7 +21,6 @@ from utils_summarization import ( compute_token_type_ids, fit_to_block_size, build_mask, - build_lm_labels, process_story, ) @@ -88,20 +87,6 @@ class SummarizationDataProcessingTest(unittest.TestCase): expected_summary_lines = ["It was the best of times."] self.assertEqual(expected_summary_lines, summary_lines) - def test_build_lm_labels_no_padding(self): - sequence = torch.tensor([1, 2, 3, 4]) - expected = sequence - np.testing.assert_array_equal( - build_lm_labels(sequence, 0).numpy(), expected.numpy() - ) - - def test_build_lm_labels(self): - sequence = torch.tensor([1, 2, 3, 4, 0, 0, 0]) - expected = torch.tensor([1, 2, 3, 4, -1, -1, -1]) - np.testing.assert_array_equal( - build_lm_labels(sequence, 0).numpy(), expected.numpy() - ) - def test_build_mask_no_padding(self): sequence = torch.tensor([1, 2, 3, 4]) expected = torch.tensor([1, 1, 1, 1]) @@ -125,7 +110,7 @@ class SummarizationDataProcessingTest(unittest.TestCase): [[1, 2, 3, 4, 5, 6], [1, 2, 3, 101, 5, 6], [1, 101, 3, 4, 101, 6]] ) expected = torch.tensor( - [[0, 0, 0, 0, 0, 0], [0, 0, 0, 1, 1, 1], [0, 1, 1, 1, 0, 0]] + [[1, 1, 1, 1, 1, 1], [1, 1, 1, 0, 0, 0], [1, 0, 0, 0, 1, 1]] ) result = compute_token_type_ids(batch, separator) diff --git a/examples/test_examples.py b/examples/test_examples.py index b04d722b..632d2f72 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -72,8 +72,7 @@ class ExamplesTests(unittest.TestCase): logger.addHandler(stream_handler) testargs = ["run_squad.py", - "--train_file=./examples/tests_samples/SQUAD/dev-v2.0-small.json", - "--predict_file=./examples/tests_samples/SQUAD/dev-v2.0-small.json", + "--data_dir=./examples/tests_samples/SQUAD", "--model_name=bert-base-uncased", "--output_dir=./examples/tests_samples/temp_dir", "--max_steps=10", diff --git a/examples/tests_samples/SQUAD/dev-v2.0-small.json b/examples/tests_samples/SQUAD/dev-v2.0.json similarity index 100% rename from examples/tests_samples/SQUAD/dev-v2.0-small.json rename to examples/tests_samples/SQUAD/dev-v2.0.json diff --git a/examples/tests_samples/SQUAD/train-v2.0.json b/examples/tests_samples/SQUAD/train-v2.0.json new file mode 100644 index 00000000..834d9ee6 --- /dev/null +++ b/examples/tests_samples/SQUAD/train-v2.0.json @@ -0,0 +1,140 @@ +{ + "version": "v2.0", + "data": [{ + "title": "Normans", + "paragraphs": [{ + "qas": [{ + "question": "In what country is Normandy located?", + "id": "56ddde6b9a695914005b9628", + "answers": [{ + "text": "France", + "answer_start": 159 + }], + "is_impossible": false + }, { + "question": "When were the Normans in Normandy?", + "id": "56ddde6b9a695914005b9629", + "answers": [{ + "text": "10th and 11th centuries", + "answer_start": 94 + }], + "is_impossible": false + }, { + "question": "From which countries did the Norse originate?", + "id": "56ddde6b9a695914005b962a", + "answers": [{ + "text": "Denmark, Iceland and Norway", + "answer_start": 256 + }], + "is_impossible": false + }, { + "plausible_answers": [{ + "text": "Rollo", + "answer_start": 308 + }], + "question": "Who did King Charles III swear fealty to?", + "id": "5ad39d53604f3c001a3fe8d3", + "answers": [], + "is_impossible": true + }, { + "plausible_answers": [{ + "text": "10th century", + "answer_start": 671 + }], + "question": "When did the Frankish identity emerge?", + "id": "5ad39d53604f3c001a3fe8d4", + "answers": [], + "is_impossible": true + }], + "context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries." + }, { + "qas": [{ + "question": "Who was the duke in the battle of Hastings?", + "id": "56dddf4066d3e219004dad5f", + "answers": [{ + "text": "William the Conqueror", + "answer_start": 1022 + }], + "is_impossible": false + }, { + "plausible_answers": [{ + "text": "Antioch", + "answer_start": 1295 + }], + "question": "What principality did William the conquerer found?", + "id": "5ad3a266604f3c001a3fea2b", + "answers": [], + "is_impossible": true + }], + "context": "The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands." + }] + }, { + "title": "Computational_complexity_theory", + "paragraphs": [{ + "qas": [{ + "question": "What branch of theoretical computer science deals with broadly classifying computational problems by difficulty and class of relationship?", + "id": "56e16182e3433e1400422e28", + "answers": [{ + "text": "Computational complexity theory", + "answer_start": 0 + }], + "is_impossible": false + }, { + "plausible_answers": [{ + "text": "algorithm", + "answer_start": 472 + }], + "question": "What is a manual application of mathematical steps?", + "id": "5ad5316b5b96ef001a10ab76", + "answers": [], + "is_impossible": true + }], + "context": "Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm." + }, { + "qas": [{ + "question": "What measure of a computational problem broadly defines the inherent difficulty of the solution?", + "id": "56e16839cd28a01900c67887", + "answers": [{ + "text": "if its solution requires significant resources", + "answer_start": 46 + }], + "is_impossible": false + }, { + "question": "What method is used to intuitively assess or quantify the amount of resources required to solve a computational problem?", + "id": "56e16839cd28a01900c67888", + "answers": [{ + "text": "mathematical models of computation", + "answer_start": 176 + }], + "is_impossible": false + }, { + "question": "What are two basic primary resources used to guage complexity?", + "id": "56e16839cd28a01900c67889", + "answers": [{ + "text": "time and storage", + "answer_start": 305 + }], + "is_impossible": false + }, { + "plausible_answers": [{ + "text": "the number of gates in a circuit", + "answer_start": 436 + }], + "question": "What unit is measured to determine circuit simplicity?", + "id": "5ad532575b96ef001a10ab7f", + "answers": [], + "is_impossible": true + }, { + "plausible_answers": [{ + "text": "the number of processors", + "answer_start": 502 + }], + "question": "What number is used in perpendicular computing?", + "id": "5ad532575b96ef001a10ab80", + "answers": [], + "is_impossible": true + }], + "context": "A problem is regarded as inherently difficult if its solution requires significant resources, whatever the algorithm used. The theory formalizes this intuition, by introducing mathematical models of computation to study these problems and quantifying the amount of resources needed to solve them, such as time and storage. Other complexity measures are also used, such as the amount of communication (used in communication complexity), the number of gates in a circuit (used in circuit complexity) and the number of processors (used in parallel computing). One of the roles of computational complexity theory is to determine the practical limits on what computers can and cannot do." + }] + }] +} \ No newline at end of file diff --git a/examples/utils_squad.py b/examples/utils_squad.py deleted file mode 100644 index c04dacf6..00000000 --- a/examples/utils_squad.py +++ /dev/null @@ -1,1016 +0,0 @@ - -# coding=utf-8 -# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. -# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Load SQuAD dataset. """ - -from __future__ import absolute_import, division, print_function - -import json -import logging -import math -import collections -from io import open -from tqdm import tqdm - -from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize - -# Required by XLNet evaluation method to compute optimal threshold (see write_predictions_extended() method) -from utils_squad_evaluate import find_all_best_thresh_v2, make_qid_to_has_ans, get_raw_scores - -logger = logging.getLogger(__name__) - - -class SquadExample(object): - """ - A single training/test example for the Squad dataset. - For examples without an answer, the start and end position are -1. - """ - - def __init__(self, - qas_id, - question_text, - doc_tokens, - orig_answer_text=None, - start_position=None, - end_position=None, - is_impossible=None): - self.qas_id = qas_id - self.question_text = question_text - self.doc_tokens = doc_tokens - self.orig_answer_text = orig_answer_text - self.start_position = start_position - self.end_position = end_position - self.is_impossible = is_impossible - - def __str__(self): - return self.__repr__() - - def __repr__(self): - s = "" - s += "qas_id: %s" % (self.qas_id) - s += ", question_text: %s" % ( - self.question_text) - s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) - if self.start_position: - s += ", start_position: %d" % (self.start_position) - if self.end_position: - s += ", end_position: %d" % (self.end_position) - if self.is_impossible: - s += ", is_impossible: %r" % (self.is_impossible) - return s - - -class InputFeatures(object): - """A single set of features of data.""" - - def __init__(self, - unique_id, - example_index, - doc_span_index, - tokens, - token_to_orig_map, - token_is_max_context, - input_ids, - input_mask, - segment_ids, - cls_index, - p_mask, - paragraph_len, - start_position=None, - end_position=None, - is_impossible=None): - self.unique_id = unique_id - self.example_index = example_index - self.doc_span_index = doc_span_index - self.tokens = tokens - self.token_to_orig_map = token_to_orig_map - self.token_is_max_context = token_is_max_context - self.input_ids = input_ids - self.input_mask = input_mask - self.segment_ids = segment_ids - self.cls_index = cls_index - self.p_mask = p_mask - self.paragraph_len = paragraph_len - self.start_position = start_position - self.end_position = end_position - self.is_impossible = is_impossible - - -def read_squad_examples(input_file, is_training, version_2_with_negative): - """Read a SQuAD json file into a list of SquadExample.""" - with open(input_file, "r", encoding='utf-8') as reader: - input_data = json.load(reader)["data"] - - def is_whitespace(c): - if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: - return True - return False - - examples = [] - for entry in input_data: - for paragraph in entry["paragraphs"]: - paragraph_text = paragraph["context"] - doc_tokens = [] - char_to_word_offset = [] - prev_is_whitespace = True - for c in paragraph_text: - if is_whitespace(c): - prev_is_whitespace = True - else: - if prev_is_whitespace: - doc_tokens.append(c) - else: - doc_tokens[-1] += c - prev_is_whitespace = False - char_to_word_offset.append(len(doc_tokens) - 1) - - for qa in paragraph["qas"]: - qas_id = qa["id"] - question_text = qa["question"] - start_position = None - end_position = None - orig_answer_text = None - is_impossible = False - if is_training: - if version_2_with_negative: - is_impossible = qa["is_impossible"] - if (len(qa["answers"]) != 1) and (not is_impossible): - raise ValueError( - "For training, each question should have exactly 1 answer.") - if not is_impossible: - answer = qa["answers"][0] - orig_answer_text = answer["text"] - answer_offset = answer["answer_start"] - answer_length = len(orig_answer_text) - start_position = char_to_word_offset[answer_offset] - end_position = char_to_word_offset[answer_offset + answer_length - 1] - # Only add answers where the text can be exactly recovered from the - # document. If this CAN'T happen it's likely due to weird Unicode - # stuff so we will just skip the example. - # - # Note that this means for training mode, every example is NOT - # guaranteed to be preserved. - actual_text = " ".join(doc_tokens[start_position:(end_position + 1)]) - cleaned_answer_text = " ".join( - whitespace_tokenize(orig_answer_text)) - if actual_text.find(cleaned_answer_text) == -1: - logger.warning("Could not find answer: '%s' vs. '%s'", - actual_text, cleaned_answer_text) - continue - else: - start_position = -1 - end_position = -1 - orig_answer_text = "" - - example = SquadExample( - qas_id=qas_id, - question_text=question_text, - doc_tokens=doc_tokens, - orig_answer_text=orig_answer_text, - start_position=start_position, - end_position=end_position, - is_impossible=is_impossible) - examples.append(example) - return examples - - -def convert_examples_to_features(examples, tokenizer, max_seq_length, - doc_stride, max_query_length, is_training, - cls_token_at_end=False, - cls_token='[CLS]', sep_token='[SEP]', pad_token=0, - sequence_a_segment_id=0, sequence_b_segment_id=1, - cls_token_segment_id=0, pad_token_segment_id=0, - mask_padding_with_zero=True, - sequence_a_is_doc=False): - """Loads a data file into a list of `InputBatch`s.""" - - unique_id = 1000000000 - # cnt_pos, cnt_neg = 0, 0 - # max_N, max_M = 1024, 1024 - # f = np.zeros((max_N, max_M), dtype=np.float32) - - features = [] - for (example_index, example) in enumerate(tqdm(examples)): - - # if example_index % 100 == 0: - # logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg) - - query_tokens = tokenizer.tokenize(example.question_text) - - if len(query_tokens) > max_query_length: - query_tokens = query_tokens[0:max_query_length] - - tok_to_orig_index = [] - orig_to_tok_index = [] - all_doc_tokens = [] - for (i, token) in enumerate(example.doc_tokens): - orig_to_tok_index.append(len(all_doc_tokens)) - sub_tokens = tokenizer.tokenize(token) - for sub_token in sub_tokens: - tok_to_orig_index.append(i) - all_doc_tokens.append(sub_token) - - tok_start_position = None - tok_end_position = None - if is_training and example.is_impossible: - tok_start_position = -1 - tok_end_position = -1 - if is_training and not example.is_impossible: - tok_start_position = orig_to_tok_index[example.start_position] - if example.end_position < len(example.doc_tokens) - 1: - tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 - else: - tok_end_position = len(all_doc_tokens) - 1 - (tok_start_position, tok_end_position) = _improve_answer_span( - all_doc_tokens, tok_start_position, tok_end_position, tokenizer, - example.orig_answer_text) - - # The -3 accounts for [CLS], [SEP] and [SEP] - max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 - - # We can have documents that are longer than the maximum sequence length. - # To deal with this we do a sliding window approach, where we take chunks - # of the up to our max length with a stride of `doc_stride`. - _DocSpan = collections.namedtuple( # pylint: disable=invalid-name - "DocSpan", ["start", "length"]) - doc_spans = [] - start_offset = 0 - while start_offset < len(all_doc_tokens): - length = len(all_doc_tokens) - start_offset - if length > max_tokens_for_doc: - length = max_tokens_for_doc - doc_spans.append(_DocSpan(start=start_offset, length=length)) - if start_offset + length == len(all_doc_tokens): - break - start_offset += min(length, doc_stride) - - for (doc_span_index, doc_span) in enumerate(doc_spans): - tokens = [] - token_to_orig_map = {} - token_is_max_context = {} - segment_ids = [] - - # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer) - # Original TF implem also keep the classification token (set to 0) (not sure why...) - p_mask = [] - - # CLS token at the beginning - if not cls_token_at_end: - tokens.append(cls_token) - segment_ids.append(cls_token_segment_id) - p_mask.append(0) - cls_index = 0 - - # XLNet: P SEP Q SEP CLS - # Others: CLS Q SEP P SEP - if not sequence_a_is_doc: - # Query - tokens += query_tokens - segment_ids += [sequence_a_segment_id] * len(query_tokens) - p_mask += [1] * len(query_tokens) - - # SEP token - tokens.append(sep_token) - segment_ids.append(sequence_a_segment_id) - p_mask.append(1) - - # Paragraph - for i in range(doc_span.length): - split_token_index = doc_span.start + i - token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] - - is_max_context = _check_is_max_context(doc_spans, doc_span_index, - split_token_index) - token_is_max_context[len(tokens)] = is_max_context - tokens.append(all_doc_tokens[split_token_index]) - if not sequence_a_is_doc: - segment_ids.append(sequence_b_segment_id) - else: - segment_ids.append(sequence_a_segment_id) - p_mask.append(0) - paragraph_len = doc_span.length - - if sequence_a_is_doc: - # SEP token - tokens.append(sep_token) - segment_ids.append(sequence_a_segment_id) - p_mask.append(1) - - tokens += query_tokens - segment_ids += [sequence_b_segment_id] * len(query_tokens) - p_mask += [1] * len(query_tokens) - - # SEP token - tokens.append(sep_token) - segment_ids.append(sequence_b_segment_id) - p_mask.append(1) - - # CLS token at the end - if cls_token_at_end: - tokens.append(cls_token) - segment_ids.append(cls_token_segment_id) - p_mask.append(0) - cls_index = len(tokens) - 1 # Index of classification token - - input_ids = tokenizer.convert_tokens_to_ids(tokens) - - # The mask has 1 for real tokens and 0 for padding tokens. Only real - # tokens are attended to. - input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) - - # Zero-pad up to the sequence length. - while len(input_ids) < max_seq_length: - input_ids.append(pad_token) - input_mask.append(0 if mask_padding_with_zero else 1) - segment_ids.append(pad_token_segment_id) - p_mask.append(1) - - assert len(input_ids) == max_seq_length - assert len(input_mask) == max_seq_length - assert len(segment_ids) == max_seq_length - - span_is_impossible = example.is_impossible - start_position = None - end_position = None - if is_training and not span_is_impossible: - # For training, if our document chunk does not contain an annotation - # we throw it out, since there is nothing to predict. - doc_start = doc_span.start - doc_end = doc_span.start + doc_span.length - 1 - out_of_span = False - if not (tok_start_position >= doc_start and - tok_end_position <= doc_end): - out_of_span = True - if out_of_span: - start_position = 0 - end_position = 0 - span_is_impossible = True - else: - if sequence_a_is_doc: - doc_offset = 0 - else: - doc_offset = len(query_tokens) + 2 - start_position = tok_start_position - doc_start + doc_offset - end_position = tok_end_position - doc_start + doc_offset - - if is_training and span_is_impossible: - start_position = cls_index - end_position = cls_index - - if example_index < 20: - logger.info("*** Example ***") - logger.info("unique_id: %s" % (unique_id)) - logger.info("example_index: %s" % (example_index)) - logger.info("doc_span_index: %s" % (doc_span_index)) - logger.info("tokens: %s" % " ".join(tokens)) - logger.info("token_to_orig_map: %s" % " ".join([ - "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()])) - logger.info("token_is_max_context: %s" % " ".join([ - "%d:%s" % (x, y) for (x, y) in token_is_max_context.items() - ])) - logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) - logger.info( - "input_mask: %s" % " ".join([str(x) for x in input_mask])) - logger.info( - "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) - if is_training and span_is_impossible: - logger.info("impossible example") - if is_training and not span_is_impossible: - answer_text = " ".join(tokens[start_position:(end_position + 1)]) - logger.info("start_position: %d" % (start_position)) - logger.info("end_position: %d" % (end_position)) - logger.info( - "answer: %s" % (answer_text)) - - features.append( - InputFeatures( - unique_id=unique_id, - example_index=example_index, - doc_span_index=doc_span_index, - tokens=tokens, - token_to_orig_map=token_to_orig_map, - token_is_max_context=token_is_max_context, - input_ids=input_ids, - input_mask=input_mask, - segment_ids=segment_ids, - cls_index=cls_index, - p_mask=p_mask, - paragraph_len=paragraph_len, - start_position=start_position, - end_position=end_position, - is_impossible=span_is_impossible)) - unique_id += 1 - - return features - - -def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, - orig_answer_text): - """Returns tokenized answer spans that better match the annotated answer.""" - - # The SQuAD annotations are character based. We first project them to - # whitespace-tokenized words. But then after WordPiece tokenization, we can - # often find a "better match". For example: - # - # Question: What year was John Smith born? - # Context: The leader was John Smith (1895-1943). - # Answer: 1895 - # - # The original whitespace-tokenized answer will be "(1895-1943).". However - # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match - # the exact answer, 1895. - # - # However, this is not always possible. Consider the following: - # - # Question: What country is the top exporter of electornics? - # Context: The Japanese electronics industry is the lagest in the world. - # Answer: Japan - # - # In this case, the annotator chose "Japan" as a character sub-span of - # the word "Japanese". Since our WordPiece tokenizer does not split - # "Japanese", we just use "Japanese" as the annotation. This is fairly rare - # in SQuAD, but does happen. - tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) - - for new_start in range(input_start, input_end + 1): - for new_end in range(input_end, new_start - 1, -1): - text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) - if text_span == tok_answer_text: - return (new_start, new_end) - - return (input_start, input_end) - - -def _check_is_max_context(doc_spans, cur_span_index, position): - """Check if this is the 'max context' doc span for the token.""" - - # Because of the sliding window approach taken to scoring documents, a single - # token can appear in multiple documents. E.g. - # Doc: the man went to the store and bought a gallon of milk - # Span A: the man went to the - # Span B: to the store and bought - # Span C: and bought a gallon of - # ... - # - # Now the word 'bought' will have two scores from spans B and C. We only - # want to consider the score with "maximum context", which we define as - # the *minimum* of its left and right context (the *sum* of left and - # right context will always be the same, of course). - # - # In the example the maximum context for 'bought' would be span C since - # it has 1 left context and 3 right context, while span B has 4 left context - # and 0 right context. - best_score = None - best_span_index = None - for (span_index, doc_span) in enumerate(doc_spans): - end = doc_span.start + doc_span.length - 1 - if position < doc_span.start: - continue - if position > end: - continue - num_left_context = position - doc_span.start - num_right_context = end - position - score = min(num_left_context, num_right_context) + 0.01 * doc_span.length - if best_score is None or score > best_score: - best_score = score - best_span_index = span_index - - return cur_span_index == best_span_index - - -RawResult = collections.namedtuple("RawResult", - ["unique_id", "start_logits", "end_logits"]) - -def write_predictions(all_examples, all_features, all_results, n_best_size, - max_answer_length, do_lower_case, output_prediction_file, - output_nbest_file, output_null_log_odds_file, verbose_logging, - version_2_with_negative, null_score_diff_threshold): - """Write final predictions to the json file and log-odds of null if needed.""" - logger.info("Writing predictions to: %s" % (output_prediction_file)) - logger.info("Writing nbest to: %s" % (output_nbest_file)) - - example_index_to_features = collections.defaultdict(list) - for feature in all_features: - example_index_to_features[feature.example_index].append(feature) - - unique_id_to_result = {} - for result in all_results: - unique_id_to_result[result.unique_id] = result - - _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name - "PrelimPrediction", - ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) - - all_predictions = collections.OrderedDict() - all_nbest_json = collections.OrderedDict() - scores_diff_json = collections.OrderedDict() - - for (example_index, example) in enumerate(all_examples): - features = example_index_to_features[example_index] - - prelim_predictions = [] - # keep track of the minimum score of null start+end of position 0 - score_null = 1000000 # large and positive - min_null_feature_index = 0 # the paragraph slice with min null score - null_start_logit = 0 # the start logit at the slice with min null score - null_end_logit = 0 # the end logit at the slice with min null score - for (feature_index, feature) in enumerate(features): - result = unique_id_to_result[feature.unique_id] - start_indexes = _get_best_indexes(result.start_logits, n_best_size) - end_indexes = _get_best_indexes(result.end_logits, n_best_size) - # if we could have irrelevant answers, get the min score of irrelevant - if version_2_with_negative: - feature_null_score = result.start_logits[0] + result.end_logits[0] - if feature_null_score < score_null: - score_null = feature_null_score - min_null_feature_index = feature_index - null_start_logit = result.start_logits[0] - null_end_logit = result.end_logits[0] - for start_index in start_indexes: - for end_index in end_indexes: - # We could hypothetically create invalid predictions, e.g., predict - # that the start of the span is in the question. We throw out all - # invalid predictions. - if start_index >= len(feature.tokens): - continue - if end_index >= len(feature.tokens): - continue - if start_index not in feature.token_to_orig_map: - continue - if end_index not in feature.token_to_orig_map: - continue - if not feature.token_is_max_context.get(start_index, False): - continue - if end_index < start_index: - continue - length = end_index - start_index + 1 - if length > max_answer_length: - continue - prelim_predictions.append( - _PrelimPrediction( - feature_index=feature_index, - start_index=start_index, - end_index=end_index, - start_logit=result.start_logits[start_index], - end_logit=result.end_logits[end_index])) - if version_2_with_negative: - prelim_predictions.append( - _PrelimPrediction( - feature_index=min_null_feature_index, - start_index=0, - end_index=0, - start_logit=null_start_logit, - end_logit=null_end_logit)) - prelim_predictions = sorted( - prelim_predictions, - key=lambda x: (x.start_logit + x.end_logit), - reverse=True) - - _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name - "NbestPrediction", ["text", "start_logit", "end_logit"]) - - seen_predictions = {} - nbest = [] - for pred in prelim_predictions: - if len(nbest) >= n_best_size: - break - feature = features[pred.feature_index] - if pred.start_index > 0: # this is a non-null prediction - tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] - orig_doc_start = feature.token_to_orig_map[pred.start_index] - orig_doc_end = feature.token_to_orig_map[pred.end_index] - orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] - tok_text = " ".join(tok_tokens) - - # De-tokenize WordPieces that have been split off. - tok_text = tok_text.replace(" ##", "") - tok_text = tok_text.replace("##", "") - - # Clean whitespace - tok_text = tok_text.strip() - tok_text = " ".join(tok_text.split()) - orig_text = " ".join(orig_tokens) - - final_text = get_final_text(tok_text, orig_text, do_lower_case, verbose_logging) - if final_text in seen_predictions: - continue - - seen_predictions[final_text] = True - else: - final_text = "" - seen_predictions[final_text] = True - - nbest.append( - _NbestPrediction( - text=final_text, - start_logit=pred.start_logit, - end_logit=pred.end_logit)) - # if we didn't include the empty option in the n-best, include it - if version_2_with_negative: - if "" not in seen_predictions: - nbest.append( - _NbestPrediction( - text="", - start_logit=null_start_logit, - end_logit=null_end_logit)) - - # In very rare edge cases we could only have single null prediction. - # So we just create a nonce prediction in this case to avoid failure. - if len(nbest)==1: - nbest.insert(0, - _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) - - # In very rare edge cases we could have no valid predictions. So we - # just create a nonce prediction in this case to avoid failure. - if not nbest: - nbest.append( - _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) - - assert len(nbest) >= 1 - - total_scores = [] - best_non_null_entry = None - for entry in nbest: - total_scores.append(entry.start_logit + entry.end_logit) - if not best_non_null_entry: - if entry.text: - best_non_null_entry = entry - - probs = _compute_softmax(total_scores) - - nbest_json = [] - for (i, entry) in enumerate(nbest): - output = collections.OrderedDict() - output["text"] = entry.text - output["probability"] = probs[i] - output["start_logit"] = entry.start_logit - output["end_logit"] = entry.end_logit - nbest_json.append(output) - - assert len(nbest_json) >= 1 - - if not version_2_with_negative: - all_predictions[example.qas_id] = nbest_json[0]["text"] - else: - # predict "" iff the null score - the score of best non-null > threshold - score_diff = score_null - best_non_null_entry.start_logit - ( - best_non_null_entry.end_logit) - scores_diff_json[example.qas_id] = score_diff - if score_diff > null_score_diff_threshold: - all_predictions[example.qas_id] = "" - else: - all_predictions[example.qas_id] = best_non_null_entry.text - all_nbest_json[example.qas_id] = nbest_json - - with open(output_prediction_file, "w") as writer: - writer.write(json.dumps(all_predictions, indent=4) + "\n") - - with open(output_nbest_file, "w") as writer: - writer.write(json.dumps(all_nbest_json, indent=4) + "\n") - - if version_2_with_negative: - with open(output_null_log_odds_file, "w") as writer: - writer.write(json.dumps(scores_diff_json, indent=4) + "\n") - - return all_predictions - - -# For XLNet (and XLM which uses the same head) -RawResultExtended = collections.namedtuple("RawResultExtended", - ["unique_id", "start_top_log_probs", "start_top_index", - "end_top_log_probs", "end_top_index", "cls_logits"]) - - -def write_predictions_extended(all_examples, all_features, all_results, n_best_size, - max_answer_length, output_prediction_file, - output_nbest_file, - output_null_log_odds_file, orig_data_file, - start_n_top, end_n_top, version_2_with_negative, - tokenizer, verbose_logging): - """ XLNet write prediction logic (more complex than Bert's). - Write final predictions to the json file and log-odds of null if needed. - - Requires utils_squad_evaluate.py - """ - _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name - "PrelimPrediction", - ["feature_index", "start_index", "end_index", - "start_log_prob", "end_log_prob"]) - - _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name - "NbestPrediction", ["text", "start_log_prob", "end_log_prob"]) - - logger.info("Writing predictions to: %s", output_prediction_file) - # logger.info("Writing nbest to: %s" % (output_nbest_file)) - - example_index_to_features = collections.defaultdict(list) - for feature in all_features: - example_index_to_features[feature.example_index].append(feature) - - unique_id_to_result = {} - for result in all_results: - unique_id_to_result[result.unique_id] = result - - all_predictions = collections.OrderedDict() - all_nbest_json = collections.OrderedDict() - scores_diff_json = collections.OrderedDict() - - for (example_index, example) in enumerate(all_examples): - features = example_index_to_features[example_index] - - prelim_predictions = [] - # keep track of the minimum score of null start+end of position 0 - score_null = 1000000 # large and positive - - for (feature_index, feature) in enumerate(features): - result = unique_id_to_result[feature.unique_id] - - cur_null_score = result.cls_logits - - # if we could have irrelevant answers, get the min score of irrelevant - score_null = min(score_null, cur_null_score) - - for i in range(start_n_top): - for j in range(end_n_top): - start_log_prob = result.start_top_log_probs[i] - start_index = result.start_top_index[i] - - j_index = i * end_n_top + j - - end_log_prob = result.end_top_log_probs[j_index] - end_index = result.end_top_index[j_index] - - # We could hypothetically create invalid predictions, e.g., predict - # that the start of the span is in the question. We throw out all - # invalid predictions. - if start_index >= feature.paragraph_len - 1: - continue - if end_index >= feature.paragraph_len - 1: - continue - - if not feature.token_is_max_context.get(start_index, False): - continue - if end_index < start_index: - continue - length = end_index - start_index + 1 - if length > max_answer_length: - continue - - prelim_predictions.append( - _PrelimPrediction( - feature_index=feature_index, - start_index=start_index, - end_index=end_index, - start_log_prob=start_log_prob, - end_log_prob=end_log_prob)) - - prelim_predictions = sorted( - prelim_predictions, - key=lambda x: (x.start_log_prob + x.end_log_prob), - reverse=True) - - seen_predictions = {} - nbest = [] - for pred in prelim_predictions: - if len(nbest) >= n_best_size: - break - feature = features[pred.feature_index] - - # XLNet un-tokenizer - # Let's keep it simple for now and see if we need all this later. - # - # tok_start_to_orig_index = feature.tok_start_to_orig_index - # tok_end_to_orig_index = feature.tok_end_to_orig_index - # start_orig_pos = tok_start_to_orig_index[pred.start_index] - # end_orig_pos = tok_end_to_orig_index[pred.end_index] - # paragraph_text = example.paragraph_text - # final_text = paragraph_text[start_orig_pos: end_orig_pos + 1].strip() - - # Previously used Bert untokenizer - tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] - orig_doc_start = feature.token_to_orig_map[pred.start_index] - orig_doc_end = feature.token_to_orig_map[pred.end_index] - orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] - tok_text = tokenizer.convert_tokens_to_string(tok_tokens) - - # Clean whitespace - tok_text = tok_text.strip() - tok_text = " ".join(tok_text.split()) - orig_text = " ".join(orig_tokens) - - final_text = get_final_text(tok_text, orig_text, tokenizer.do_lower_case, - verbose_logging) - - if final_text in seen_predictions: - continue - - seen_predictions[final_text] = True - - nbest.append( - _NbestPrediction( - text=final_text, - start_log_prob=pred.start_log_prob, - end_log_prob=pred.end_log_prob)) - - # In very rare edge cases we could have no valid predictions. So we - # just create a nonce prediction in this case to avoid failure. - if not nbest: - nbest.append( - _NbestPrediction(text="", start_log_prob=-1e6, - end_log_prob=-1e6)) - - total_scores = [] - best_non_null_entry = None - for entry in nbest: - total_scores.append(entry.start_log_prob + entry.end_log_prob) - if not best_non_null_entry: - best_non_null_entry = entry - - probs = _compute_softmax(total_scores) - - nbest_json = [] - for (i, entry) in enumerate(nbest): - output = collections.OrderedDict() - output["text"] = entry.text - output["probability"] = probs[i] - output["start_log_prob"] = entry.start_log_prob - output["end_log_prob"] = entry.end_log_prob - nbest_json.append(output) - - assert len(nbest_json) >= 1 - assert best_non_null_entry is not None - - score_diff = score_null - scores_diff_json[example.qas_id] = score_diff - # note(zhiliny): always predict best_non_null_entry - # and the evaluation script will search for the best threshold - all_predictions[example.qas_id] = best_non_null_entry.text - - all_nbest_json[example.qas_id] = nbest_json - - with open(output_prediction_file, "w") as writer: - writer.write(json.dumps(all_predictions, indent=4) + "\n") - - with open(output_nbest_file, "w") as writer: - writer.write(json.dumps(all_nbest_json, indent=4) + "\n") - - if version_2_with_negative: - with open(output_null_log_odds_file, "w") as writer: - writer.write(json.dumps(scores_diff_json, indent=4) + "\n") - - with open(orig_data_file, "r", encoding='utf-8') as reader: - orig_data = json.load(reader)["data"] - - qid_to_has_ans = make_qid_to_has_ans(orig_data) - has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] - no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] - exact_raw, f1_raw = get_raw_scores(orig_data, all_predictions) - out_eval = {} - - find_all_best_thresh_v2(out_eval, all_predictions, exact_raw, f1_raw, scores_diff_json, qid_to_has_ans) - - return out_eval - - -def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False): - """Project the tokenized prediction back to the original text.""" - - # When we created the data, we kept track of the alignment between original - # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So - # now `orig_text` contains the span of our original text corresponding to the - # span that we predicted. - # - # However, `orig_text` may contain extra characters that we don't want in - # our prediction. - # - # For example, let's say: - # pred_text = steve smith - # orig_text = Steve Smith's - # - # We don't want to return `orig_text` because it contains the extra "'s". - # - # We don't want to return `pred_text` because it's already been normalized - # (the SQuAD eval script also does punctuation stripping/lower casing but - # our tokenizer does additional normalization like stripping accent - # characters). - # - # What we really want to return is "Steve Smith". - # - # Therefore, we have to apply a semi-complicated alignment heuristic between - # `pred_text` and `orig_text` to get a character-to-character alignment. This - # can fail in certain cases in which case we just return `orig_text`. - - def _strip_spaces(text): - ns_chars = [] - ns_to_s_map = collections.OrderedDict() - for (i, c) in enumerate(text): - if c == " ": - continue - ns_to_s_map[len(ns_chars)] = i - ns_chars.append(c) - ns_text = "".join(ns_chars) - return (ns_text, ns_to_s_map) - - # We first tokenize `orig_text`, strip whitespace from the result - # and `pred_text`, and check if they are the same length. If they are - # NOT the same length, the heuristic has failed. If they are the same - # length, we assume the characters are one-to-one aligned. - tokenizer = BasicTokenizer(do_lower_case=do_lower_case) - - tok_text = " ".join(tokenizer.tokenize(orig_text)) - - start_position = tok_text.find(pred_text) - if start_position == -1: - if verbose_logging: - logger.info( - "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) - return orig_text - end_position = start_position + len(pred_text) - 1 - - (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) - (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) - - if len(orig_ns_text) != len(tok_ns_text): - if verbose_logging: - logger.info("Length not equal after stripping spaces: '%s' vs '%s'", - orig_ns_text, tok_ns_text) - return orig_text - - # We then project the characters in `pred_text` back to `orig_text` using - # the character-to-character alignment. - tok_s_to_ns_map = {} - for (i, tok_index) in tok_ns_to_s_map.items(): - tok_s_to_ns_map[tok_index] = i - - orig_start_position = None - if start_position in tok_s_to_ns_map: - ns_start_position = tok_s_to_ns_map[start_position] - if ns_start_position in orig_ns_to_s_map: - orig_start_position = orig_ns_to_s_map[ns_start_position] - - if orig_start_position is None: - if verbose_logging: - logger.info("Couldn't map start position") - return orig_text - - orig_end_position = None - if end_position in tok_s_to_ns_map: - ns_end_position = tok_s_to_ns_map[end_position] - if ns_end_position in orig_ns_to_s_map: - orig_end_position = orig_ns_to_s_map[ns_end_position] - - if orig_end_position is None: - if verbose_logging: - logger.info("Couldn't map end position") - return orig_text - - output_text = orig_text[orig_start_position:(orig_end_position + 1)] - return output_text - - -def _get_best_indexes(logits, n_best_size): - """Get the n-best logits from a list.""" - index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) - - best_indexes = [] - for i in range(len(index_and_score)): - if i >= n_best_size: - break - best_indexes.append(index_and_score[i][0]) - return best_indexes - - -def _compute_softmax(scores): - """Compute softmax probability over raw logits.""" - if not scores: - return [] - - max_score = None - for score in scores: - if max_score is None or score > max_score: - max_score = score - - exp_scores = [] - total_sum = 0.0 - for score in scores: - x = math.exp(score - max_score) - exp_scores.append(x) - total_sum += x - - probs = [] - for score in exp_scores: - probs.append(score / total_sum) - return probs diff --git a/examples/utils_squad_evaluate.py b/examples/utils_squad_evaluate.py deleted file mode 100644 index ed162e6f..00000000 --- a/examples/utils_squad_evaluate.py +++ /dev/null @@ -1,330 +0,0 @@ -""" Official evaluation script for SQuAD version 2.0. - Modified by XLNet authors to update `find_best_threshold` scripts for SQuAD V2.0 - -In addition to basic functionality, we also compute additional statistics and -plot precision-recall curves if an additional na_prob.json file is provided. -This file is expected to map question ID's to the model's predicted probability -that a question is unanswerable. -""" -import argparse -import collections -import json -import numpy as np -import os -import re -import string -import sys - -class EVAL_OPTS(): - def __init__(self, data_file, pred_file, out_file="", - na_prob_file="na_prob.json", na_prob_thresh=1.0, - out_image_dir=None, verbose=False): - self.data_file = data_file - self.pred_file = pred_file - self.out_file = out_file - self.na_prob_file = na_prob_file - self.na_prob_thresh = na_prob_thresh - self.out_image_dir = out_image_dir - self.verbose = verbose - -OPTS = None - -def parse_args(): - parser = argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.') - parser.add_argument('data_file', metavar='data.json', help='Input data JSON file.') - parser.add_argument('pred_file', metavar='pred.json', help='Model predictions.') - parser.add_argument('--out-file', '-o', metavar='eval.json', - help='Write accuracy metrics to file (default is stdout).') - parser.add_argument('--na-prob-file', '-n', metavar='na_prob.json', - help='Model estimates of probability of no answer.') - parser.add_argument('--na-prob-thresh', '-t', type=float, default=1.0, - help='Predict "" if no-answer probability exceeds this (default = 1.0).') - parser.add_argument('--out-image-dir', '-p', metavar='out_images', default=None, - help='Save precision-recall curves to directory.') - parser.add_argument('--verbose', '-v', action='store_true') - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - return parser.parse_args() - -def make_qid_to_has_ans(dataset): - qid_to_has_ans = {} - for article in dataset: - for p in article['paragraphs']: - for qa in p['qas']: - qid_to_has_ans[qa['id']] = bool(qa['answers']) - return qid_to_has_ans - -def normalize_answer(s): - """Lower text and remove punctuation, articles and extra whitespace.""" - def remove_articles(text): - regex = re.compile(r'\b(a|an|the)\b', re.UNICODE) - return re.sub(regex, ' ', text) - def white_space_fix(text): - return ' '.join(text.split()) - def remove_punc(text): - exclude = set(string.punctuation) - return ''.join(ch for ch in text if ch not in exclude) - def lower(text): - return text.lower() - return white_space_fix(remove_articles(remove_punc(lower(s)))) - -def get_tokens(s): - if not s: return [] - return normalize_answer(s).split() - -def compute_exact(a_gold, a_pred): - return int(normalize_answer(a_gold) == normalize_answer(a_pred)) - -def compute_f1(a_gold, a_pred): - gold_toks = get_tokens(a_gold) - pred_toks = get_tokens(a_pred) - common = collections.Counter(gold_toks) & collections.Counter(pred_toks) - num_same = sum(common.values()) - if len(gold_toks) == 0 or len(pred_toks) == 0: - # If either is no-answer, then F1 is 1 if they agree, 0 otherwise - return int(gold_toks == pred_toks) - if num_same == 0: - return 0 - precision = 1.0 * num_same / len(pred_toks) - recall = 1.0 * num_same / len(gold_toks) - f1 = (2 * precision * recall) / (precision + recall) - return f1 - -def get_raw_scores(dataset, preds): - exact_scores = {} - f1_scores = {} - for article in dataset: - for p in article['paragraphs']: - for qa in p['qas']: - qid = qa['id'] - gold_answers = [a['text'] for a in qa['answers'] - if normalize_answer(a['text'])] - if not gold_answers: - # For unanswerable questions, only correct answer is empty string - gold_answers = [''] - if qid not in preds: - print('Missing prediction for %s' % qid) - continue - a_pred = preds[qid] - # Take max over all gold answers - exact_scores[qid] = max(compute_exact(a, a_pred) for a in gold_answers) - f1_scores[qid] = max(compute_f1(a, a_pred) for a in gold_answers) - return exact_scores, f1_scores - -def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh): - new_scores = {} - for qid, s in scores.items(): - pred_na = na_probs[qid] > na_prob_thresh - if pred_na: - new_scores[qid] = float(not qid_to_has_ans[qid]) - else: - new_scores[qid] = s - return new_scores - -def make_eval_dict(exact_scores, f1_scores, qid_list=None): - if not qid_list: - total = len(exact_scores) - return collections.OrderedDict([ - ('exact', 100.0 * sum(exact_scores.values()) / total), - ('f1', 100.0 * sum(f1_scores.values()) / total), - ('total', total), - ]) - else: - total = len(qid_list) - return collections.OrderedDict([ - ('exact', 100.0 * sum(exact_scores[k] for k in qid_list) / total), - ('f1', 100.0 * sum(f1_scores[k] for k in qid_list) / total), - ('total', total), - ]) - -def merge_eval(main_eval, new_eval, prefix): - for k in new_eval: - main_eval['%s_%s' % (prefix, k)] = new_eval[k] - -def plot_pr_curve(precisions, recalls, out_image, title): - plt.step(recalls, precisions, color='b', alpha=0.2, where='post') - plt.fill_between(recalls, precisions, step='post', alpha=0.2, color='b') - plt.xlabel('Recall') - plt.ylabel('Precision') - plt.xlim([0.0, 1.05]) - plt.ylim([0.0, 1.05]) - plt.title(title) - plt.savefig(out_image) - plt.clf() - -def make_precision_recall_eval(scores, na_probs, num_true_pos, qid_to_has_ans, - out_image=None, title=None): - qid_list = sorted(na_probs, key=lambda k: na_probs[k]) - true_pos = 0.0 - cur_p = 1.0 - cur_r = 0.0 - precisions = [1.0] - recalls = [0.0] - avg_prec = 0.0 - for i, qid in enumerate(qid_list): - if qid_to_has_ans[qid]: - true_pos += scores[qid] - cur_p = true_pos / float(i+1) - cur_r = true_pos / float(num_true_pos) - if i == len(qid_list) - 1 or na_probs[qid] != na_probs[qid_list[i+1]]: - # i.e., if we can put a threshold after this point - avg_prec += cur_p * (cur_r - recalls[-1]) - precisions.append(cur_p) - recalls.append(cur_r) - if out_image: - plot_pr_curve(precisions, recalls, out_image, title) - return {'ap': 100.0 * avg_prec} - -def run_precision_recall_analysis(main_eval, exact_raw, f1_raw, na_probs, - qid_to_has_ans, out_image_dir): - if out_image_dir and not os.path.exists(out_image_dir): - os.makedirs(out_image_dir) - num_true_pos = sum(1 for v in qid_to_has_ans.values() if v) - if num_true_pos == 0: - return - pr_exact = make_precision_recall_eval( - exact_raw, na_probs, num_true_pos, qid_to_has_ans, - out_image=os.path.join(out_image_dir, 'pr_exact.png'), - title='Precision-Recall curve for Exact Match score') - pr_f1 = make_precision_recall_eval( - f1_raw, na_probs, num_true_pos, qid_to_has_ans, - out_image=os.path.join(out_image_dir, 'pr_f1.png'), - title='Precision-Recall curve for F1 score') - oracle_scores = {k: float(v) for k, v in qid_to_has_ans.items()} - pr_oracle = make_precision_recall_eval( - oracle_scores, na_probs, num_true_pos, qid_to_has_ans, - out_image=os.path.join(out_image_dir, 'pr_oracle.png'), - title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)') - merge_eval(main_eval, pr_exact, 'pr_exact') - merge_eval(main_eval, pr_f1, 'pr_f1') - merge_eval(main_eval, pr_oracle, 'pr_oracle') - -def histogram_na_prob(na_probs, qid_list, image_dir, name): - if not qid_list: - return - x = [na_probs[k] for k in qid_list] - weights = np.ones_like(x) / float(len(x)) - plt.hist(x, weights=weights, bins=20, range=(0.0, 1.0)) - plt.xlabel('Model probability of no-answer') - plt.ylabel('Proportion of dataset') - plt.title('Histogram of no-answer probability: %s' % name) - plt.savefig(os.path.join(image_dir, 'na_prob_hist_%s.png' % name)) - plt.clf() - -def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): - num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) - cur_score = num_no_ans - best_score = cur_score - best_thresh = 0.0 - qid_list = sorted(na_probs, key=lambda k: na_probs[k]) - for i, qid in enumerate(qid_list): - if qid not in scores: continue - if qid_to_has_ans[qid]: - diff = scores[qid] - else: - if preds[qid]: - diff = -1 - else: - diff = 0 - cur_score += diff - if cur_score > best_score: - best_score = cur_score - best_thresh = na_probs[qid] - return 100.0 * best_score / len(scores), best_thresh - -def find_best_thresh_v2(preds, scores, na_probs, qid_to_has_ans): - num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) - cur_score = num_no_ans - best_score = cur_score - best_thresh = 0.0 - qid_list = sorted(na_probs, key=lambda k: na_probs[k]) - for i, qid in enumerate(qid_list): - if qid not in scores: continue - if qid_to_has_ans[qid]: - diff = scores[qid] - else: - if preds[qid]: - diff = -1 - else: - diff = 0 - cur_score += diff - if cur_score > best_score: - best_score = cur_score - best_thresh = na_probs[qid] - - has_ans_score, has_ans_cnt = 0, 0 - for qid in qid_list: - if not qid_to_has_ans[qid]: continue - has_ans_cnt += 1 - - if qid not in scores: continue - has_ans_score += scores[qid] - - return 100.0 * best_score / len(scores), best_thresh, 1.0 * has_ans_score / has_ans_cnt - -def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): - best_exact, exact_thresh = find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans) - best_f1, f1_thresh = find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans) - main_eval['best_exact'] = best_exact - main_eval['best_exact_thresh'] = exact_thresh - main_eval['best_f1'] = best_f1 - main_eval['best_f1_thresh'] = f1_thresh - -def find_all_best_thresh_v2(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): - best_exact, exact_thresh, has_ans_exact = find_best_thresh_v2(preds, exact_raw, na_probs, qid_to_has_ans) - best_f1, f1_thresh, has_ans_f1 = find_best_thresh_v2(preds, f1_raw, na_probs, qid_to_has_ans) - main_eval['best_exact'] = best_exact - main_eval['best_exact_thresh'] = exact_thresh - main_eval['best_f1'] = best_f1 - main_eval['best_f1_thresh'] = f1_thresh - main_eval['has_ans_exact'] = has_ans_exact - main_eval['has_ans_f1'] = has_ans_f1 - -def main(OPTS): - with open(OPTS.data_file) as f: - dataset_json = json.load(f) - dataset = dataset_json['data'] - with open(OPTS.pred_file) as f: - preds = json.load(f) - if OPTS.na_prob_file: - with open(OPTS.na_prob_file) as f: - na_probs = json.load(f) - else: - na_probs = {k: 0.0 for k in preds} - qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False - has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] - no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] - exact_raw, f1_raw = get_raw_scores(dataset, preds) - exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans, - OPTS.na_prob_thresh) - f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans, - OPTS.na_prob_thresh) - out_eval = make_eval_dict(exact_thresh, f1_thresh) - if has_ans_qids: - has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids) - merge_eval(out_eval, has_ans_eval, 'HasAns') - if no_ans_qids: - no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) - merge_eval(out_eval, no_ans_eval, 'NoAns') - if OPTS.na_prob_file: - find_all_best_thresh(out_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans) - if OPTS.na_prob_file and OPTS.out_image_dir: - run_precision_recall_analysis(out_eval, exact_raw, f1_raw, na_probs, - qid_to_has_ans, OPTS.out_image_dir) - histogram_na_prob(na_probs, has_ans_qids, OPTS.out_image_dir, 'hasAns') - histogram_na_prob(na_probs, no_ans_qids, OPTS.out_image_dir, 'noAns') - if OPTS.out_file: - with open(OPTS.out_file, 'w') as f: - json.dump(out_eval, f) - else: - print(json.dumps(out_eval, indent=2)) - return out_eval - -if __name__ == '__main__': - OPTS = parse_args() - if OPTS.out_image_dir: - import matplotlib - matplotlib.use('Agg') - import matplotlib.pyplot as plt - main(OPTS) diff --git a/requirements.txt b/requirements.txt index 9c43abc6..32edee07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ boto3 # Used for downloading models over HTTP requests # For OpenAI GPT -regex +regex != 2019.12.17 # For XLNet sentencepiece # For XLM diff --git a/setup.py b/setup.py index f49aee68..fe2e1526 100644 --- a/setup.py +++ b/setup.py @@ -36,9 +36,17 @@ To create the package for pypi. from io import open from setuptools import find_packages, setup + +extras = { + 'serving': ['pydantic', 'uvicorn', 'fastapi'], + 'serving-tf': ['pydantic', 'uvicorn', 'fastapi', 'tensorflow'], + 'serving-torch': ['pydantic', 'uvicorn', 'fastapi', 'torch'] +} +extras['all'] = [package for package in extras.values()] + setup( name="transformers", - version="2.1.1", + version="2.3.0", author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors", author_email="thomas@huggingface.co", description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch", @@ -51,18 +59,17 @@ setup( "tests.*", "tests"]), install_requires=['numpy', 'boto3', + 'filelock', 'requests', 'tqdm', - 'regex', + 'regex != 2019.12.17', 'sentencepiece', 'sacremoses'], - entry_points={ - 'console_scripts': [ - "transformers=transformers.__main__:main", - ] - }, + extras_require=extras, + scripts=[ + 'transformers-cli' + ], # python_requires='>=3.5.0', - tests_require=['pytest'], classifiers=[ 'Intended Audience :: Science/Research', 'License :: OSI Approved :: Apache Software License', diff --git a/templates/adding_a_new_model/configuration_xxx.py b/templates/adding_a_new_model/configuration_xxx.py index b1614e71..12d69799 100644 --- a/templates/adding_a_new_model/configuration_xxx.py +++ b/templates/adding_a_new_model/configuration_xxx.py @@ -39,7 +39,7 @@ class XxxConfig(PretrainedConfig): Arguments: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `XxxModel`. + vocab_size: Vocabulary size of `inputs_ids` in `XxxModel`. hidden_size: Size of the encoder layers and the pooler layer. num_hidden_layers: Number of hidden layers in the Transformer encoder. num_attention_heads: Number of attention heads for each attention layer in @@ -64,7 +64,7 @@ class XxxConfig(PretrainedConfig): pretrained_config_archive_map = XXX_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=50257, + vocab_size=50257, n_positions=1024, n_ctx=1024, n_embd=768, @@ -75,8 +75,6 @@ class XxxConfig(PretrainedConfig): attn_pdrop=0.1, layer_norm_epsilon=1e-5, initializer_range=0.02, - - num_labels=1, summary_type='cls_index', summary_use_proj=True, summary_activation=None, @@ -84,7 +82,7 @@ class XxxConfig(PretrainedConfig): summary_first_dropout=0.1, **kwargs): super(XxxConfig, self).__init__(**kwargs) - self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, six.string_types) else -1 + self.vocab_size = vocab_size self.n_ctx = n_ctx self.n_positions = n_positions self.n_embd = n_embd @@ -95,23 +93,11 @@ class XxxConfig(PretrainedConfig): self.attn_pdrop = attn_pdrop self.layer_norm_epsilon = layer_norm_epsilon self.initializer_range = initializer_range - - self.num_labels = num_labels self.summary_type = summary_type self.summary_use_proj = summary_use_proj self.summary_activation = summary_activation self.summary_first_dropout = summary_first_dropout self.summary_proj_to_labels = summary_proj_to_labels - if isinstance(vocab_size_or_config_json_file, six.string_types): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif not isinstance(vocab_size_or_config_json_file, int): - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) @property def max_position_embeddings(self): diff --git a/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py b/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py index d50d129c..9d389dea 100755 --- a/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py +++ b/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py @@ -26,9 +26,9 @@ from transformers import XxxConfig, XxxForPreTraining, load_tf_weights_in_xxx import logging logging.basicConfig(level=logging.INFO) -def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, xxx_config_file, pytorch_dump_path): +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, config_file, pytorch_dump_path): # Initialise PyTorch model - config = XxxConfig.from_json_file(xxx_config_file) + config = XxxConfig.from_json_file(config_file) print("Building PyTorch model from configuration: {}".format(str(config))) model = XxxForPreTraining(config) @@ -48,11 +48,11 @@ if __name__ == "__main__": type = str, required = True, help = "Path to the TensorFlow checkpoint path.") - parser.add_argument("--xxx_config_file", + parser.add_argument("--config_file", default = None, type = str, required = True, - help = "The config json file corresponding to the pre-trained XXX model. \n" + help = "The config json file corresponding to the pre-trained model. \n" "This specifies the model architecture.") parser.add_argument("--pytorch_dump_path", default = None, @@ -61,5 +61,5 @@ if __name__ == "__main__": help = "Path to the output PyTorch model.") args = parser.parse_args() convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, - args.xxx_config_file, + args.config_file, args.pytorch_dump_path) diff --git a/templates/adding_a_new_model/modeling_tf_xxx.py b/templates/adding_a_new_model/modeling_tf_xxx.py index f1d898b4..17836209 100644 --- a/templates/adding_a_new_model/modeling_tf_xxx.py +++ b/templates/adding_a_new_model/modeling_tf_xxx.py @@ -26,13 +26,15 @@ import logging import math import os import sys +import copy +import itertools from io import open import numpy as np import tensorflow as tf from .configuration_xxx import XxxConfig -from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -121,9 +123,9 @@ class TFXxxMainLayer(tf.keras.layers.Layer): input_ids = inputs if attention_mask is None: - attention_mask = tf.fill(tf.shape(input_ids), 1) + attention_mask = tf.fill(shape_list(input_ids), 1) if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(shape_list(input_ids), 0) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py index 94c4b0db..c8be58e2 100644 --- a/templates/adding_a_new_model/modeling_xxx.py +++ b/templates/adding_a_new_model/modeling_xxx.py @@ -25,6 +25,8 @@ import logging import math import os import sys +import copy +import itertools from io import open import torch diff --git a/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py index 90837ca1..6eba932a 100644 --- a/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py +++ b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow from transformers import XxxConfig, is_tf_available @@ -33,10 +32,9 @@ if is_tf_available(): TFXxxForTokenClassification, TFXxxForQuestionAnswering, TF_XXX_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFXxxModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFXxxModel, TFXxxForMaskedLM, TFXxxForQuestionAnswering, @@ -112,7 +110,7 @@ class TFXxxModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = XxxConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, @@ -244,12 +242,10 @@ class TFXxxModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xxx_for_token_classification(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in ['xxx-base-uncased']: - model = TFXxxModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFXxxModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/templates/adding_a_new_model/tests/modeling_xxx_test.py b/templates/adding_a_new_model/tests/modeling_xxx_test.py index 8c0cc3cf..5e22392d 100644 --- a/templates/adding_a_new_model/tests/modeling_xxx_test.py +++ b/templates/adding_a_new_model/tests/modeling_xxx_test.py @@ -17,13 +17,12 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device if is_torch_available(): from transformers import (XxxConfig, XxxModel, XxxForMaskedLM, @@ -31,10 +30,9 @@ if is_torch_available(): XxxForQuestionAnswering, XxxForSequenceClassification, XxxForTokenClassification, XxxForMultipleChoice) from transformers.modeling_xxx import XXX_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") +@require_torch class XxxModelTest(CommonTestCases.CommonModelTester): all_model_classes = (XxxModel, XxxForMaskedLM, XxxForQuestionAnswering, @@ -110,7 +108,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = XxxConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, @@ -131,6 +129,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = XxxModel(config=config) + model.to(torch_device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) @@ -148,6 +147,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = XxxForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) result = { @@ -162,6 +162,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = XxxForQuestionAnswering(config=config) + model.to(torch_device) model.eval() loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, start_positions=sequence_labels, end_positions=sequence_labels) @@ -182,6 +183,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = XxxForSequenceClassification(config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) result = { @@ -197,6 +199,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = XxxForTokenClassification(config=config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) result = { @@ -243,12 +246,10 @@ class XxxModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xxx_for_token_classification(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(XXX_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = XxxModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = XxxModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/templates/adding_a_new_model/tokenization_xxx.py b/templates/adding_a_new_model/tokenization_xxx.py index 3d6b4ad9..7a10a41e 100644 --- a/templates/adding_a_new_model/tokenization_xxx.py +++ b/templates/adding_a_new_model/tokenization_xxx.py @@ -85,7 +85,7 @@ class XxxTokenizer(PreTrainedTokenizer): Args: vocab_file: Path to a one-wordpiece-per-line vocabulary file - do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False + do_lower_case: Whether to lower case the input. Only has an effect when do_basic_tokenize=True """ vocab_files_names = VOCAB_FILES_NAMES diff --git a/transformers-cli b/transformers-cli new file mode 100755 index 00000000..0a980a35 --- /dev/null +++ b/transformers-cli @@ -0,0 +1,30 @@ +#!/usr/bin/env python +from argparse import ArgumentParser + +from transformers.commands.download import DownloadCommand +from transformers.commands.run import RunCommand +from transformers.commands.user import UserCommands +from transformers.commands.convert import ConvertCommand +from transformers.commands.serving import ServeCommand + +if __name__ == '__main__': + parser = ArgumentParser('Transformers CLI tool', usage='transformers-cli []') + commands_parser = parser.add_subparsers(help='transformers-cli command helpers') + + # Register commands + ConvertCommand.register_subcommand(commands_parser) + DownloadCommand.register_subcommand(commands_parser) + RunCommand.register_subcommand(commands_parser) + ServeCommand.register_subcommand(commands_parser) + UserCommands.register_subcommand(commands_parser) + + # Let's go + args = parser.parse_args() + + if not hasattr(args, 'func'): + parser.print_help() + exit(1) + + # Run + service = args.func(args) + service.run() diff --git a/transformers/__init__.py b/transformers/__init__.py old mode 100644 new mode 100755 index dd9e18a0..c0c0901d --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.1.1" +__version__ = "2.3.0" # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. @@ -19,21 +19,29 @@ logger = logging.getLogger(__name__) # pylint: disable=invalid-name # Files and general utilities from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, - WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME, + WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME, MODEL_CARD_NAME, is_tf_available, is_torch_available) from .data import (is_sklearn_available, InputExample, InputFeatures, DataProcessor, + SingleSentenceClassificationProcessor, glue_output_modes, glue_convert_examples_to_features, - glue_processors, glue_tasks_num_labels) + glue_processors, glue_tasks_num_labels, + xnli_output_modes, xnli_processors, xnli_tasks_num_labels, + squad_convert_examples_to_features, SquadFeatures, + SquadExample, SquadV1Processor, SquadV2Processor) if is_sklearn_available(): - from .data import glue_compute_metrics + from .data import glue_compute_metrics, xnli_compute_metrics + +# Model Cards +from .modelcard import ModelCard # Tokenizers from .tokenization_utils import (PreTrainedTokenizer) from .tokenization_auto import AutoTokenizer from .tokenization_bert import BertTokenizer, BasicTokenizer, WordpieceTokenizer +from .tokenization_bert_japanese import BertJapaneseTokenizer, MecabTokenizer, CharacterTokenizer from .tokenization_openai import OpenAIGPTTokenizer from .tokenization_transfo_xl import (TransfoXLTokenizer, TransfoXLCorpus) from .tokenization_gpt2 import GPT2Tokenizer @@ -42,26 +50,33 @@ from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_albert import AlbertTokenizer +from .tokenization_camembert import CamembertTokenizer +from .tokenization_t5 import T5Tokenizer +from .tokenization_xlm_roberta import XLMRobertaTokenizer # Configurations from .configuration_utils import PretrainedConfig -from .configuration_auto import AutoConfig +from .configuration_auto import AutoConfig, ALL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_bert import BertConfig, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_openai import OpenAIGPTConfig, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_transfo_xl import TransfoXLConfig, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_gpt2 import GPT2Config, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlnet import XLNetConfig, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_t5 import T5Config, T5_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlm_roberta import XLMRobertaConfig, XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling if is_torch_available(): from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, - AutoModelWithLMHead) + AutoModelWithLMHead, AutoModelForTokenClassification, ALL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_bert import (BertPreTrainedModel, BertModel, BertForPreTraining, BertForMaskedLM, BertForNextSentencePrediction, @@ -69,8 +84,8 @@ if is_torch_available(): BertForTokenClassification, BertForQuestionAnswering, load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel, - OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, - load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) + OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, + load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_transfo_xl import (TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel, AdaptiveEmbedding, load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) @@ -81,9 +96,10 @@ if is_torch_available(): CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_xlnet import (XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, - XLNetForSequenceClassification, XLNetForMultipleChoice, - XLNetForQuestionAnsweringSimple, XLNetForQuestionAnswering, - load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) + XLNetForSequenceClassification, XLNetForTokenClassification, + XLNetForMultipleChoice, XLNetForQuestionAnsweringSimple, + XLNetForQuestionAnswering, load_tf_weights_in_xlnet, + XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_xlm import (XLMPreTrainedModel , XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering, XLMForQuestionAnsweringSimple, @@ -92,11 +108,25 @@ if is_torch_available(): RobertaForSequenceClassification, RobertaForMultipleChoice, RobertaForTokenClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) - from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, + from .modeling_distilbert import (DistilBertPreTrainedModel, DistilBertForMaskedLM, DistilBertModel, DistilBertForSequenceClassification, DistilBertForQuestionAnswering, DistilBertForTokenClassification, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_camembert import (CamembertForMaskedLM, CamembertModel, + CamembertForSequenceClassification, CamembertForMultipleChoice, + CamembertForTokenClassification, + CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model + from .modeling_t5 import (T5PreTrainedModel, T5Model, T5WithLMHeadModel, + load_tf_weights_in_t5, + T5_PRETRAINED_MODEL_ARCHIVE_MAP) + + from .modeling_albert import (AlbertPreTrainedModel, AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, + AlbertForQuestionAnswering, + load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + + from .modeling_xlm_roberta import (XLMRobertaForMaskedLM, XLMRobertaModel, XLMRobertaForMultipleChoice, + XLMRobertaForSequenceClassification, XLMRobertaForTokenClassification) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, @@ -105,9 +135,9 @@ if is_torch_available(): # TensorFlow if is_tf_available(): - from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary + from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list from .modeling_tf_auto import (TFAutoModel, TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, - TFAutoModelWithLMHead) + TFAutoModelWithLMHead, TFAutoModelForTokenClassification, TF_ALL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertMainLayer, TFBertEmbeddings, TFBertModel, TFBertForPreTraining, @@ -131,6 +161,7 @@ if is_tf_available(): from .modeling_tf_xlnet import (TFXLNetPreTrainedModel, TFXLNetMainLayer, TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, + TFXLNetForTokenClassification, TFXLNetForQuestionAnsweringSimple, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) @@ -149,6 +180,7 @@ if is_tf_available(): from .modeling_tf_distilbert import (TFDistilBertPreTrainedModel, TFDistilBertMainLayer, TFDistilBertModel, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification, + TFDistilBertForTokenClassification, TFDistilBertForQuestionAnswering, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) @@ -156,6 +188,16 @@ if is_tf_available(): TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM, + TFAlbertForSequenceClassification, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + + from .modeling_tf_t5 import (TFT5PreTrainedModel, TFT5Model, TFT5WithLMHeadModel, + TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP) + + # Optimization + from .optimization_tf import (WarmUp, create_optimizer, AdamWeightDecay, GradientAccumulator) + # TF 2.0 <=> PyTorch conversion utilities from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name, load_pytorch_checkpoint_in_tf2_model, @@ -165,6 +207,10 @@ from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name load_tf2_weights_in_pytorch_model, load_tf2_model_in_pytorch_model) +# Pipelines +from .pipelines import pipeline, PipelineDataFormat, CsvPipelineDataFormat, JsonPipelineDataFormat, PipedPipelineDataFormat, \ + Pipeline, FeatureExtractionPipeline, QuestionAnsweringPipeline, NerPipeline, TextClassificationPipeline + if not is_tf_available() and not is_torch_available(): logger.warning("Neither PyTorch nor TensorFlow >= 2.0 have been found." "Models won't be available and only tokenizers, configuration" diff --git a/transformers/__main__.py b/transformers/__main__.py index 31dbd249..dd259b04 100644 --- a/transformers/__main__.py +++ b/transformers/__main__.py @@ -1,129 +1,37 @@ # coding: utf8 + def main(): import sys - if (len(sys.argv) < 4 or len(sys.argv) > 6) or sys.argv[1] not in ["bert", "gpt", "transfo_xl", "gpt2", "xlnet", "xlm"]: + if len(sys.argv) < 2 or sys.argv[1] not in ["convert", "train", "predict", "serve"]: print( - "This command line utility let you convert original (author released) model checkpoint to pytorch.\n" - "It should be used as one of: \n" - ">> transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT, \n" - ">> transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG], \n" - ">> transformers transfo_xl TF_CHECKPOINT_OR_DATASET PYTORCH_DUMP_OUTPUT [TF_CONFIG] or \n" - ">> transformers gpt2 TF_CHECKPOINT PYTORCH_DUMP_OUTPUT [GPT2_CONFIG] or \n" - ">> transformers xlnet TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT [FINETUNING_TASK_NAME] or \n" - ">> transformers xlm XLM_CHECKPOINT_PATH PYTORCH_DUMP_OUTPUT") - else: - if sys.argv[1] == "bert": - try: - from .convert_bert_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch - except ImportError: - print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " - "In that case, it requires TensorFlow to be installed. Please see " - "https://www.tensorflow.org/install/ for installation instructions.") - raise + "First argument to `transformers` command line interface should be one of: \n" + ">> convert serve train predict") + if sys.argv[1] == "convert": + from transformers.commands import convert + convert(sys.argv) + elif sys.argv[1] == "train": + from transformers.commands import train + train(sys.argv) + elif sys.argv[1] == "serve": + pass + # from argparse import ArgumentParser + # from transformers.commands.serving import ServeCommand + # parser = ArgumentParser('Transformers CLI tool', usage='transformers serve []') + # commands_parser = parser.add_subparsers(help='transformers-cli command helpers') - if len(sys.argv) != 5: - # pylint: disable=line-too-long - print("Should be used as `transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT`") - else: - PYTORCH_DUMP_OUTPUT = sys.argv.pop() - TF_CONFIG = sys.argv.pop() - TF_CHECKPOINT = sys.argv.pop() - convert_tf_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT) - elif sys.argv[1] == "gpt": - from .convert_openai_original_tf_checkpoint_to_pytorch import convert_openai_checkpoint_to_pytorch - if len(sys.argv) < 4 or len(sys.argv) > 5: - # pylint: disable=line-too-long - print("Should be used as `transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG]`") - else: - OPENAI_GPT_CHECKPOINT_FOLDER_PATH = sys.argv[2] - PYTORCH_DUMP_OUTPUT = sys.argv[3] - if len(sys.argv) == 5: - OPENAI_GPT_CONFIG = sys.argv[4] - else: - OPENAI_GPT_CONFIG = "" - convert_openai_checkpoint_to_pytorch(OPENAI_GPT_CHECKPOINT_FOLDER_PATH, - OPENAI_GPT_CONFIG, - PYTORCH_DUMP_OUTPUT) - elif sys.argv[1] == "transfo_xl": - try: - from .convert_transfo_xl_original_tf_checkpoint_to_pytorch import convert_transfo_xl_checkpoint_to_pytorch - except ImportError: - print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " - "In that case, it requires TensorFlow to be installed. Please see " - "https://www.tensorflow.org/install/ for installation instructions.") - raise - if len(sys.argv) < 4 or len(sys.argv) > 5: - # pylint: disable=line-too-long - print("Should be used as `transformers transfo_xl TF_CHECKPOINT/TF_DATASET_FILE PYTORCH_DUMP_OUTPUT [TF_CONFIG]`") - else: - if 'ckpt' in sys.argv[2].lower(): - TF_CHECKPOINT = sys.argv[2] - TF_DATASET_FILE = "" - else: - TF_DATASET_FILE = sys.argv[2] - TF_CHECKPOINT = "" - PYTORCH_DUMP_OUTPUT = sys.argv[3] - if len(sys.argv) == 5: - TF_CONFIG = sys.argv[4] - else: - TF_CONFIG = "" - convert_transfo_xl_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT, TF_DATASET_FILE) - elif sys.argv[1] == "gpt2": - try: - from .convert_gpt2_original_tf_checkpoint_to_pytorch import convert_gpt2_checkpoint_to_pytorch - except ImportError: - print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " - "In that case, it requires TensorFlow to be installed. Please see " - "https://www.tensorflow.org/install/ for installation instructions.") - raise - if len(sys.argv) < 4 or len(sys.argv) > 5: - # pylint: disable=line-too-long - print("Should be used as `transformers gpt2 TF_CHECKPOINT PYTORCH_DUMP_OUTPUT [TF_CONFIG]`") - else: - TF_CHECKPOINT = sys.argv[2] - PYTORCH_DUMP_OUTPUT = sys.argv[3] - if len(sys.argv) == 5: - TF_CONFIG = sys.argv[4] - else: - TF_CONFIG = "" - convert_gpt2_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT) - elif sys.argv[1] == "xlnet": - try: - from .convert_xlnet_original_tf_checkpoint_to_pytorch import convert_xlnet_checkpoint_to_pytorch - except ImportError: - print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " - "In that case, it requires TensorFlow to be installed. Please see " - "https://www.tensorflow.org/install/ for installation instructions.") - raise + # # Register commands + # ServeCommand.register_subcommand(commands_parser) - if len(sys.argv) < 5 or len(sys.argv) > 6: - # pylint: disable=line-too-long - print("Should be used as `transformers xlnet TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT [FINETUNING_TASK_NAME]`") - else: - TF_CHECKPOINT = sys.argv[2] - TF_CONFIG = sys.argv[3] - PYTORCH_DUMP_OUTPUT = sys.argv[4] - if len(sys.argv) == 6: - FINETUNING_TASK = sys.argv[5] - else: - FINETUNING_TASK = None + # # Let's go + # args = parser.parse_args() - convert_xlnet_checkpoint_to_pytorch(TF_CHECKPOINT, - TF_CONFIG, - PYTORCH_DUMP_OUTPUT, - FINETUNING_TASK) - elif sys.argv[1] == "xlm": - from .convert_xlm_original_pytorch_checkpoint_to_pytorch import convert_xlm_checkpoint_to_pytorch - - if len(sys.argv) != 4: - # pylint: disable=line-too-long - print("Should be used as `transformers xlm XLM_CHECKPOINT_PATH PYTORCH_DUMP_OUTPUT`") - else: - XLM_CHECKPOINT_PATH = sys.argv[2] - PYTORCH_DUMP_OUTPUT = sys.argv[3] - - convert_xlm_checkpoint_to_pytorch(XLM_CHECKPOINT_PATH, PYTORCH_DUMP_OUTPUT) + # if not hasattr(args, 'func'): + # parser.print_help() + # exit(1) + # # Run + # service = args.func(args) + # service.run() if __name__ == '__main__': main() diff --git a/transformers/commands/__init__.py b/transformers/commands/__init__.py new file mode 100644 index 00000000..bbdd5655 --- /dev/null +++ b/transformers/commands/__init__.py @@ -0,0 +1,12 @@ +from abc import ABC, abstractmethod +from argparse import ArgumentParser + +class BaseTransformersCLICommand(ABC): + @staticmethod + @abstractmethod + def register_subcommand(parser: ArgumentParser): + raise NotImplementedError() + + @abstractmethod + def run(self): + raise NotImplementedError() diff --git a/transformers/commands/convert.py b/transformers/commands/convert.py new file mode 100644 index 00000000..55dbf537 --- /dev/null +++ b/transformers/commands/convert.py @@ -0,0 +1,115 @@ +from argparse import ArgumentParser, Namespace + +from logging import getLogger + +from transformers import AutoModel, AutoTokenizer +from transformers.commands import BaseTransformersCLICommand + + +def convert_command_factory(args: Namespace): + """ + Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint. + :return: ServeCommand + """ + return ConvertCommand(args.model_type, args.tf_checkpoint, args.pytorch_dump_output, + args.config, args.finetuning_task_name) + + +class ConvertCommand(BaseTransformersCLICommand): + + @staticmethod + def register_subcommand(parser: ArgumentParser): + """ + Register this command to argparse so it's available for the transformer-cli + :param parser: Root parser to register command-specific arguments + :return: + """ + train_parser = parser.add_parser('convert', help="CLI tool to run convert model from original " + "author checkpoints to Transformesr PyTorch checkpoints.") + train_parser.add_argument('--model_type', type=str, required=True, + help='Model\'s type.') + train_parser.add_argument('--tf_checkpoint', type=str, required=True, + help='TensorFlow checkpoint path or folder.') + train_parser.add_argument('--pytorch_dump_output', type=str, required=True, + help='Path to the PyTorch savd model output.') + train_parser.add_argument('--config', type=str, default="", + help='Configuration file path or folder.') + train_parser.add_argument('--finetuning_task_name', type=str, default=None, + help='Optional fine-tuning task name if the TF model was a finetuned model.') + train_parser.set_defaults(func=convert_command_factory) + + def __init__(self, model_type: str, tf_checkpoint: str, pytorch_dump_output: str, + config: str, finetuning_task_name: str, *args): + self._logger = getLogger('transformers-cli/converting') + + self._logger.info('Loading model {}'.format(model_type)) + self._model_type = model_type + self._tf_checkpoint = tf_checkpoint + self._pytorch_dump_output = pytorch_dump_output + self._config = config + self._finetuning_task_name = finetuning_task_name + + def run(self): + if self._model_type == "bert": + try: + from transformers.convert_bert_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch + except ImportError: + msg = "transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " \ + "In that case, it requires TensorFlow to be installed. Please see " \ + "https://www.tensorflow.org/install/ for installation instructions." + raise ImportError(msg) + + convert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output) + elif self._model_type == "gpt": + from transformers.convert_openai_original_tf_checkpoint_to_pytorch import convert_openai_checkpoint_to_pytorch + convert_openai_checkpoint_to_pytorch(self._tf_checkpoint, + self._config, + self._pytorch_dump_output) + elif self._model_type == "transfo_xl": + try: + from transformers.convert_transfo_xl_original_tf_checkpoint_to_pytorch import convert_transfo_xl_checkpoint_to_pytorch + except ImportError: + msg = "transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " \ + "In that case, it requires TensorFlow to be installed. Please see " \ + "https://www.tensorflow.org/install/ for installation instructions." + raise ImportError(msg) + + if 'ckpt' in self._tf_checkpoint.lower(): + TF_CHECKPOINT = self._tf_checkpoint + TF_DATASET_FILE = "" + else: + TF_DATASET_FILE = self._tf_checkpoint + TF_CHECKPOINT = "" + convert_transfo_xl_checkpoint_to_pytorch(TF_CHECKPOINT, + self._config, + self._pytorch_dump_output, + TF_DATASET_FILE) + elif self._model_type == "gpt2": + try: + from transformers.convert_gpt2_original_tf_checkpoint_to_pytorch import convert_gpt2_checkpoint_to_pytorch + except ImportError: + msg = "transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " \ + "In that case, it requires TensorFlow to be installed. Please see " \ + "https://www.tensorflow.org/install/ for installation instructions." + raise ImportError(msg) + + convert_gpt2_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output) + elif self._model_type == "xlnet": + try: + from transformers.convert_xlnet_original_tf_checkpoint_to_pytorch import convert_xlnet_checkpoint_to_pytorch + except ImportError: + msg = "transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " \ + "In that case, it requires TensorFlow to be installed. Please see " \ + "https://www.tensorflow.org/install/ for installation instructions." + raise ImportError(msg) + + convert_xlnet_checkpoint_to_pytorch(self._tf_checkpoint, + self._config, + self._pytorch_dump_output, + self._finetuning_task_name) + elif self._model_type == "xlm": + from transformers.convert_xlm_original_pytorch_checkpoint_to_pytorch import convert_xlm_checkpoint_to_pytorch + + convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint, self._pytorch_dump_output) + else: + raise ValueError("--model_type should be selected in the list [bert, gpt, gpt2, transfo_xl, xlnet, xlm]") diff --git a/transformers/commands/download.py b/transformers/commands/download.py new file mode 100644 index 00000000..0938f135 --- /dev/null +++ b/transformers/commands/download.py @@ -0,0 +1,29 @@ +from argparse import ArgumentParser + +from transformers.commands import BaseTransformersCLICommand + + +def download_command_factory(args): + return DownloadCommand(args.model, args.cache_dir, args.force) + + +class DownloadCommand(BaseTransformersCLICommand): + + @staticmethod + def register_subcommand(parser: ArgumentParser): + download_parser = parser.add_parser('download') + download_parser.add_argument('--cache-dir', type=str, default=None, help='Path to location to store the models') + download_parser.add_argument('--force', action='store_true', help='Force the model to be download even if already in cache-dir') + download_parser.add_argument('model', type=str, help='Name of the model to download') + download_parser.set_defaults(func=download_command_factory) + + def __init__(self, model: str, cache: str, force: bool): + self._model = model + self._cache = cache + self._force = force + + def run(self): + from transformers import AutoModel, AutoTokenizer + + AutoModel.from_pretrained(self._model, cache_dir=self._cache, force_download=self._force) + AutoTokenizer.from_pretrained(self._model, cache_dir=self._cache, force_download=self._force) \ No newline at end of file diff --git a/transformers/commands/run.py b/transformers/commands/run.py new file mode 100644 index 00000000..df03cee9 --- /dev/null +++ b/transformers/commands/run.py @@ -0,0 +1,79 @@ +import logging +from argparse import ArgumentParser + +from transformers.commands import BaseTransformersCLICommand +from transformers.pipelines import pipeline, Pipeline, PipelineDataFormat, SUPPORTED_TASKS + + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + + +def try_infer_format_from_ext(path: str): + if not path: + return 'pipe' + + for ext in PipelineDataFormat.SUPPORTED_FORMATS: + if path.endswith(ext): + return ext + + raise Exception( + 'Unable to determine file format from file extension {}. ' + 'Please provide the format through --format {}'.format(path, PipelineDataFormat.SUPPORTED_FORMATS) + ) + + +def run_command_factory(args): + nlp = pipeline(task=args.task, + model=args.model if args.model else None, + config=args.config, + tokenizer=args.tokenizer, + device=args.device) + format = try_infer_format_from_ext(args.input) if args.format == 'infer' else args.format + reader = PipelineDataFormat.from_str(format=format, + output_path=args.output, + input_path=args.input, + column=args.column if args.column else nlp.default_input_names, + overwrite=args.overwrite) + return RunCommand(nlp, reader) + + +class RunCommand(BaseTransformersCLICommand): + + def __init__(self, nlp: Pipeline, reader: PipelineDataFormat): + self._nlp = nlp + self._reader = reader + + @staticmethod + def register_subcommand(parser: ArgumentParser): + run_parser = parser.add_parser('run', help="Run a pipeline through the CLI") + run_parser.add_argument('--task', choices=SUPPORTED_TASKS.keys(), help='Task to run') + run_parser.add_argument('--input', type=str, help='Path to the file to use for inference') + run_parser.add_argument('--output', type=str, help='Path to the file that will be used post to write results.') + run_parser.add_argument('--model', type=str, help='Name or path to the model to instantiate.') + run_parser.add_argument('--config', type=str, help='Name or path to the model\'s config to instantiate.') + run_parser.add_argument('--tokenizer', type=str, help='Name of the tokenizer to use. (default: same as the model name)') + run_parser.add_argument('--column', type=str, help='Name of the column to use as input. (For multi columns input as QA use column1,columns2)') + run_parser.add_argument('--format', type=str, default='infer', choices=PipelineDataFormat.SUPPORTED_FORMATS, help='Input format to read from') + run_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') + run_parser.add_argument('--overwrite', action='store_true', help='Allow overwriting the output file.') + run_parser.set_defaults(func=run_command_factory) + + def run(self): + nlp, outputs = self._nlp, [] + + for entry in self._reader: + output = nlp(**entry) if self._reader.is_multi_columns else nlp(entry) + if isinstance(output, dict): + outputs.append(output) + else: + outputs += output + + # Saving data + if self._nlp.binary_output: + binary_path = self._reader.save_binary(outputs) + logger.warning('Current pipeline requires output to be in binary format, saving at {}'.format(binary_path)) + else: + self._reader.save(outputs) + + + diff --git a/transformers/commands/serving.py b/transformers/commands/serving.py new file mode 100644 index 00000000..4f41f797 --- /dev/null +++ b/transformers/commands/serving.py @@ -0,0 +1,158 @@ +from argparse import ArgumentParser, Namespace +from typing import List, Optional, Union, Any + +import logging + +try: + from uvicorn import run + from fastapi import FastAPI, HTTPException, Body + from pydantic import BaseModel + _serve_dependancies_installed = True +except (ImportError, AttributeError): + BaseModel = object + Body = lambda *x, **y: None + _serve_dependancies_installed = False + +from transformers import Pipeline +from transformers.commands import BaseTransformersCLICommand +from transformers.pipelines import SUPPORTED_TASKS, pipeline + +logger = logging.getLogger('transformers-cli/serving') + +def serve_command_factory(args: Namespace): + """ + Factory function used to instantiate serving server from provided command line arguments. + :return: ServeCommand + """ + nlp = pipeline(task=args.task, + model=args.model if args.model else None, + config=args.config, + tokenizer=args.tokenizer, + device=args.device) + return ServeCommand(nlp, args.host, args.port) + + +class ServeModelInfoResult(BaseModel): + """ + Expose model information + """ + infos: dict + + +class ServeTokenizeResult(BaseModel): + """ + Tokenize result model + """ + tokens: List[str] + tokens_ids: Optional[List[int]] + + +class ServeDeTokenizeResult(BaseModel): + """ + DeTokenize result model + """ + text: str + + +class ServeForwardResult(BaseModel): + """ + Forward result model + """ + output: Any + + +class ServeCommand(BaseTransformersCLICommand): + + @staticmethod + def register_subcommand(parser: ArgumentParser): + """ + Register this command to argparse so it's available for the transformer-cli + :param parser: Root parser to register command-specific arguments + :return: + """ + serve_parser = parser.add_parser('serve', help='CLI tool to run inference requests through REST and GraphQL endpoints.') + serve_parser.add_argument('--task', type=str, choices=SUPPORTED_TASKS.keys(), help='The task to run the pipeline on') + serve_parser.add_argument('--host', type=str, default='localhost', help='Interface the server will listen on.') + serve_parser.add_argument('--port', type=int, default=8888, help='Port the serving will listen to.') + serve_parser.add_argument('--model', type=str, help='Model\'s name or path to stored model.') + serve_parser.add_argument('--config', type=str, help='Model\'s config name or path to stored model.') + serve_parser.add_argument('--tokenizer', type=str, help='Tokenizer name to use.') + serve_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') + serve_parser.set_defaults(func=serve_command_factory) + + def __init__(self, pipeline: Pipeline, host: str, port: int): + + self._pipeline = pipeline + + self._host = host + self._port = port + if not _serve_dependancies_installed: + raise ImportError("Using serve command requires FastAPI and unicorn. " + "Please install transformers with [serving]: pip install transformers[serving]." + "Or install FastAPI and unicorn separatly.") + else: + logger.info('Serving model over {}:{}'.format(host, port)) + self._app = FastAPI() + + # Register routes + self._app.add_api_route('/', self.model_info, response_model=ServeModelInfoResult, methods=['GET']) + self._app.add_api_route('/tokenize', self.tokenize, response_model=ServeTokenizeResult, methods=['POST']) + self._app.add_api_route('/detokenize', self.detokenize, response_model=ServeDeTokenizeResult, methods=['POST']) + self._app.add_api_route('/forward', self.forward, response_model=ServeForwardResult, methods=['POST']) + + def run(self): + run(self._app, host=self._host, port=self._port) + + def model_info(self): + return ServeModelInfoResult(infos=vars(self._pipeline.model.config)) + + def tokenize(self, text_input: str = Body(None, embed=True), return_ids: bool = Body(False, embed=True)): + """ + Tokenize the provided input and eventually returns corresponding tokens id: + - **text_input**: String to tokenize + - **return_ids**: Boolean flags indicating if the tokens have to be converted to their integer mapping. + """ + try: + tokens_txt = self._pipeline.tokenizer.tokenize(text_input) + + if return_ids: + tokens_ids = self._pipeline.tokenizer.convert_tokens_to_ids(tokens_txt) + return ServeTokenizeResult(tokens=tokens_txt, tokens_ids=tokens_ids) + else: + return ServeTokenizeResult(tokens=tokens_txt) + + except Exception as e: + raise HTTPException(status_code=500, detail={"model": '', "error": str(e)}) + + def detokenize(self, tokens_ids: List[int] = Body(None, embed=True), + skip_special_tokens: bool = Body(False, embed=True), + cleanup_tokenization_spaces: bool = Body(True, embed=True)): + """ + Detokenize the provided tokens ids to readable text: + - **tokens_ids**: List of tokens ids + - **skip_special_tokens**: Flag indicating to not try to decode special tokens + - **cleanup_tokenization_spaces**: Flag indicating to remove all leading/trailing spaces and intermediate ones. + """ + try: + decoded_str = self._pipeline.tokenizer.decode(tokens_ids, skip_special_tokens, cleanup_tokenization_spaces) + return ServeDeTokenizeResult(model='', text=decoded_str) + except Exception as e: + raise HTTPException(status_code=500, detail={"model": '', "error": str(e)}) + + def forward(self, inputs: Union[str, dict, List[str], List[int], List[dict]] = Body(None, embed=True)): + """ + **inputs**: + **attention_mask**: + **tokens_type_ids**: + """ + + # Check we don't have empty string + if len(inputs) == 0: + return ServeForwardResult(output=[], attention=[]) + + try: + # Forward through the model + output = self._pipeline(inputs) + return ServeForwardResult(output=output) + except Exception as e: + raise HTTPException(500, {"error": str(e)}) diff --git a/transformers/commands/train.py b/transformers/commands/train.py new file mode 100644 index 00000000..7b267458 --- /dev/null +++ b/transformers/commands/train.py @@ -0,0 +1,131 @@ +import os +from argparse import ArgumentParser, Namespace +from logging import getLogger + +from transformers.commands import BaseTransformersCLICommand +from transformers import (is_tf_available, is_torch_available, + TextClassificationPipeline, + SingleSentenceClassificationProcessor as Processor) + +if not is_tf_available() and not is_torch_available(): + raise ImportError("At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training") + +# TF training parameters +USE_XLA = False +USE_AMP = False + +def train_command_factory(args: Namespace): + """ + Factory function used to instantiate serving server from provided command line arguments. + :return: ServeCommand + """ + return TrainCommand(args) + + +class TrainCommand(BaseTransformersCLICommand): + + @staticmethod + def register_subcommand(parser: ArgumentParser): + """ + Register this command to argparse so it's available for the transformer-cli + :param parser: Root parser to register command-specific arguments + :return: + """ + train_parser = parser.add_parser('train', help='CLI tool to train a model on a task.') + + train_parser.add_argument('--train_data', type=str, required=True, + help="path to train (and optionally evaluation) dataset as a csv with " + "tab separated labels and sentences.") + train_parser.add_argument('--column_label', type=int, default=0, + help='Column of the dataset csv file with example labels.') + train_parser.add_argument('--column_text', type=int, default=1, + help='Column of the dataset csv file with example texts.') + train_parser.add_argument('--column_id', type=int, default=2, + help='Column of the dataset csv file with example ids.') + train_parser.add_argument('--skip_first_row', action='store_true', + help='Skip the first row of the csv file (headers).') + + train_parser.add_argument('--validation_data', type=str, default='', + help='path to validation dataset.') + train_parser.add_argument('--validation_split', type=float, default=0.1, + help="if validation dataset is not provided, fraction of train dataset " + "to use as validation dataset.") + + train_parser.add_argument('--output', type=str, default='./', + help='path to saved the trained model.') + + train_parser.add_argument('--task', type=str, default='text_classification', + help='Task to train the model on.') + train_parser.add_argument('--model', type=str, default='bert-base-uncased', + help='Model\'s name or path to stored model.') + train_parser.add_argument('--train_batch_size', type=int, default=32, + help='Batch size for training.') + train_parser.add_argument('--valid_batch_size', type=int, default=64, + help='Batch size for validation.') + train_parser.add_argument('--learning_rate', type=float, default=3e-5, + help="Learning rate.") + train_parser.add_argument('--adam_epsilon', type=float, default=1e-08, + help="Epsilon for Adam optimizer.") + train_parser.set_defaults(func=train_command_factory) + + def __init__(self, args: Namespace): + self.logger = getLogger('transformers-cli/training') + + self.framework = 'tf' if is_tf_available() else 'torch' + + os.makedirs(args.output, exist_ok=True) + assert os.path.isdir(args.output) + self.output = args.output + + self.column_label = args.column_label + self.column_text = args.column_text + self.column_id = args.column_id + + self.logger.info('Loading {} pipeline for {}'.format(args.task, args.model)) + if args.task == 'text_classification': + self.pipeline = TextClassificationPipeline.from_pretrained(args.model) + elif args.task == 'token_classification': + raise NotImplementedError + elif args.task == 'question_answering': + raise NotImplementedError + + self.logger.info('Loading dataset from {}'.format(args.train_data)) + self.train_dataset = Processor.create_from_csv(args.train_data, + column_label=args.column_label, + column_text=args.column_text, + column_id=args.column_id, + skip_first_row=args.skip_first_row) + self.valid_dataset = None + if args.validation_data: + self.logger.info('Loading validation dataset from {}'.format(args.validation_data)) + self.valid_dataset = Processor.create_from_csv(args.validation_data, + column_label=args.column_label, + column_text=args.column_text, + column_id=args.column_id, + skip_first_row=args.skip_first_row) + + self.validation_split = args.validation_split + self.train_batch_size = args.train_batch_size + self.valid_batch_size = args.valid_batch_size + self.learning_rate = args.learning_rate + self.adam_epsilon = args.adam_epsilon + + def run(self): + if self.framework == 'tf': + return self.run_tf() + return self.run_torch() + + def run_torch(self): + raise NotImplementedError + + def run_tf(self): + self.pipeline.fit(self.train_dataset, + validation_data=self.valid_dataset, + validation_split=self.validation_split, + learning_rate=self.learning_rate, + adam_epsilon=self.adam_epsilon, + train_batch_size=self.train_batch_size, + valid_batch_size=self.valid_batch_size) + + # Save trained pipeline + self.pipeline.save_pretrained(self.output) diff --git a/transformers/commands/user.py b/transformers/commands/user.py new file mode 100644 index 00000000..8e0e5634 --- /dev/null +++ b/transformers/commands/user.py @@ -0,0 +1,194 @@ +from argparse import ArgumentParser +from getpass import getpass +import os + +from transformers.commands import BaseTransformersCLICommand +from transformers.hf_api import HfApi, HfFolder, HTTPError + + +class UserCommands(BaseTransformersCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + login_parser = parser.add_parser('login') + login_parser.set_defaults(func=lambda args: LoginCommand(args)) + whoami_parser = parser.add_parser('whoami') + whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args)) + logout_parser = parser.add_parser('logout') + logout_parser.set_defaults(func=lambda args: LogoutCommand(args)) + list_parser = parser.add_parser('ls') + list_parser.set_defaults(func=lambda args: ListObjsCommand(args)) + # upload + upload_parser = parser.add_parser('upload') + upload_parser.add_argument('path', type=str, help='Local path of the folder or individual file to upload.') + upload_parser.add_argument('--filename', type=str, default=None, help='Optional: override individual object filename on S3.') + upload_parser.set_defaults(func=lambda args: UploadCommand(args)) + + + +class ANSI: + """ + Helper for en.wikipedia.org/wiki/ANSI_escape_code + """ + _bold = u"\u001b[1m" + _reset = u"\u001b[0m" + @classmethod + def bold(cls, s): + return "{}{}{}".format(cls._bold, s, cls._reset) + + +class BaseUserCommand: + def __init__(self, args): + self.args = args + self._api = HfApi() + + +class LoginCommand(BaseUserCommand): + def run(self): + print(""" + _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_| + _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _| + _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_| + _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _| + _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_| + + """) + username = input("Username: ") + password = getpass() + try: + token = self._api.login(username, password) + except HTTPError as e: + # probably invalid credentials, display error message. + print(e) + exit(1) + HfFolder.save_token(token) + print("Login successful") + print("Your token:", token, "\n") + print("Your token has been saved to", HfFolder.path_token) + + +class WhoamiCommand(BaseUserCommand): + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit() + try: + user = self._api.whoami(token) + print(user) + except HTTPError as e: + print(e) + + +class LogoutCommand(BaseUserCommand): + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit() + HfFolder.delete_token() + self._api.logout(token) + print("Successfully logged out.") + + +class ListObjsCommand(BaseUserCommand): + def tabulate(self, rows, headers): + # type: (List[List[Union[str, int]]], List[str]) -> str + """ + Inspired by: + stackoverflow.com/a/8356620/593036 + stackoverflow.com/questions/9535954/printing-lists-as-tabular-data + """ + col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] + row_format = ("{{:{}}} " * len(headers)).format(*col_widths) + lines = [] + lines.append( + row_format.format(*headers) + ) + lines.append( + row_format.format(*["-" * w for w in col_widths]) + ) + for row in rows: + lines.append( + row_format.format(*row) + ) + return "\n".join(lines) + + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit(1) + try: + objs = self._api.list_objs(token) + except HTTPError as e: + print(e) + exit(1) + if len(objs) == 0: + print("No shared file yet") + exit() + rows = [ [ + obj.filename, + obj.LastModified, + obj.ETag, + obj.Size + ] for obj in objs ] + print( + self.tabulate(rows, headers=["Filename", "LastModified", "ETag", "Size"]) + ) + + +class UploadCommand(BaseUserCommand): + def walk_dir(self, rel_path): + """ + Recursively list all files in a folder. + """ + entries: List[os.DirEntry] = list(os.scandir(rel_path)) + files = [ + ( + os.path.join(os.getcwd(), f.path), # filepath + f.path # filename + ) + for f in entries if f.is_file() + ] + for f in entries: + if f.is_dir(): + files += self.walk_dir(f.path) + return files + + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit(1) + local_path = os.path.abspath(self.args.path) + if os.path.isdir(local_path): + if self.args.filename is not None: + raise ValueError("Cannot specify a filename override when uploading a folder.") + rel_path = os.path.basename(local_path) + files = self.walk_dir(rel_path) + elif os.path.isfile(local_path): + filename = self.args.filename if self.args.filename is not None else os.path.basename(local_path) + files = [(local_path, filename)] + else: + raise ValueError("Not a valid file or directory: {}".format(local_path)) + + for filepath, filename in files: + print( + "About to upload file {} to S3 under filename {}".format( + ANSI.bold(filepath), ANSI.bold(filename) + ) + ) + + choice = input("Proceed? [Y/n] ").lower() + if not(choice == "" or choice == "y" or choice == "yes"): + print("Abort") + exit() + print( + ANSI.bold("Uploading... This might take a while if files are large") + ) + for filepath, filename in files: + access_url = self._api.presign_and_upload( + token=token, filename=filename, filepath=filepath + ) + print("Your file now lives at:") + print(access_url) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py new file mode 100644 index 00000000..6a1ef78d --- /dev/null +++ b/transformers/configuration_albert.py @@ -0,0 +1,100 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" ALBERT model configuration """ + +from .configuration_utils import PretrainedConfig + +ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-config.json", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-config.json", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-config.json", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-config.json", +} + +class AlbertConfig(PretrainedConfig): + """Configuration for `AlbertModel`. + + The default settings match the configuration of model `albert_xxlarge`. + """ + + pretrained_config_archive_map = ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__(self, + vocab_size=30000, + embedding_size=128, + hidden_size=4096, + num_hidden_layers=12, + num_hidden_groups=1, + num_attention_heads=64, + intermediate_size=16384, + inner_group_num=1, + hidden_act="gelu_new", + hidden_dropout_prob=0, + attention_probs_dropout_prob=0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, **kwargs): + """Constructs AlbertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `AlbertModel`. + embedding_size: size of voc embeddings. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_hidden_groups: Number of group for the hidden layers, parameters in + the same group are shared. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + inner_group_num: int, number of inner repetition of attention and ffn. + down_scale_factor: float, the scale to apply + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probability for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `AlbertModel`. + initializer_range: The stdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + super(AlbertConfig, self).__init__(**kwargs) + + self.vocab_size = vocab_size + self.embedding_size = embedding_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_hidden_groups = num_hidden_groups + self.num_attention_heads = num_attention_heads + self.inner_group_num = inner_group_num + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index edd21a67..28125638 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -18,19 +18,42 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging -from .configuration_bert import BertConfig -from .configuration_openai import OpenAIGPTConfig -from .configuration_gpt2 import GPT2Config -from .configuration_transfo_xl import TransfoXLConfig -from .configuration_xlnet import XLNetConfig -from .configuration_xlm import XLMConfig -from .configuration_roberta import RobertaConfig -from .configuration_distilbert import DistilBertConfig -from .configuration_ctrl import CTRLConfig +from .configuration_bert import BertConfig, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_openai import OpenAIGPTConfig, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_transfo_xl import TransfoXLConfig, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_gpt2 import GPT2Config, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlnet import XLNetConfig, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_t5 import T5Config, T5_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlm_roberta import XLMRobertaConfig, XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP logger = logging.getLogger(__name__) +ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict((key, value) + for pretrained_map in [ + BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, + TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, + GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, + CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, + ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, + DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + T5_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, + ] + for key, value, in pretrained_map.items()) + + class AutoConfig(object): r""":class:`~transformers.AutoConfig` is a generic configuration class that will be instantiated as one of the configuration classes of the library @@ -43,13 +66,16 @@ class AutoConfig(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertConfig (DistilBERT model) + - contains `albert`: AlbertConfig (ALBERT model) + - contains `camembert`: CamembertConfig (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaConfig (XLM-RoBERTa model) + - contains `roberta`: RobertaConfig (RoBERTa model) - contains `bert`: BertConfig (Bert model) - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) - contains `gpt2`: GPT2Config (OpenAI GPT-2 model) - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model) - contains `xlnet`: XLNetConfig (XLNet model) - contains `xlm`: XLMConfig (XLM model) - - contains `roberta`: RobertaConfig (RoBERTa model) - contains `ctrl` : CTRLConfig (CTRL model) This class cannot be instantiated using `__init__()` (throw an error). """ @@ -57,6 +83,34 @@ class AutoConfig(object): raise EnvironmentError("AutoConfig is designed to be instantiated " "using the `AutoConfig.from_pretrained(pretrained_model_name_or_path)` method.") + @classmethod + def for_model(cls, model_type, *args, **kwargs): + if 'distilbert' in model_type: + return DistilBertConfig(*args, **kwargs) + elif 'roberta' in model_type: + return RobertaConfig(*args, **kwargs) + elif 'bert' in model_type: + return BertConfig(*args, **kwargs) + elif 'openai-gpt' in model_type: + return OpenAIGPTConfig(*args, **kwargs) + elif 'gpt2' in model_type: + return GPT2Config(*args, **kwargs) + elif 'transfo-xl' in model_type: + return TransfoXLConfig(*args, **kwargs) + elif 'xlnet' in model_type: + return XLNetConfig(*args, **kwargs) + elif 'xlm' in model_type: + return XLMConfig(*args, **kwargs) + elif 'ctrl' in model_type: + return CTRLConfig(*args, **kwargs) + elif 'albert' in model_type: + return AlbertConfig(*args, **kwargs) + elif 'camembert' in model_type: + return CamembertConfig(*args, **kwargs) + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'distilbert', 'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'xlm', 'roberta', 'ctrl', 'camembert', 'albert'".format(model_type)) + @classmethod def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): r""" Instantiate a one of the configuration classes of the library @@ -64,19 +118,24 @@ class AutoConfig(object): The configuration class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Config (T5 model) - contains `distilbert`: DistilBertConfig (DistilBERT model) + - contains `albert`: AlbertConfig (ALBERT model) + - contains `camembert`: CamembertConfig (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaConfig (XLM-RoBERTa model) + - contains `roberta`: RobertaConfig (RoBERTa model) - contains `bert`: BertConfig (Bert model) - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) - contains `gpt2`: GPT2Config (OpenAI GPT-2 model) - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model) - contains `xlnet`: XLNetConfig (XLNet model) - contains `xlm`: XLMConfig (XLM model) - - contains `roberta`: RobertaConfig (RoBERTa model) - contains `ctrl` : CTRLConfig (CTRL model) Params: pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model configuration that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing a configuration file saved using the :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. @@ -92,6 +151,9 @@ class AutoConfig(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -114,8 +176,16 @@ class AutoConfig(object): assert unused_kwargs == {'foo': False} """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return T5Config.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return DistilBertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -134,4 +204,4 @@ class AutoConfig(object): return CTRLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm-roberta', 'xlm', 'roberta', 'distilbert', 'camembert', 'ctrl', 'albert'".format(pretrained_model_name_or_path)) diff --git a/transformers/configuration_bert.py b/transformers/configuration_bert.py index d63be963..7b495013 100644 --- a/transformers/configuration_bert.py +++ b/transformers/configuration_bert.py @@ -42,6 +42,12 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-config.json", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-config.json", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-config.json", + 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-config.json", + 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-config.json", + 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-config.json", + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-config.json", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/config.json", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1/config.json", } @@ -52,7 +58,7 @@ class BertConfig(PretrainedConfig): Arguments: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `BertModel`. + vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. hidden_size: Size of the encoder layers and the pooler layer. num_hidden_layers: Number of hidden layers in the Transformer encoder. num_attention_heads: Number of attention heads for each attention layer in @@ -77,7 +83,7 @@ class BertConfig(PretrainedConfig): pretrained_config_archive_map = BERT_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=30522, + vocab_size=30522, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, @@ -91,25 +97,15 @@ class BertConfig(PretrainedConfig): layer_norm_eps=1e-12, **kwargs): super(BertConfig, self).__init__(**kwargs) - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.hidden_size = hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.hidden_act = hidden_act - self.intermediate_size = intermediate_size - self.hidden_dropout_prob = hidden_dropout_prob - self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.max_position_embeddings = max_position_embeddings - self.type_vocab_size = type_vocab_size - self.initializer_range = initializer_range - self.layer_norm_eps = layer_norm_eps - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps diff --git a/transformers/configuration_camembert.py b/transformers/configuration_camembert.py new file mode 100644 index 00000000..3ff64454 --- /dev/null +++ b/transformers/configuration_camembert.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" CamemBERT configuration """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .configuration_roberta import RobertaConfig + +logger = logging.getLogger(__name__) + +CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json", +} + + +class CamembertConfig(RobertaConfig): + pretrained_config_archive_map = CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP diff --git a/transformers/configuration_ctrl.py b/transformers/configuration_ctrl.py index fcbd848d..f9b9e409 100644 --- a/transformers/configuration_ctrl.py +++ b/transformers/configuration_ctrl.py @@ -31,7 +31,7 @@ class CTRLConfig(PretrainedConfig): """Configuration class to store the configuration of a `CTRLModel`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). dff: Size of the inner dimension of the FFN. @@ -52,7 +52,7 @@ class CTRLConfig(PretrainedConfig): def __init__( self, - vocab_size_or_config_json_file=246534, + vocab_size=246534, n_positions=256, n_ctx=256, n_embd=1280, @@ -64,8 +64,6 @@ class CTRLConfig(PretrainedConfig): attn_pdrop=0.1, layer_norm_epsilon=1e-6, initializer_range=0.02, - - num_labels=1, summary_type='cls_index', summary_use_proj=True, summary_activation=None, @@ -76,7 +74,7 @@ class CTRLConfig(PretrainedConfig): """Constructs CTRLConfig. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). dff: Size of the inner dimension of the FFN. @@ -94,8 +92,7 @@ class CTRLConfig(PretrainedConfig): initializing all weight matrices. """ super(CTRLConfig, self).__init__(**kwargs) - - self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1 + self.vocab_size = vocab_size self.n_ctx = n_ctx self.n_positions = n_positions self.n_embd = n_embd @@ -108,23 +105,11 @@ class CTRLConfig(PretrainedConfig): self.layer_norm_epsilon = layer_norm_epsilon self.initializer_range = initializer_range - self.num_labels = num_labels self.summary_type = summary_type self.summary_use_proj = summary_use_proj self.summary_activation = summary_activation self.summary_first_dropout = summary_first_dropout self.summary_proj_to_labels = summary_proj_to_labels - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif not isinstance(vocab_size_or_config_json_file, int): - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) @property def max_position_embeddings(self): diff --git a/transformers/configuration_distilbert.py b/transformers/configuration_distilbert.py index 2a8a149a..d9f7cc63 100644 --- a/transformers/configuration_distilbert.py +++ b/transformers/configuration_distilbert.py @@ -27,7 +27,9 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json", + 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-config.json", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-config.json", } @@ -35,7 +37,7 @@ class DistilBertConfig(PretrainedConfig): pretrained_config_archive_map = DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=30522, + vocab_size=30522, max_position_embeddings=512, sinusoidal_pos_embds=False, n_layers=6, @@ -51,31 +53,21 @@ class DistilBertConfig(PretrainedConfig): seq_classif_dropout=0.2, **kwargs): super(DistilBertConfig, self).__init__(**kwargs) + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.sinusoidal_pos_embds = sinusoidal_pos_embds + self.n_layers = n_layers + self.n_heads = n_heads + self.dim = dim + self.hidden_dim = hidden_dim + self.dropout = dropout + self.attention_dropout = attention_dropout + self.activation = activation + self.initializer_range = initializer_range + self.tie_weights_ = tie_weights_ + self.qa_dropout = qa_dropout + self.seq_classif_dropout = seq_classif_dropout - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.max_position_embeddings = max_position_embeddings - self.sinusoidal_pos_embds = sinusoidal_pos_embds - self.n_layers = n_layers - self.n_heads = n_heads - self.dim = dim - self.hidden_dim = hidden_dim - self.dropout = dropout - self.attention_dropout = attention_dropout - self.activation = activation - self.initializer_range = initializer_range - self.tie_weights_ = tie_weights_ - self.qa_dropout = qa_dropout - self.seq_classif_dropout = seq_classif_dropout - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") @property def hidden_size(self): return self.dim diff --git a/transformers/configuration_gpt2.py b/transformers/configuration_gpt2.py index c2fb4948..4c200c07 100644 --- a/transformers/configuration_gpt2.py +++ b/transformers/configuration_gpt2.py @@ -36,7 +36,7 @@ class GPT2Config(PretrainedConfig): """Configuration class to store the configuration of a `GPT2Model`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). n_embd: Dimensionality of the embeddings and hidden states. @@ -56,7 +56,7 @@ class GPT2Config(PretrainedConfig): def __init__( self, - vocab_size_or_config_json_file=50257, + vocab_size=50257, n_positions=1024, n_ctx=1024, n_embd=768, @@ -67,8 +67,6 @@ class GPT2Config(PretrainedConfig): attn_pdrop=0.1, layer_norm_epsilon=1e-5, initializer_range=0.02, - - num_labels=1, summary_type='cls_index', summary_use_proj=True, summary_activation=None, @@ -79,7 +77,7 @@ class GPT2Config(PretrainedConfig): """Constructs GPT2Config. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). n_embd: Dimensionality of the embeddings and hidden states. @@ -96,37 +94,22 @@ class GPT2Config(PretrainedConfig): initializing all weight matrices. """ super(GPT2Config, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.n_ctx = n_ctx - self.n_positions = n_positions - self.n_embd = n_embd - self.n_layer = n_layer - self.n_head = n_head - self.resid_pdrop = resid_pdrop - self.embd_pdrop = embd_pdrop - self.attn_pdrop = attn_pdrop - self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range - - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_first_dropout = summary_first_dropout - self.summary_proj_to_labels = summary_proj_to_labels - else: - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) + self.vocab_size = vocab_size + self.n_ctx = n_ctx + self.n_positions = n_positions + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_first_dropout = summary_first_dropout + self.summary_proj_to_labels = summary_proj_to_labels @property def max_position_embeddings(self): diff --git a/transformers/configuration_openai.py b/transformers/configuration_openai.py index 886b7f5b..7776a0bb 100644 --- a/transformers/configuration_openai.py +++ b/transformers/configuration_openai.py @@ -35,7 +35,7 @@ class OpenAIGPTConfig(PretrainedConfig): Configuration class to store the configuration of a `OpenAIGPTModel`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `OpenAIGPTModel` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `OpenAIGPTModel` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). n_embd: Dimensionality of the embeddings and hidden states. @@ -58,7 +58,7 @@ class OpenAIGPTConfig(PretrainedConfig): def __init__( self, - vocab_size_or_config_json_file=40478, + vocab_size=40478, n_positions=512, n_ctx=512, n_embd=768, @@ -71,8 +71,6 @@ class OpenAIGPTConfig(PretrainedConfig): layer_norm_epsilon=1e-5, initializer_range=0.02, predict_special_tokens=True, - - num_labels=1, summary_type='cls_index', summary_use_proj=True, summary_activation=None, @@ -83,39 +81,24 @@ class OpenAIGPTConfig(PretrainedConfig): """Constructs OpenAIGPTConfig. """ super(OpenAIGPTConfig, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.n_ctx = n_ctx - self.n_positions = n_positions - self.n_embd = n_embd - self.n_layer = n_layer - self.n_head = n_head - self.afn = afn - self.resid_pdrop = resid_pdrop - self.embd_pdrop = embd_pdrop - self.attn_pdrop = attn_pdrop - self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range - self.predict_special_tokens = predict_special_tokens - - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_first_dropout = summary_first_dropout - self.summary_proj_to_labels = summary_proj_to_labels - else: - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) + self.vocab_size = vocab_size + self.n_ctx = n_ctx + self.n_positions = n_positions + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.afn = afn + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.predict_special_tokens = predict_special_tokens + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_first_dropout = summary_first_dropout + self.summary_proj_to_labels = summary_proj_to_labels @property def max_position_embeddings(self): diff --git a/transformers/configuration_t5.py b/transformers/configuration_t5.py new file mode 100644 index 00000000..377a0919 --- /dev/null +++ b/transformers/configuration_t5.py @@ -0,0 +1,108 @@ +# coding=utf-8 +# Copyright 2010, The T5 Authors and HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" T5 model configuration """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +import six +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +T5_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-config.json", + 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-config.json", + 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-config.json", + 't5-3b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3b-config.json", + 't5-11b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11b-config.json", +} + + +class T5Config(PretrainedConfig): + r""" + :class:`~transformers.T5Config` is the configuration class to store the configuration of a + `T5Model`. + + + Arguments: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `T5Model`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu", "swish" and "gelu_new" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `T5Model`. + initializer_factor: A factor for initializing all weight matrices (should be kept to 1.0, used for initialization testing). + layer_norm_eps: The epsilon used by LayerNorm. + """ + pretrained_config_archive_map = T5_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__(self, + vocab_size=32128, + n_positions=512, + d_model=512, + d_kv=64, + d_ff=2048, + num_layers=6, + num_heads=8, + relative_attention_num_buckets=32, + dropout_rate=0.1, + layer_norm_epsilon=1e-6, + initializer_factor=1.0, + **kwargs): + super(T5Config, self).__init__(**kwargs) + self.vocab_size = vocab_size + self.n_positions = n_positions + self.d_model = d_model + self.d_kv = d_kv + self.d_ff = d_ff + self.num_layers = num_layers + self.num_heads = num_heads + self.relative_attention_num_buckets = relative_attention_num_buckets + self.dropout_rate = dropout_rate + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_factor = initializer_factor + + @property + def max_position_embeddings(self): + return self.n_positions + + @property + def hidden_size(self): + return self.d_model + + @property + def num_attention_heads(self): + return self.num_heads + + @property + def num_hidden_layers(self): + return self.num_layers diff --git a/transformers/configuration_transfo_xl.py b/transformers/configuration_transfo_xl.py index d55a6adb..52f0f45a 100644 --- a/transformers/configuration_transfo_xl.py +++ b/transformers/configuration_transfo_xl.py @@ -34,7 +34,7 @@ class TransfoXLConfig(PretrainedConfig): """Configuration class to store the configuration of a `TransfoXLModel`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `TransfoXLModel` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `TransfoXLModel` or a configuration json file. cutoffs: cutoffs for the adaptive softmax d_model: Dimensionality of the model's hidden states. d_embed: Dimensionality of the embeddings @@ -68,7 +68,7 @@ class TransfoXLConfig(PretrainedConfig): pretrained_config_archive_map = TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=267735, + vocab_size=267735, cutoffs=[20000, 40000, 200000], d_model=1024, d_embed=1024, @@ -100,7 +100,7 @@ class TransfoXLConfig(PretrainedConfig): """Constructs TransfoXLConfig. """ super(TransfoXLConfig, self).__init__(**kwargs) - self.n_token = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1 + self.vocab_size = vocab_size self.cutoffs = [] self.cutoffs.extend(cutoffs) self.tie_weight = tie_weight @@ -133,27 +133,17 @@ class TransfoXLConfig(PretrainedConfig): self.init_std = init_std self.layer_norm_epsilon = layer_norm_epsilon - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif not isinstance(vocab_size_or_config_json_file, int): - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") - @property def max_position_embeddings(self): return self.tgt_len + self.ext_len + self.mem_len @property - def vocab_size(self): - return self.n_token + def n_token(self): # Backward compatibility + return self.vocab_size - @vocab_size.setter - def vocab_size(self, value): - self.n_token = value + @n_token.setter + def n_token(self, value): # Backward compatibility + self.vocab_size = value @property def hidden_size(self): diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index 547bb69c..f692c9b1 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -24,7 +24,7 @@ import logging import os from io import open -from .file_utils import cached_path, CONFIG_NAME +from .file_utils import CONFIG_NAME, cached_path, is_remote_url, hf_bucket_url logger = logging.getLogger(__name__) @@ -49,8 +49,7 @@ class PretrainedConfig(object): pretrained_config_archive_map = {} def __init__(self, **kwargs): - self.finetuning_task = kwargs.pop('finetuning_task', None) - self.num_labels = kwargs.pop('num_labels', 2) + # Attributes with defaults self.output_attentions = kwargs.pop('output_attentions', False) self.output_hidden_states = kwargs.pop('output_hidden_states', False) self.output_past = kwargs.pop('output_past', True) # Not used by all models @@ -59,6 +58,22 @@ class PretrainedConfig(object): self.pruned_heads = kwargs.pop('pruned_heads', {}) self.is_decoder = kwargs.pop('is_decoder', False) + # Fine-tuning task arguments + self.finetuning_task = kwargs.pop('finetuning_task', None) + self.num_labels = kwargs.pop('num_labels', 2) + self.id2label = kwargs.pop('id2label', {i: 'LABEL_{}'.format(i) for i in range(self.num_labels)}) + self.id2label = dict((int(key), value) for key, value in self.id2label.items()) + self.label2id = kwargs.pop('label2id', dict(zip(self.id2label.values(), self.id2label.keys()))) + self.label2id = dict((key, int(value)) for key, value in self.label2id.items()) + + # Additional attributes without default values + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error("Can't set {} with value {} for {}".format(key, value, self)) + raise err + def save_pretrained(self, save_directory): """ Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the :func:`~transformers.PretrainedConfig.from_pretrained` class method. @@ -79,6 +94,7 @@ class PretrainedConfig(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model configuration that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing a configuration file saved using the :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. @@ -94,6 +110,9 @@ class PretrainedConfig(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -120,6 +139,7 @@ class PretrainedConfig(object): """ cache_dir = kwargs.pop('cache_dir', None) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) @@ -127,11 +147,18 @@ class PretrainedConfig(object): config_file = cls.pretrained_config_archive_map[pretrained_model_name_or_path] elif os.path.isdir(pretrained_model_name_or_path): config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME) - else: + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): config_file = pretrained_model_name_or_path - # redirect to the cache, if necessary + else: + config_file = hf_bucket_url(pretrained_model_name_or_path, postfix=CONFIG_NAME) + try: - resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + # Load from URL or cache if already cached + resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, + proxies=proxies, resume_download=resume_download) + # Load config + config = cls.from_json_file(resolved_config_file) + except EnvironmentError: if pretrained_model_name_or_path in cls.pretrained_config_archive_map: msg = "Couldn't reach server at '{}' to download pretrained model configuration file.".format( @@ -145,15 +172,18 @@ class PretrainedConfig(object): config_file, CONFIG_NAME) raise EnvironmentError(msg) + except json.JSONDecodeError: + msg = "Couldn't reach server at '{}' to download configuration file or " \ + "configuration file is not a valid JSON file. " \ + "Please check network or file content here: {}.".format(config_file, resolved_config_file) + raise EnvironmentError(msg) + if resolved_config_file == config_file: logger.info("loading configuration file {}".format(config_file)) else: logger.info("loading configuration file {} from cache at {}".format( config_file, resolved_config_file)) - # Load config - config = cls.from_json_file(resolved_config_file) - if hasattr(config, 'pruned_heads'): config.pruned_heads = dict((int(key), value) for key, value in config.pruned_heads.items()) @@ -175,17 +205,15 @@ class PretrainedConfig(object): @classmethod def from_dict(cls, json_object): """Constructs a `Config` from a Python dictionary of parameters.""" - config = cls(vocab_size_or_config_json_file=-1) - for key, value in json_object.items(): - setattr(config, key, value) - return config + return cls(**json_object) @classmethod def from_json_file(cls, json_file): - """Constructs a `BertConfig` from a json file of parameters.""" + """Constructs a `Config` from a json file of parameters.""" with open(json_file, "r", encoding='utf-8') as reader: text = reader.read() - return cls.from_dict(json.loads(text)) + dict_obj = json.loads(text) + return cls(**dict_obj) def __eq__(self, other): return self.__dict__ == other.__dict__ diff --git a/transformers/configuration_xlm.py b/transformers/configuration_xlm.py index fa3a5f40..6839a457 100644 --- a/transformers/configuration_xlm.py +++ b/transformers/configuration_xlm.py @@ -42,7 +42,7 @@ class XLMConfig(PretrainedConfig): """Configuration class to store the configuration of a `XLMModel`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `XLMModel`. + vocab_size: Vocabulary size of `inputs_ids` in `XLMModel`. d_model: Size of the encoder layers and the pooler layer. n_layer: Number of hidden layers in the Transformer encoder. n_head: Number of attention heads for each attention layer in @@ -81,7 +81,7 @@ class XLMConfig(PretrainedConfig): pretrained_config_archive_map = XLM_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=30145, + vocab_size=30145, emb_dim=2048, n_layers=12, n_heads=16, @@ -103,9 +103,6 @@ class XLMConfig(PretrainedConfig): unk_index=3, mask_index=5, is_encoder=True, - - finetuning_task=None, - num_labels=2, summary_type='first', summary_use_proj=True, summary_activation=None, @@ -117,56 +114,46 @@ class XLMConfig(PretrainedConfig): """Constructs XLMConfig. """ super(XLMConfig, self).__init__(**kwargs) + self.vocab_size = vocab_size + self.emb_dim = emb_dim + self.n_layers = n_layers + self.n_heads = n_heads + self.dropout = dropout + self.attention_dropout = attention_dropout + self.gelu_activation = gelu_activation + self.sinusoidal_embeddings = sinusoidal_embeddings + self.causal = causal + self.asm = asm + self.n_langs = n_langs + self.use_lang_emb = use_lang_emb + self.layer_norm_eps = layer_norm_eps + self.bos_index = bos_index + self.eos_index = eos_index + self.pad_index = pad_index + self.unk_index = unk_index + self.mask_index = mask_index + self.is_encoder = is_encoder + self.max_position_embeddings = max_position_embeddings + self.embed_init_std = embed_init_std + self.init_std = init_std + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_proj_to_labels = summary_proj_to_labels + self.summary_first_dropout = summary_first_dropout + self.start_n_top = start_n_top + self.end_n_top = end_n_top - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.n_words = vocab_size_or_config_json_file - self.emb_dim = emb_dim - self.n_layers = n_layers - self.n_heads = n_heads - self.dropout = dropout - self.attention_dropout = attention_dropout - self.gelu_activation = gelu_activation - self.sinusoidal_embeddings = sinusoidal_embeddings - self.causal = causal - self.asm = asm - self.n_langs = n_langs - self.use_lang_emb = use_lang_emb - self.layer_norm_eps = layer_norm_eps - self.bos_index = bos_index - self.eos_index = eos_index - self.pad_index = pad_index - self.unk_index = unk_index - self.mask_index = mask_index - self.is_encoder = is_encoder - self.max_position_embeddings = max_position_embeddings - self.embed_init_std = embed_init_std - self.init_std = init_std - self.finetuning_task = finetuning_task - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_proj_to_labels = summary_proj_to_labels - self.summary_first_dropout = summary_first_dropout - self.start_n_top = start_n_top - self.end_n_top = end_n_top - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") + if "n_words" in kwargs: + self.n_words = kwargs["n_words"] @property - def vocab_size(self): - return self.n_words + def n_words(self): # For backward compatibility + return self.vocab_size - @vocab_size.setter - def vocab_size(self, value): - self.n_words = value + @n_words.setter + def n_words(self, value): # For backward compatibility + self.vocab_size = value @property def hidden_size(self): diff --git a/transformers/configuration_xlm_roberta.py b/transformers/configuration_xlm_roberta.py new file mode 100644 index 00000000..5b6955f4 --- /dev/null +++ b/transformers/configuration_xlm_roberta.py @@ -0,0 +1,38 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XLM-RoBERTa configuration """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .configuration_roberta import RobertaConfig + +logger = logging.getLogger(__name__) + +XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'xlm-roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-base-config.json", + 'xlm-roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-config.json", + 'xlm-roberta-large-finetuned-conll02-dutch': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-dutch-config.json", + 'xlm-roberta-large-finetuned-conll02-spanish': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-spanish-config.json", + 'xlm-roberta-large-finetuned-conll03-english': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-english-config.json", + 'xlm-roberta-large-finetuned-conll03-german': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-german-config.json", +} + + +class XLMRobertaConfig(RobertaConfig): + pretrained_config_archive_map = XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP diff --git a/transformers/configuration_xlnet.py b/transformers/configuration_xlnet.py index 0dbf5188..017c57cf 100644 --- a/transformers/configuration_xlnet.py +++ b/transformers/configuration_xlnet.py @@ -35,7 +35,7 @@ class XLNetConfig(PretrainedConfig): """Configuration class to store the configuration of a ``XLNetModel``. Args: - vocab_size_or_config_json_file: Vocabulary size of ``inputs_ids`` in ``XLNetModel``. + vocab_size: Vocabulary size of ``inputs_ids`` in ``XLNetModel``. d_model: Size of the encoder layers and the pooler layer. n_layer: Number of hidden layers in the Transformer encoder. n_head: Number of attention heads for each attention layer in @@ -72,28 +72,22 @@ class XLNetConfig(PretrainedConfig): pretrained_config_archive_map = XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=32000, + vocab_size=32000, d_model=1024, n_layer=24, n_head=16, d_inner=4096, - max_position_embeddings=512, ff_activation="gelu", untie_r=True, attn_type="bi", - initializer_range=0.02, layer_norm_eps=1e-12, - dropout=0.1, mem_len=None, reuse_len=None, bi_data=False, clamp_len=-1, same_length=False, - - finetuning_task=None, - num_labels=2, summary_type='last', summary_use_proj=True, summary_activation='tanh', @@ -104,58 +98,45 @@ class XLNetConfig(PretrainedConfig): """Constructs XLNetConfig. """ super(XLNetConfig, self).__init__(**kwargs) + self.vocab_size = vocab_size + self.d_model = d_model + self.n_layer = n_layer + self.n_head = n_head + assert d_model % n_head == 0 + self.d_head = d_model // n_head + self.ff_activation = ff_activation + self.d_inner = d_inner + self.untie_r = untie_r + self.attn_type = attn_type - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - setattr(config, key, value) - elif isinstance(vocab_size_or_config_json_file, int): - self.n_token = vocab_size_or_config_json_file - self.d_model = d_model - self.n_layer = n_layer - self.n_head = n_head - assert d_model % n_head == 0 - self.d_head = d_model // n_head - self.ff_activation = ff_activation - self.d_inner = d_inner - self.untie_r = untie_r - self.attn_type = attn_type + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps - self.initializer_range = initializer_range - self.layer_norm_eps = layer_norm_eps + self.dropout = dropout + self.mem_len = mem_len + self.reuse_len = reuse_len + self.bi_data = bi_data + self.clamp_len = clamp_len + self.same_length = same_length - self.dropout = dropout - self.mem_len = mem_len - self.reuse_len = reuse_len - self.bi_data = bi_data - self.clamp_len = clamp_len - self.same_length = same_length - - self.finetuning_task = finetuning_task - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_last_dropout = summary_last_dropout - self.start_n_top = start_n_top - self.end_n_top = end_n_top - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_last_dropout = summary_last_dropout + self.start_n_top = start_n_top + self.end_n_top = end_n_top @property def max_position_embeddings(self): return -1 @property - def vocab_size(self): - return self.n_token + def n_token(self): # Backward compatibility + return self.vocab_size - @vocab_size.setter - def vocab_size(self, value): - self.n_token = value + @n_token.setter + def n_token(self, value): # Backward compatibility + self.vocab_size = value @property def hidden_size(self): diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py new file mode 100644 index 00000000..b6476b4f --- /dev/null +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -0,0 +1,67 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert ALBERT checkpoint.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import torch + +from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_albert + +import logging +logging.basicConfig(level=logging.INFO) + + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, albert_config_file, pytorch_dump_path): + # Initialise PyTorch model + config = AlbertConfig.from_json_file(albert_config_file) + print("Building PyTorch model from configuration: {}".format(str(config))) + model = AlbertForMaskedLM(config) + + # Load weights from tf checkpoint + load_tf_weights_in_albert(model, config, tf_checkpoint_path) + + # Save pytorch-model + print("Save PyTorch model to {}".format(pytorch_dump_path)) + torch.save(model.state_dict(), pytorch_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path to the TensorFlow checkpoint path.") + parser.add_argument("--albert_config_file", + default = None, + type = str, + required = True, + help = "The config json file corresponding to the pre-trained ALBERT model. \n" + "This specifies the model architecture.") + parser.add_argument("--pytorch_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output PyTorch model.") + args = parser.parse_args() + convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, + args.albert_config_file, + args.pytorch_dump_path) + \ No newline at end of file diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index e673b77d..0edac6fb 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -32,8 +32,10 @@ from transformers import (load_pytorch_checkpoint_in_tf2_model, TransfoXLConfig, TFTransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, - DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, TFDistilBertForSequenceClassification, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, + AlbertConfig, TFAlbertForMaskedLM, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + T5Config, TFT5WithLMHeadModel, T5_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_torch_available(): import torch @@ -45,8 +47,10 @@ if is_torch_available(): TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, - DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + DistilBertForMaskedLM, DistilBertForQuestionAnswering, DistilBertForSequenceClassification, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP) else: (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, @@ -55,8 +59,10 @@ else: TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, - DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) = ( + DistilBertForMaskedLM, DistilBertForSequenceClassification, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP) = ( None, None, None, None, None, None, None, None, @@ -64,7 +70,9 @@ else: None, None, None, None, None, None, None, - None, None, None, + None, None, None, None, + None, None, + None, None, None, None) @@ -85,7 +93,10 @@ MODEL_CLASSES = { 'roberta-large-mnli': (RobertaConfig, TFRobertaForSequenceClassification, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), - 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + 'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'albert': (AlbertConfig, TFAlbertForMaskedLM, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), + 't5': (T5Config, TFT5WithLMHeadModel, T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP, T5_PRETRAINED_CONFIG_ARCHIVE_MAP), } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False, use_cached_models=True): @@ -110,23 +121,21 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tf_model = load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) if compare_with_pt_model: - inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] - tf_inputs = tf.constant(inputs_list) - tfo = tf_model(tf_inputs, training=False) # build the network + tfo = tf_model(tf_model.dummy_inputs, training=False) # build the network - pt_model = pt_model_class.from_pretrained(None, + state_dict = torch.load(pytorch_checkpoint_path, map_location='cpu') + pt_model = pt_model_class.from_pretrained(pretrained_model_name_or_path=None, config=config, - state_dict=torch.load(pytorch_checkpoint_path, - map_location='cpu')) - pt_inputs = torch.tensor(inputs_list) - with torch.no_grad(): - pto = pt_model(pt_inputs) + state_dict=state_dict) - np_pt = pto[0].detach().numpy() + with torch.no_grad(): + pto = pt_model(**pt_model.dummy_inputs) + + np_pt = pto[0].numpy() np_tf = tfo[0].numpy() diff = np.amax(np.abs(np_pt - np_tf)) print("Max absolute difference between models outputs {}".format(diff)) - assert diff <= 2e-2, "Error, model absolute difference is >2e-2" + assert diff <= 2e-2, "Error, model absolute difference is >2e-2: {}".format(diff) # Save pytorch-model print("Save TensorFlow model to {}".format(tf_dump_path)) @@ -134,7 +143,7 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, model_shortcut_names_or_path=None, config_shortcut_names_or_path=None, - compare_with_pt_model=False, use_cached_models=False, only_convert_finetuned_models=False): + compare_with_pt_model=False, use_cached_models=False, remove_cached_files=False, only_convert_finetuned_models=False): assert os.path.isdir(args.tf_dump_path), "--tf_dump_path should be a directory" if args_model_type is None: @@ -182,13 +191,15 @@ def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, model_shortc if os.path.isfile(model_shortcut_name): model_shortcut_name = 'converted_model' + convert_pt_checkpoint_to_tf(model_type=model_type, pytorch_checkpoint_path=model_file, config_file=config_file, tf_dump_path=os.path.join(tf_dump_path, model_shortcut_name + '-tf_model.h5'), compare_with_pt_model=compare_with_pt_model) - os.remove(config_file) - os.remove(model_file) + if remove_cached_files: + os.remove(config_file) + os.remove(model_file) if __name__ == "__main__": @@ -221,6 +232,9 @@ if __name__ == "__main__": parser.add_argument("--use_cached_models", action='store_true', help = "Use cached models if possible instead of updating to latest checkpoint versions.") + parser.add_argument("--remove_cached_files", + action='store_true', + help = "Remove pytorch models after conversion (save memory when converting in batches).") parser.add_argument("--only_convert_finetuned_models", action='store_true', help = "Only convert finetuned models.") @@ -240,4 +254,5 @@ if __name__ == "__main__": config_shortcut_names_or_path=[args.config_file] if args.config_file is not None else None, compare_with_pt_model=args.compare_with_pt_model, use_cached_models=args.use_cached_models, + remove_cached_files=args.remove_cached_files, only_convert_finetuned_models=args.only_convert_finetuned_models) diff --git a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py index 60935add..fedfc1ec 100644 --- a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py +++ b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py @@ -20,6 +20,13 @@ import argparse import logging import numpy as np import torch +import pathlib + +import fairseq +from packaging import version + +if version.parse(fairseq.__version__) < version.parse("0.9.0"): + raise Exception("requires fairseq >= 0.9.0") from fairseq.models.roberta import RobertaModel as FairseqRobertaModel from fairseq.modules import TransformerSentenceEncoderLayer @@ -45,8 +52,9 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ """ roberta = FairseqRobertaModel.from_pretrained(roberta_checkpoint_path) roberta.eval() # disable dropout + roberta_sent_encoder = roberta.model.decoder.sentence_encoder config = BertConfig( - vocab_size_or_config_json_file=50265, + vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings, hidden_size=roberta.args.encoder_embed_dim, num_hidden_layers=roberta.args.encoder_layers, num_attention_heads=roberta.args.encoder_attention_heads, @@ -64,7 +72,6 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ # Now let's copy all the weights. # Embeddings - roberta_sent_encoder = roberta.model.decoder.sentence_encoder model.roberta.embeddings.word_embeddings.weight = roberta_sent_encoder.embed_tokens.weight model.roberta.embeddings.position_embeddings.weight = roberta_sent_encoder.embed_positions.weight model.roberta.embeddings.token_type_embeddings.weight.data = torch.zeros_like(model.roberta.embeddings.token_type_embeddings.weight) # just zero them out b/c RoBERTa doesn't use them. @@ -79,15 +86,18 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ ### self attention self_attn: BertSelfAttention = layer.attention.self assert( - roberta_layer.self_attn.in_proj_weight.shape == torch.Size((3 * config.hidden_size, config.hidden_size)) + roberta_layer.self_attn.k_proj.weight.data.shape == \ + roberta_layer.self_attn.q_proj.weight.data.shape == \ + roberta_layer.self_attn.v_proj.weight.data.shape == \ + torch.Size((config.hidden_size, config.hidden_size)) ) - # we use three distinct linear layers so we split the source layer here. - self_attn.query.weight.data = roberta_layer.self_attn.in_proj_weight[:config.hidden_size, :] - self_attn.query.bias.data = roberta_layer.self_attn.in_proj_bias[:config.hidden_size] - self_attn.key.weight.data = roberta_layer.self_attn.in_proj_weight[config.hidden_size:2*config.hidden_size, :] - self_attn.key.bias.data = roberta_layer.self_attn.in_proj_bias[config.hidden_size:2*config.hidden_size] - self_attn.value.weight.data = roberta_layer.self_attn.in_proj_weight[2*config.hidden_size:, :] - self_attn.value.bias.data = roberta_layer.self_attn.in_proj_bias[2*config.hidden_size:] + + self_attn.query.weight.data = roberta_layer.self_attn.q_proj.weight + self_attn.query.bias.data = roberta_layer.self_attn.q_proj.bias + self_attn.key.weight.data = roberta_layer.self_attn.k_proj.weight + self_attn.key.bias.data = roberta_layer.self_attn.k_proj.bias + self_attn.value.weight.data = roberta_layer.self_attn.v_proj.weight + self_attn.value.bias.data = roberta_layer.self_attn.v_proj.bias ### self-attention output self_output: BertSelfOutput = layer.attention.output @@ -151,6 +161,7 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ if not success: raise Exception("Something went wRoNg") + pathlib.Path(pytorch_dump_folder_path).mkdir(parents=True, exist_ok=True) print(f"Saving model to {pytorch_dump_folder_path}") model.save_pretrained(pytorch_dump_folder_path) diff --git a/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py b/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py new file mode 100755 index 00000000..2b74d2dd --- /dev/null +++ b/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py @@ -0,0 +1,65 @@ +# coding=utf-8 +# Copyright 2018 The T5 authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert T5 checkpoint.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import torch + +from transformers import T5Config, T5Model, load_tf_weights_in_t5 + +import logging +logging.basicConfig(level=logging.INFO) + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, config_file, pytorch_dump_path): + # Initialise PyTorch model + config = T5Config.from_json_file(config_file) + print("Building PyTorch model from configuration: {}".format(str(config))) + model = T5Model(config) + + # Load weights from tf checkpoint + load_tf_weights_in_t5(model, config, tf_checkpoint_path) + + # Save pytorch-model + print("Save PyTorch model to {}".format(pytorch_dump_path)) + torch.save(model.state_dict(), pytorch_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path to the TensorFlow checkpoint path.") + parser.add_argument("--config_file", + default = None, + type = str, + required = True, + help = "The config json file corresponding to the pre-trained T5 model. \n" + "This specifies the model architecture.") + parser.add_argument("--pytorch_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output PyTorch model.") + args = parser.parse_args() + convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, + args.config_file, + args.pytorch_dump_path) diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index e910d6da..5567952f 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -1,6 +1,8 @@ -from .processors import InputExample, InputFeatures, DataProcessor +from .processors import InputExample, InputFeatures, DataProcessor, SquadFeatures, SingleSentenceClassificationProcessor from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features +from .processors import squad_convert_examples_to_features, SquadExample, SquadV1Processor, SquadV2Processor +from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_labels from .metrics import is_sklearn_available if is_sklearn_available(): - from .metrics import glue_compute_metrics + from .metrics import glue_compute_metrics, xnli_compute_metrics diff --git a/transformers/data/metrics/__init__.py b/transformers/data/metrics/__init__.py index c9ebaac3..5a46eb05 100644 --- a/transformers/data/metrics/__init__.py +++ b/transformers/data/metrics/__init__.py @@ -81,3 +81,11 @@ if _has_sklearn: return {"acc": simple_accuracy(preds, labels)} else: raise KeyError(task_name) + + + def xnli_compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "xnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise KeyError(task_name) diff --git a/transformers/data/metrics/squad_metrics.py b/transformers/data/metrics/squad_metrics.py new file mode 100644 index 00000000..7b03255f --- /dev/null +++ b/transformers/data/metrics/squad_metrics.py @@ -0,0 +1,763 @@ +""" Very heavily inspired by the official evaluation script for SQuAD version 2.0 which was +modified by XLNet authors to update `find_best_threshold` scripts for SQuAD V2.0 + +In addition to basic functionality, we also compute additional statistics and +plot precision-recall curves if an additional na_prob.json file is provided. +This file is expected to map question ID's to the model's predicted probability +that a question is unanswerable. +""" + + +import json +import logging +import math +import collections +from io import open +from tqdm import tqdm +import string +import re + +from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize + +logger = logging.getLogger(__name__) + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + def remove_articles(text): + regex = re.compile(r'\b(a|an|the)\b', re.UNICODE) + return re.sub(regex, ' ', text) + + def white_space_fix(text): + return ' '.join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return ''.join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def get_tokens(s): + if not s: + return [] + return normalize_answer(s).split() + + +def compute_exact(a_gold, a_pred): + return int(normalize_answer(a_gold) == normalize_answer(a_pred)) + + +def compute_f1(a_gold, a_pred): + gold_toks = get_tokens(a_gold) + pred_toks = get_tokens(a_pred) + common = collections.Counter(gold_toks) & collections.Counter(pred_toks) + num_same = sum(common.values()) + if len(gold_toks) == 0 or len(pred_toks) == 0: + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + return int(gold_toks == pred_toks) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def get_raw_scores(examples, preds): + """ + Computes the exact and f1 scores from the examples and the model predictions + """ + exact_scores = {} + f1_scores = {} + + for example in examples: + qas_id = example.qas_id + gold_answers = [answer['text'] for answer in example.answers if normalize_answer(answer['text'])] + + if not gold_answers: + # For unanswerable questions, only correct answer is empty string + gold_answers = [''] + + if qas_id not in preds: + print('Missing prediction for %s' % qas_id) + continue + + prediction = preds[qas_id] + exact_scores[qas_id] = max(compute_exact(a, prediction) for a in gold_answers) + f1_scores[qas_id] = max(compute_f1(a, prediction) for a in gold_answers) + + return exact_scores, f1_scores + + +def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh): + new_scores = {} + for qid, s in scores.items(): + pred_na = na_probs[qid] > na_prob_thresh + if pred_na: + new_scores[qid] = float(not qid_to_has_ans[qid]) + else: + new_scores[qid] = s + return new_scores + + +def make_eval_dict(exact_scores, f1_scores, qid_list=None): + if not qid_list: + total = len(exact_scores) + return collections.OrderedDict([ + ('exact', 100.0 * sum(exact_scores.values()) / total), + ('f1', 100.0 * sum(f1_scores.values()) / total), + ('total', total), + ]) + else: + total = len(qid_list) + return collections.OrderedDict([ + ('exact', 100.0 * sum(exact_scores[k] for k in qid_list) / total), + ('f1', 100.0 * sum(f1_scores[k] for k in qid_list) / total), + ('total', total), + ]) + + +def merge_eval(main_eval, new_eval, prefix): + for k in new_eval: + main_eval['%s_%s' % (prefix, k)] = new_eval[k] + + +def find_best_thresh_v2(preds, scores, na_probs, qid_to_has_ans): + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for i, qid in enumerate(qid_list): + if qid not in scores: + continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if preds[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + + has_ans_score, has_ans_cnt = 0, 0 + for qid in qid_list: + if not qid_to_has_ans[qid]: + continue + has_ans_cnt += 1 + + if qid not in scores: + continue + has_ans_score += scores[qid] + + return 100.0 * best_score / len(scores), best_thresh, 1.0 * has_ans_score / has_ans_cnt + + +def find_all_best_thresh_v2(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): + best_exact, exact_thresh, has_ans_exact = find_best_thresh_v2( + preds, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh, has_ans_f1 = find_best_thresh_v2( + preds, f1_raw, na_probs, qid_to_has_ans) + main_eval['best_exact'] = best_exact + main_eval['best_exact_thresh'] = exact_thresh + main_eval['best_f1'] = best_f1 + main_eval['best_f1_thresh'] = f1_thresh + main_eval['has_ans_exact'] = has_ans_exact + main_eval['has_ans_f1'] = has_ans_f1 + + +def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for _, qid in enumerate(qid_list): + if qid not in scores: + continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if preds[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + return 100.0 * best_score / len(scores), best_thresh + + +def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): + best_exact, exact_thresh = find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh = find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans) + + main_eval['best_exact'] = best_exact + main_eval['best_exact_thresh'] = exact_thresh + main_eval['best_f1'] = best_f1 + main_eval['best_f1_thresh'] = f1_thresh + + +def squad_evaluate(examples, preds, no_answer_probs=None, no_answer_probability_threshold=1.0): + qas_id_to_has_answer = {example.qas_id: bool(example.answers) for example in examples} + has_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if has_answer] + no_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if not has_answer] + + if no_answer_probs is None: + no_answer_probs = {k: 0.0 for k in preds} + + exact, f1 = get_raw_scores(examples, preds) + + exact_threshold = apply_no_ans_threshold(exact, no_answer_probs, qas_id_to_has_answer, no_answer_probability_threshold) + f1_threshold = apply_no_ans_threshold(f1, no_answer_probs, qas_id_to_has_answer, no_answer_probability_threshold) + + evaluation = make_eval_dict(exact_threshold, f1_threshold) + + if has_answer_qids: + has_ans_eval = make_eval_dict(exact_threshold, f1_threshold, qid_list=has_answer_qids) + merge_eval(evaluation, has_ans_eval, 'HasAns') + + if no_answer_qids: + no_ans_eval = make_eval_dict(exact_threshold, f1_threshold, qid_list=no_answer_qids) + merge_eval(evaluation, no_ans_eval, 'NoAns') + + if no_answer_probs: + find_all_best_thresh(evaluation, preds, exact, f1, no_answer_probs, qas_id_to_has_answer) + + return evaluation + + +def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False): + """Project the tokenized prediction back to the original text.""" + + # When we created the data, we kept track of the alignment between original + # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So + # now `orig_text` contains the span of our original text corresponding to the + # span that we predicted. + # + # However, `orig_text` may contain extra characters that we don't want in + # our prediction. + # + # For example, let's say: + # pred_text = steve smith + # orig_text = Steve Smith's + # + # We don't want to return `orig_text` because it contains the extra "'s". + # + # We don't want to return `pred_text` because it's already been normalized + # (the SQuAD eval script also does punctuation stripping/lower casing but + # our tokenizer does additional normalization like stripping accent + # characters). + # + # What we really want to return is "Steve Smith". + # + # Therefore, we have to apply a semi-complicated alignment heuristic between + # `pred_text` and `orig_text` to get a character-to-character alignment. This + # can fail in certain cases in which case we just return `orig_text`. + + def _strip_spaces(text): + ns_chars = [] + ns_to_s_map = collections.OrderedDict() + for (i, c) in enumerate(text): + if c == " ": + continue + ns_to_s_map[len(ns_chars)] = i + ns_chars.append(c) + ns_text = "".join(ns_chars) + return (ns_text, ns_to_s_map) + + # We first tokenize `orig_text`, strip whitespace from the result + # and `pred_text`, and check if they are the same length. If they are + # NOT the same length, the heuristic has failed. If they are the same + # length, we assume the characters are one-to-one aligned. + tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + + tok_text = " ".join(tokenizer.tokenize(orig_text)) + + start_position = tok_text.find(pred_text) + if start_position == -1: + if verbose_logging: + logger.info( + "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) + return orig_text + end_position = start_position + len(pred_text) - 1 + + (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) + (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + + if len(orig_ns_text) != len(tok_ns_text): + if verbose_logging: + logger.info("Length not equal after stripping spaces: '%s' vs '%s'", + orig_ns_text, tok_ns_text) + return orig_text + + # We then project the characters in `pred_text` back to `orig_text` using + # the character-to-character alignment. + tok_s_to_ns_map = {} + for (i, tok_index) in tok_ns_to_s_map.items(): + tok_s_to_ns_map[tok_index] = i + + orig_start_position = None + if start_position in tok_s_to_ns_map: + ns_start_position = tok_s_to_ns_map[start_position] + if ns_start_position in orig_ns_to_s_map: + orig_start_position = orig_ns_to_s_map[ns_start_position] + + if orig_start_position is None: + if verbose_logging: + logger.info("Couldn't map start position") + return orig_text + + orig_end_position = None + if end_position in tok_s_to_ns_map: + ns_end_position = tok_s_to_ns_map[end_position] + if ns_end_position in orig_ns_to_s_map: + orig_end_position = orig_ns_to_s_map[ns_end_position] + + if orig_end_position is None: + if verbose_logging: + logger.info("Couldn't map end position") + return orig_text + + output_text = orig_text[orig_start_position:(orig_end_position + 1)] + return output_text + + +def _get_best_indexes(logits, n_best_size): + """Get the n-best logits from a list.""" + index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) + + best_indexes = [] + for i in range(len(index_and_score)): + if i >= n_best_size: + break + best_indexes.append(index_and_score[i][0]) + return best_indexes + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs + + +def compute_predictions_logits( + all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + output_prediction_file, + output_nbest_file, + output_null_log_odds_file, + verbose_logging, + version_2_with_negative, + null_score_diff_threshold +): + """Write final predictions to the json file and log-odds of null if needed.""" + logger.info("Writing predictions to: %s" % (output_prediction_file)) + logger.info("Writing nbest to: %s" % (output_nbest_file)) + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", + ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + min_null_feature_index = 0 # the paragraph slice with min null score + null_start_logit = 0 # the start logit at the slice with min null score + null_end_logit = 0 # the end logit at the slice with min null score + for (feature_index, feature) in enumerate(features): + result = unique_id_to_result[feature.unique_id] + start_indexes = _get_best_indexes(result.start_logits, n_best_size) + end_indexes = _get_best_indexes(result.end_logits, n_best_size) + # if we could have irrelevant answers, get the min score of irrelevant + if version_2_with_negative: + feature_null_score = result.start_logits[0] + result.end_logits[0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = result.start_logits[0] + null_end_logit = result.end_logits[0] + for start_index in start_indexes: + for end_index in end_indexes: + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index >= len(feature.tokens): + continue + if end_index >= len(feature.tokens): + continue + if start_index not in feature.token_to_orig_map: + continue + if end_index not in feature.token_to_orig_map: + continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_logit=result.start_logits[start_index], + end_logit=result.end_logits[end_index])) + if version_2_with_negative: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=0, + end_index=0, + start_logit=null_start_logit, + end_logit=null_end_logit)) + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_logit + x.end_logit), + reverse=True) + + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + if pred.start_index > 0: # this is a non-null prediction + tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] + tok_text = " ".join(tok_tokens) + + # De-tokenize WordPieces that have been split off. + tok_text = tok_text.replace(" ##", "") + tok_text = tok_text.replace("##", "") + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + final_text = get_final_text(tok_text, orig_text, do_lower_case, verbose_logging) + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_logit=pred.start_logit, + end_logit=pred.end_logit)) + # if we didn't include the empty option in the n-best, include it + if version_2_with_negative: + if "" not in seen_predictions: + nbest.append( + _NbestPrediction( + text="", + start_logit=null_start_logit, + end_logit=null_end_logit)) + + # In very rare edge cases we could only have single null prediction. + # So we just create a nonce prediction in this case to avoid failure. + if len(nbest) == 1: + nbest.insert(0, + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = entry.start_logit + output["end_logit"] = entry.end_logit + nbest_json.append(output) + + assert len(nbest_json) >= 1 + + if not version_2_with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + # predict "" iff the null score - the score of best non-null > threshold + score_diff = score_null - best_non_null_entry.start_logit - ( + best_non_null_entry.end_logit) + scores_diff_json[example.qas_id] = score_diff + if score_diff > null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + all_nbest_json[example.qas_id] = nbest_json + + with open(output_prediction_file, "w") as writer: + writer.write(json.dumps(all_predictions, indent=4) + "\n") + + with open(output_nbest_file, "w") as writer: + writer.write(json.dumps(all_nbest_json, indent=4) + "\n") + + if version_2_with_negative: + with open(output_null_log_odds_file, "w") as writer: + writer.write(json.dumps(scores_diff_json, indent=4) + "\n") + + return all_predictions + + +def compute_predictions_log_probs( + all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + output_prediction_file, + output_nbest_file, + output_null_log_odds_file, + start_n_top, + end_n_top, + version_2_with_negative, + tokenizer, + verbose_logging +): + """ XLNet write prediction logic (more complex than Bert's). + Write final predictions to the json file and log-odds of null if needed. + + Requires utils_squad_evaluate.py + """ + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", + ["feature_index", "start_index", "end_index", + "start_log_prob", "end_log_prob"]) + + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_log_prob", "end_log_prob"]) + + logger.info("Writing predictions to: %s", output_prediction_file) + # logger.info("Writing nbest to: %s" % (output_nbest_file)) + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + + for (feature_index, feature) in enumerate(features): + result = unique_id_to_result[feature.unique_id] + + cur_null_score = result.cls_logits + + # if we could have irrelevant answers, get the min score of irrelevant + score_null = min(score_null, cur_null_score) + + for i in range(start_n_top): + for j in range(end_n_top): + start_log_prob = result.start_logits[i] + start_index = result.start_top_index[i] + + j_index = i * end_n_top + j + + end_log_prob = result.end_logits[j_index] + end_index = result.end_top_index[j_index] + + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index >= feature.paragraph_len - 1: + continue + if end_index >= feature.paragraph_len - 1: + continue + + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_log_prob=start_log_prob, + end_log_prob=end_log_prob)) + + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_log_prob + x.end_log_prob), + reverse=True) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + + # XLNet un-tokenizer + # Let's keep it simple for now and see if we need all this later. + # + # tok_start_to_orig_index = feature.tok_start_to_orig_index + # tok_end_to_orig_index = feature.tok_end_to_orig_index + # start_orig_pos = tok_start_to_orig_index[pred.start_index] + # end_orig_pos = tok_end_to_orig_index[pred.end_index] + # paragraph_text = example.paragraph_text + # final_text = paragraph_text[start_orig_pos: end_orig_pos + 1].strip() + + # Previously used Bert untokenizer + tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] + tok_text = tokenizer.convert_tokens_to_string(tok_tokens) + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + if hasattr(tokenizer, "do_lower_case"): + do_lower_case = tokenizer.do_lower_case + else: + do_lower_case = tokenizer.do_lowercase_and_remove_accent + + final_text = get_final_text(tok_text, orig_text, do_lower_case, + verbose_logging) + + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_log_prob=pred.start_log_prob, + end_log_prob=pred.end_log_prob)) + + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="", start_log_prob=-1e6, + end_log_prob=-1e6)) + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_log_prob + entry.end_log_prob) + if not best_non_null_entry: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_log_prob"] = entry.start_log_prob + output["end_log_prob"] = entry.end_log_prob + nbest_json.append(output) + + assert len(nbest_json) >= 1 + assert best_non_null_entry is not None + + score_diff = score_null + scores_diff_json[example.qas_id] = score_diff + # note(zhiliny): always predict best_non_null_entry + # and the evaluation script will search for the best threshold + all_predictions[example.qas_id] = best_non_null_entry.text + + all_nbest_json[example.qas_id] = nbest_json + + with open(output_prediction_file, "w") as writer: + writer.write(json.dumps(all_predictions, indent=4) + "\n") + + with open(output_nbest_file, "w") as writer: + writer.write(json.dumps(all_nbest_json, indent=4) + "\n") + + if version_2_with_negative: + with open(output_null_log_odds_file, "w") as writer: + writer.write(json.dumps(scores_diff_json, indent=4) + "\n") + + return all_predictions diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index af38c54b..4f7307bb 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,3 +1,4 @@ -from .utils import InputExample, InputFeatures, DataProcessor +from .utils import InputExample, InputFeatures, DataProcessor, SingleSentenceClassificationProcessor from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features - +from .squad import squad_convert_examples_to_features, SquadFeatures, SquadExample, SquadV1Processor, SquadV2Processor +from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels \ No newline at end of file diff --git a/transformers/data/processors/glue.py b/transformers/data/processors/glue.py index 518251b0..11ebd949 100644 --- a/transformers/data/processors/glue.py +++ b/transformers/data/processors/glue.py @@ -133,7 +133,7 @@ def glue_convert_examples_to_features(examples, tokenizer, if is_tf_available() and is_tf_dataset: def gen(): for ex in features: - yield ({'input_ids': ex.input_ids, + yield ({'input_ids': ex.input_ids, 'attention_mask': ex.attention_mask, 'token_type_ids': ex.token_type_ids}, ex.label) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py new file mode 100644 index 00000000..8e72bbbd --- /dev/null +++ b/transformers/data/processors/squad.py @@ -0,0 +1,655 @@ +from tqdm import tqdm +import collections +import logging +import os +import json +import numpy as np + +from ...tokenization_bert import BasicTokenizer, whitespace_tokenize +from .utils import DataProcessor, InputExample, InputFeatures +from ...file_utils import is_tf_available, is_torch_available + +if is_torch_available(): + import torch + from torch.utils.data import TensorDataset + +if is_tf_available(): + import tensorflow as tf + +logger = logging.getLogger(__name__) + + +def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, orig_answer_text): + """Returns tokenized answer spans that better match the annotated answer.""" + tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start : (new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) + + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +def _new_check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + # if len(doc_spans) == 1: + # return True + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span["start"] + doc_span["length"] - 1 + if position < doc_span["start"]: + continue + if position > end: + continue + num_left_context = position - doc_span["start"] + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span["length"] + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +def _is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False + + +def squad_convert_examples_to_features( + examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, return_dataset=False +): + """ + Converts a list of examples into a list of features that can be directly given as input to a model. + It is model-dependant and takes advantage of many of the tokenizer's features to create the model's inputs. + + Args: + examples: list of :class:`~transformers.data.processors.squad.SquadExample` + tokenizer: an instance of a child of :class:`~transformers.PreTrainedTokenizer` + max_seq_length: The maximum sequence length of the inputs. + doc_stride: The stride used when the context is too large and is split across several features. + max_query_length: The maximum length of the query. + is_training: whether to create features for model evaluation or model training. + return_dataset: Default False. Either 'pt' or 'tf'. + if 'pt': returns a torch.data.TensorDataset, + if 'tf': returns a tf.data.Dataset + + Returns: + list of :class:`~transformers.data.processors.squad.SquadFeatures` + + Example:: + + processor = SquadV2Processor() + examples = processor.get_dev_examples(data_dir) + + features = squad_convert_examples_to_features( + examples=examples, + tokenizer=tokenizer, + max_seq_length=args.max_seq_length, + doc_stride=args.doc_stride, + max_query_length=args.max_query_length, + is_training=not evaluate, + ) + """ + + # Defining helper methods + unique_id = 1000000000 + + features = [] + for (example_index, example) in enumerate(tqdm(examples, desc="Converting examples to features")): + if is_training and not example.is_impossible: + # Get start and end position + start_position = example.start_position + end_position = example.end_position + + # If the answer cannot be found in the text, then skip this example. + actual_text = " ".join(example.doc_tokens[start_position : (end_position + 1)]) + cleaned_answer_text = " ".join(whitespace_tokenize(example.answer_text)) + if actual_text.find(cleaned_answer_text) == -1: + logger.warning("Could not find answer: '%s' vs. '%s'", actual_text, cleaned_answer_text) + continue + + tok_to_orig_index = [] + orig_to_tok_index = [] + all_doc_tokens = [] + for (i, token) in enumerate(example.doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + sub_tokens = tokenizer.tokenize(token) + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + if is_training and not example.is_impossible: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + + (tok_start_position, tok_end_position) = _improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, tokenizer, example.answer_text + ) + + spans = [] + + truncated_query = tokenizer.encode( + example.question_text, add_special_tokens=False, max_length=max_query_length + ) + sequence_added_tokens = tokenizer.max_len - tokenizer.max_len_single_sentence + sequence_pair_added_tokens = tokenizer.max_len - tokenizer.max_len_sentences_pair + + span_doc_tokens = all_doc_tokens + while len(spans) * doc_stride < len(all_doc_tokens): + + encoded_dict = tokenizer.encode_plus( + truncated_query if tokenizer.padding_side == "right" else span_doc_tokens, + span_doc_tokens if tokenizer.padding_side == "right" else truncated_query, + max_length=max_seq_length, + return_overflowing_tokens=True, + pad_to_max_length=True, + stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens, + truncation_strategy="only_second" if tokenizer.padding_side == "right" else "only_first", + ) + + paragraph_len = min( + len(all_doc_tokens) - len(spans) * doc_stride, + max_seq_length - len(truncated_query) - sequence_pair_added_tokens, + ) + + if tokenizer.pad_token_id in encoded_dict["input_ids"]: + non_padded_ids = encoded_dict["input_ids"][: encoded_dict["input_ids"].index(tokenizer.pad_token_id)] + else: + non_padded_ids = encoded_dict["input_ids"] + + tokens = tokenizer.convert_ids_to_tokens(non_padded_ids) + + token_to_orig_map = {} + for i in range(paragraph_len): + index = len(truncated_query) + sequence_added_tokens + i if tokenizer.padding_side == "right" else i + token_to_orig_map[index] = tok_to_orig_index[len(spans) * doc_stride + i] + + encoded_dict["paragraph_len"] = paragraph_len + encoded_dict["tokens"] = tokens + encoded_dict["token_to_orig_map"] = token_to_orig_map + encoded_dict["truncated_query_with_special_tokens_length"] = len(truncated_query) + sequence_added_tokens + encoded_dict["token_is_max_context"] = {} + encoded_dict["start"] = len(spans) * doc_stride + encoded_dict["length"] = paragraph_len + + spans.append(encoded_dict) + + if "overflowing_tokens" not in encoded_dict: + break + span_doc_tokens = encoded_dict["overflowing_tokens"] + + for doc_span_index in range(len(spans)): + for j in range(spans[doc_span_index]["paragraph_len"]): + is_max_context = _new_check_is_max_context(spans, doc_span_index, doc_span_index * doc_stride + j) + index = ( + j + if tokenizer.padding_side == "left" + else spans[doc_span_index]["truncated_query_with_special_tokens_length"] + j + ) + spans[doc_span_index]["token_is_max_context"][index] = is_max_context + + for span in spans: + # Identify the position of the CLS token + cls_index = span["input_ids"].index(tokenizer.cls_token_id) + + # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer) + # Original TF implem also keep the classification token (set to 0) (not sure why...) + p_mask = np.array(span["token_type_ids"]) + + p_mask = np.minimum(p_mask, 1) + + if tokenizer.padding_side == "right": + # Limit positive values to one + p_mask = 1 - p_mask + + p_mask[np.where(np.array(span["input_ids"]) == tokenizer.sep_token_id)[0]] = 1 + + # Set the CLS index to '0' + p_mask[cls_index] = 0 + + span_is_impossible = example.is_impossible + start_position = 0 + end_position = 0 + if is_training and not span_is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = span["start"] + doc_end = span["start"] + span["length"] - 1 + out_of_span = False + + if not (tok_start_position >= doc_start and tok_end_position <= doc_end): + out_of_span = True + + if out_of_span: + start_position = cls_index + end_position = cls_index + span_is_impossible = True + else: + if tokenizer.padding_side == "left": + doc_offset = 0 + else: + doc_offset = len(truncated_query) + sequence_added_tokens + + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + features.append( + SquadFeatures( + span["input_ids"], + span["attention_mask"], + span["token_type_ids"], + cls_index, + p_mask.tolist(), + example_index=example_index, + unique_id=unique_id, + paragraph_len=span["paragraph_len"], + token_is_max_context=span["token_is_max_context"], + tokens=span["tokens"], + token_to_orig_map=span["token_to_orig_map"], + start_position=start_position, + end_position=end_position, + ) + ) + + unique_id += 1 + + if return_dataset == "pt": + if not is_torch_available(): + raise ImportError("Pytorch must be installed to return a pytorch dataset.") + + # Convert to Tensors and build dataset + all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) + all_attention_masks = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) + all_cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long) + all_p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.float) + + if not is_training: + all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long) + dataset = TensorDataset( + all_input_ids, all_attention_masks, all_token_type_ids, all_example_index, all_cls_index, all_p_mask + ) + else: + all_start_positions = torch.tensor([f.start_position for f in features], dtype=torch.long) + all_end_positions = torch.tensor([f.end_position for f in features], dtype=torch.long) + dataset = TensorDataset( + all_input_ids, + all_attention_masks, + all_token_type_ids, + all_start_positions, + all_end_positions, + all_cls_index, + all_p_mask, + ) + + return features, dataset + elif return_dataset == "tf": + if not is_tf_available(): + raise ImportError("TensorFlow must be installed to return a TensorFlow dataset.") + + def gen(): + for ex in features: + yield ( + { + "input_ids": ex.input_ids, + "attention_mask": ex.attention_mask, + "token_type_ids": ex.token_type_ids, + }, { + "start_position": ex.start_position, + "end_position": ex.end_position, + "cls_index": ex.cls_index, + "p_mask": ex.p_mask, + } + ) + + return tf.data.Dataset.from_generator( + gen, + ( + {"input_ids": tf.int32, "attention_mask": tf.int32, "token_type_ids": tf.int32}, + {"start_position": tf.int64, "end_position": tf.int64, "cls_index": tf.int64, "p_mask": tf.int32}, + ), + ( + { + "input_ids": tf.TensorShape([None]), + "attention_mask": tf.TensorShape([None]), + "token_type_ids": tf.TensorShape([None]), + }, + { + "start_position": tf.TensorShape([]), + "end_position": tf.TensorShape([]), + "cls_index": tf.TensorShape([]), + "p_mask": tf.TensorShape([None]), + }, + ), + ) + + return features + + +class SquadProcessor(DataProcessor): + """ + Processor for the SQuAD data set. + Overriden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and version 2.0 of SQuAD, respectively. + """ + + train_file = None + dev_file = None + + def _get_example_from_tensor_dict(self, tensor_dict, evaluate=False): + if not evaluate: + answer = tensor_dict["answers"]["text"][0].numpy().decode("utf-8") + answer_start = tensor_dict["answers"]["answer_start"][0].numpy() + answers = [] + else: + answers = [ + {"answer_start": start.numpy(), "text": text.numpy().decode("utf-8")} + for start, text in zip(tensor_dict["answers"]["answer_start"], tensor_dict["answers"]["text"]) + ] + + answer = None + answer_start = None + + return SquadExample( + qas_id=tensor_dict["id"].numpy().decode("utf-8"), + question_text=tensor_dict["question"].numpy().decode("utf-8"), + context_text=tensor_dict["context"].numpy().decode("utf-8"), + answer_text=answer, + start_position_character=answer_start, + title=tensor_dict["title"].numpy().decode("utf-8"), + answers=answers, + ) + + def get_examples_from_dataset(self, dataset, evaluate=False): + """ + Creates a list of :class:`~transformers.data.processors.squad.SquadExample` using a TFDS dataset. + + Args: + dataset: The tfds dataset loaded from `tensorflow_datasets.load("squad")` + evaluate: boolean specifying if in evaluation mode or in training mode + + Returns: + List of SquadExample + + Examples:: + + import tensorflow_datasets as tfds + dataset = tfds.load("squad") + + training_examples = get_examples_from_dataset(dataset, evaluate=False) + evaluation_examples = get_examples_from_dataset(dataset, evaluate=True) + """ + + if evaluate: + dataset = dataset["validation"] + else: + dataset = dataset["train"] + + examples = [] + for tensor_dict in tqdm(dataset): + examples.append(self._get_example_from_tensor_dict(tensor_dict, evaluate=evaluate)) + + return examples + + def get_train_examples(self, data_dir, filename=None): + """ + Returns the training examples from the data directory. + + Args: + data_dir: Directory containing the data files used for training and evaluating. + filename: None by default, specify this if the training file has a different name than the original one + which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively. + + """ + if data_dir is None: + data_dir = "" + + if self.train_file is None: + raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") + + with open( + os.path.join(data_dir, self.train_file if filename is None else filename), "r", encoding="utf-8" + ) as reader: + input_data = json.load(reader)["data"] + return self._create_examples(input_data, "train") + + def get_dev_examples(self, data_dir, filename=None): + """ + Returns the evaluation example from the data directory. + + Args: + data_dir: Directory containing the data files used for training and evaluating. + filename: None by default, specify this if the evaluation file has a different name than the original one + which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively. + """ + if data_dir is None: + data_dir = "" + + if self.dev_file is None: + raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") + + with open( + os.path.join(data_dir, self.dev_file if filename is None else filename), "r", encoding="utf-8" + ) as reader: + input_data = json.load(reader)["data"] + return self._create_examples(input_data, "dev") + + def _create_examples(self, input_data, set_type): + is_training = set_type == "train" + examples = [] + for entry in tqdm(input_data): + title = entry["title"] + for paragraph in entry["paragraphs"]: + context_text = paragraph["context"] + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position_character = None + answer_text = None + answers = [] + + if "is_impossible" in qa: + is_impossible = qa["is_impossible"] + else: + is_impossible = False + + if not is_impossible: + if is_training: + answer = qa["answers"][0] + answer_text = answer["text"] + start_position_character = answer["answer_start"] + else: + answers = qa["answers"] + + example = SquadExample( + qas_id=qas_id, + question_text=question_text, + context_text=context_text, + answer_text=answer_text, + start_position_character=start_position_character, + title=title, + is_impossible=is_impossible, + answers=answers, + ) + + examples.append(example) + return examples + + +class SquadV1Processor(SquadProcessor): + train_file = "train-v1.1.json" + dev_file = "dev-v1.1.json" + + +class SquadV2Processor(SquadProcessor): + train_file = "train-v2.0.json" + dev_file = "dev-v2.0.json" + + +class SquadExample(object): + """ + A single training/test example for the Squad dataset, as loaded from disk. + + Args: + qas_id: The example's unique identifier + question_text: The question string + context_text: The context string + answer_text: The answer string + start_position_character: The character position of the start of the answer + title: The title of the example + answers: None by default, this is used during evaluation. Holds answers as well as their start positions. + is_impossible: False by default, set to True if the example has no possible answer. + """ + + def __init__( + self, + qas_id, + question_text, + context_text, + answer_text, + start_position_character, + title, + answers=[], + is_impossible=False, + ): + self.qas_id = qas_id + self.question_text = question_text + self.context_text = context_text + self.answer_text = answer_text + self.title = title + self.is_impossible = is_impossible + self.answers = answers + + self.start_position, self.end_position = 0, 0 + + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + + # Split on whitespace so that different tokens may be attributed to their original position. + for c in self.context_text: + if _is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + self.doc_tokens = doc_tokens + self.char_to_word_offset = char_to_word_offset + + # Start end end positions only has a value during evaluation. + if start_position_character is not None and not is_impossible: + self.start_position = char_to_word_offset[start_position_character] + self.end_position = char_to_word_offset[ + min(start_position_character + len(answer_text) - 1, len(char_to_word_offset) - 1) + ] + + +class SquadFeatures(object): + """ + Single squad example features to be fed to a model. + Those features are model-specific and can be crafted from :class:`~transformers.data.processors.squad.SquadExample` + using the :method:`~transformers.data.processors.squad.squad_convert_examples_to_features` method. + + Args: + input_ids: Indices of input sequence tokens in the vocabulary. + attention_mask: Mask to avoid performing attention on padding token indices. + token_type_ids: Segment token indices to indicate first and second portions of the inputs. + cls_index: the index of the CLS token. + p_mask: Mask identifying tokens that can be answers vs. tokens that cannot. + Mask with 1 for tokens than cannot be in the answer and 0 for token that can be in an answer + example_index: the index of the example + unique_id: The unique Feature identifier + paragraph_len: The length of the context + token_is_max_context: List of booleans identifying which tokens have their maximum context in this feature object. + If a token does not have their maximum context in this feature object, it means that another feature object + has more information related to that token and should be prioritized over this feature for that token. + tokens: list of tokens corresponding to the input ids + token_to_orig_map: mapping between the tokens and the original text, needed in order to identify the answer. + start_position: start of the answer token index + end_position: end of the answer token index + """ + + def __init__( + self, + input_ids, + attention_mask, + token_type_ids, + cls_index, + p_mask, + example_index, + unique_id, + paragraph_len, + token_is_max_context, + tokens, + token_to_orig_map, + start_position, + end_position, + ): + self.input_ids = input_ids + self.attention_mask = attention_mask + self.token_type_ids = token_type_ids + self.cls_index = cls_index + self.p_mask = p_mask + + self.example_index = example_index + self.unique_id = unique_id + self.paragraph_len = paragraph_len + self.token_is_max_context = token_is_max_context + self.tokens = tokens + self.token_to_orig_map = token_to_orig_map + + self.start_position = start_position + self.end_position = end_position + + +class SquadResult(object): + """ + Constructs a SquadResult which can be used to evaluate a model's output on the SQuAD dataset. + + Args: + unique_id: The unique identifier corresponding to that example. + start_logits: The logits corresponding to the start of the answer + end_logits: The logits corresponding to the end of the answer + """ + + def __init__(self, unique_id, start_logits, end_logits, start_top_index=None, end_top_index=None, cls_logits=None): + self.start_logits = start_logits + self.end_logits = end_logits + self.unique_id = unique_id + + if start_top_index: + self.start_top_index = start_top_index + self.end_top_index = end_top_index + self.cls_logits = cls_logits diff --git a/transformers/data/processors/utils.py b/transformers/data/processors/utils.py index 07bdf315..ee234e6e 100644 --- a/transformers/data/processors/utils.py +++ b/transformers/data/processors/utils.py @@ -18,6 +18,11 @@ import csv import sys import copy import json +import logging + +from ...file_utils import is_tf_available, is_torch_available + +logger = logging.getLogger(__name__) class InputExample(object): """ @@ -64,7 +69,7 @@ class InputFeatures(object): label: Label corresponding to the input """ - def __init__(self, input_ids, attention_mask, token_type_ids, label): + def __init__(self, input_ids, attention_mask=None, token_type_ids=None, label=None): self.input_ids = input_ids self.attention_mask = attention_mask self.token_type_ids = token_type_ids @@ -86,34 +91,6 @@ class InputFeatures(object): class DataProcessor(object): """Base class for data converters for sequence classification data sets.""" - def get_example_from_tensor_dict(self, tensor_dict): - """Gets an example from a dict with tensorflow tensors - - Args: - tensor_dict: Keys and values should match the corresponding Glue - tensorflow_dataset examples. - """ - raise NotImplementedError() - - def get_train_examples(self, data_dir): - """Gets a collection of `InputExample`s for the train set.""" - raise NotImplementedError() - - def get_dev_examples(self, data_dir): - """Gets a collection of `InputExample`s for the dev set.""" - raise NotImplementedError() - - def get_labels(self): - """Gets the list of labels for this data set.""" - raise NotImplementedError() - - def tfds_map(self, example): - """Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. - This method converts examples to the correct format.""" - if len(self.get_labels()) > 1: - example.label = self.get_labels()[int(example.label)] - return example - @classmethod def _read_tsv(cls, input_file, quotechar=None): """Reads a tab separated value file.""" @@ -125,3 +102,215 @@ class DataProcessor(object): line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines + + +class SingleSentenceClassificationProcessor(DataProcessor): + """ Generic processor for a single sentence classification data set.""" + def __init__(self, labels=None, examples=None, mode='classification', verbose=False): + self.labels = [] if labels is None else labels + self.examples = [] if examples is None else examples + self.mode = mode + self.verbose = verbose + + def __len__(self): + return len(self.examples) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return SingleSentenceClassificationProcessor(labels=self.labels, + examples=self.examples[idx]) + return self.examples[idx] + + @classmethod + def create_from_csv(cls, file_name, split_name='', column_label=0, column_text=1, + column_id=None, skip_first_row=False, **kwargs): + processor = cls(**kwargs) + processor.add_examples_from_csv(file_name, + split_name=split_name, + column_label=column_label, + column_text=column_text, + column_id=column_id, + skip_first_row=skip_first_row, + overwrite_labels=True, + overwrite_examples=True) + return processor + + @classmethod + def create_from_examples(cls, texts_or_text_and_labels, labels=None, **kwargs): + processor = cls(**kwargs) + processor.add_examples(texts_or_text_and_labels, labels=labels) + return processor + + def add_examples_from_csv(self, file_name, split_name='', column_label=0, column_text=1, column_id=None, + skip_first_row=False, overwrite_labels=False, overwrite_examples=False): + lines = self._read_tsv(file_name) + if skip_first_row: + lines = lines[1:] + texts = [] + labels = [] + ids = [] + for (i, line) in enumerate(lines): + texts.append(line[column_text]) + labels.append(line[column_label]) + if column_id is not None: + ids.append(line[column_id]) + else: + guid = "%s-%s" % (split_name, i) if split_name else "%s" % i + ids.append(guid) + + return self.add_examples(texts, labels, ids, overwrite_labels=overwrite_labels, overwrite_examples=overwrite_examples) + + def add_examples(self, texts_or_text_and_labels, labels=None, ids=None, + overwrite_labels=False, overwrite_examples=False): + assert labels is None or len(texts_or_text_and_labels) == len(labels) + assert ids is None or len(texts_or_text_and_labels) == len(ids) + if ids is None: + ids = [None] * len(texts_or_text_and_labels) + if labels is None: + labels = [None] * len(texts_or_text_and_labels) + examples = [] + added_labels = set() + for (text_or_text_and_label, label, guid) in zip(texts_or_text_and_labels, labels, ids): + if isinstance(text_or_text_and_label, (tuple, list)) and label is None: + text, label = text_or_text_and_label + else: + text = text_or_text_and_label + added_labels.add(label) + examples.append(InputExample(guid=guid, text_a=text, text_b=None, label=label)) + + # Update examples + if overwrite_examples: + self.examples = examples + else: + self.examples.extend(examples) + + # Update labels + if overwrite_labels: + self.labels = list(added_labels) + else: + self.labels = list(set(self.labels).union(added_labels)) + + return self.examples + + def get_features(self, + tokenizer, + max_length=None, + pad_on_left=False, + pad_token=0, + mask_padding_with_zero=True, + return_tensors=None): + """ + Convert examples in a list of ``InputFeatures`` + + Args: + tokenizer: Instance of a tokenizer that will tokenize the examples + max_length: Maximum example length + task: GLUE task + label_list: List of labels. Can be obtained from the processor using the ``processor.get_labels()`` method + output_mode: String indicating the output mode. Either ``regression`` or ``classification`` + pad_on_left: If set to ``True``, the examples will be padded on the left rather than on the right (default) + pad_token: Padding token + mask_padding_with_zero: If set to ``True``, the attention mask will be filled by ``1`` for actual values + and by ``0`` for padded values. If set to ``False``, inverts it (``1`` for padded values, ``0`` for + actual values) + + Returns: + If the ``examples`` input is a ``tf.data.Dataset``, will return a ``tf.data.Dataset`` + containing the task-specific features. If the input is a list of ``InputExamples``, will return + a list of task-specific ``InputFeatures`` which can be fed to the model. + + """ + if max_length is None: + max_length = tokenizer.max_len + + label_map = {label: i for i, label in enumerate(self.labels)} + + all_input_ids = [] + for (ex_index, example) in enumerate(self.examples): + if ex_index % 10000 == 0: + logger.info("Tokenizing example %d", ex_index) + + input_ids = tokenizer.encode( + example.text_a, + add_special_tokens=True, + max_length=min(max_length, tokenizer.max_len), + ) + all_input_ids.append(input_ids) + + batch_length = max(len(input_ids) for input_ids in all_input_ids) + + features = [] + for (ex_index, (input_ids, example)) in enumerate(zip(all_input_ids, self.examples)): + if ex_index % 10000 == 0: + logger.info("Writing example %d", ex_index) + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) + + # Zero-pad up to the sequence length. + padding_length = batch_length - len(input_ids) + if pad_on_left: + input_ids = ([pad_token] * padding_length) + input_ids + attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask + else: + input_ids = input_ids + ([pad_token] * padding_length) + attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length) + + assert len(input_ids) == batch_length, "Error with input length {} vs {}".format(len(input_ids), batch_length) + assert len(attention_mask) == batch_length, "Error with input length {} vs {}".format(len(attention_mask), batch_length) + + if self.mode == "classification": + label = label_map[example.label] + elif self.mode == "regression": + label = float(example.label) + else: + raise ValueError(self.mode) + + if ex_index < 5 and self.verbose: + logger.info("*** Example ***") + logger.info("guid: %s" % (example.guid)) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask])) + logger.info("label: %s (id = %d)" % (example.label, label)) + + features.append( + InputFeatures(input_ids=input_ids, + attention_mask=attention_mask, + label=label)) + + if return_tensors is None: + return features + elif return_tensors == 'tf': + if not is_tf_available(): + raise ImportError("return_tensors set to 'tf' but TensorFlow 2.0 can't be imported") + import tensorflow as tf + def gen(): + for ex in features: + yield ({'input_ids': ex.input_ids, + 'attention_mask': ex.attention_mask}, + ex.label) + + dataset = tf.data.Dataset.from_generator(gen, + ({'input_ids': tf.int32, + 'attention_mask': tf.int32}, + tf.int64), + ({'input_ids': tf.TensorShape([None]), + 'attention_mask': tf.TensorShape([None])}, + tf.TensorShape([]))) + return dataset + elif return_tensors == 'pt': + if not is_torch_available(): + raise ImportError("return_tensors set to 'pt' but PyTorch can't be imported") + import torch + from torch.utils.data import TensorDataset + all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) + all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + if self.mode == "classification": + all_labels = torch.tensor([f.label for f in features], dtype=torch.long) + elif self.mode == "regression": + all_labels = torch.tensor([f.label for f in features], dtype=torch.float) + + dataset = TensorDataset(all_input_ids, all_attention_mask, all_labels) + return dataset + else: + raise ValueError("return_tensors should be one of 'tf' or 'pt'") diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py new file mode 100644 index 00000000..958bdf62 --- /dev/null +++ b/transformers/data/processors/xnli.py @@ -0,0 +1,85 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XNLI utils (dataset loading and evaluation) """ + +from __future__ import absolute_import, division, print_function + +import logging +import os + +from .utils import DataProcessor, InputExample + +logger = logging.getLogger(__name__) + +class XnliProcessor(DataProcessor): + """Processor for the XNLI dataset. + Adapted from https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/run_classifier.py#L207""" + + def __init__(self, language, train_language = None): + self.language = language + self.train_language = train_language + + def get_train_examples(self, data_dir): + """See base class.""" + lg = self.language if self.train_language is None else self.train_language + lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{}.tsv".format(lg))) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % ('train', i) + text_a = line[0] + text_b = line[1] + label = "contradiction" if line[2] == "contradictory" else line[2] + assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_test_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.test.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + language = line[0] + if language != self.language: + continue + guid = "%s-%s" % ('test', i) + text_a = line[6] + text_b = line[7] + label = line[1] + assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + +xnli_processors = { + "xnli": XnliProcessor, +} + +xnli_output_modes = { + "xnli": "classification", +} + +xnli_tasks_num_labels = { + "xnli": 3, +} diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 27875212..ec925c61 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -10,10 +10,9 @@ import json import logging import os import six -import shutil import tempfile import fnmatch -from functools import wraps +from functools import partial, wraps from hashlib import sha256 from io import open @@ -21,25 +20,38 @@ import boto3 from botocore.config import Config from botocore.exceptions import ClientError import requests -from tqdm import tqdm +from tqdm.auto import tqdm +from contextlib import contextmanager +from . import __version__ + +from filelock import FileLock logger = logging.getLogger(__name__) # pylint: disable=invalid-name try: - import tensorflow as tf - assert hasattr(tf, '__version__') and int(tf.__version__[0]) >= 2 - _tf_available = True # pylint: disable=invalid-name - logger.info("TensorFlow version {} available.".format(tf.__version__)) -except (ImportError, AssertionError): - _tf_available = False # pylint: disable=invalid-name - -try: - import torch - _torch_available = True # pylint: disable=invalid-name - logger.info("PyTorch version {} available.".format(torch.__version__)) + os.environ.setdefault('USE_TORCH', 'YES') + if os.environ['USE_TORCH'].upper() in ('1', 'ON', 'YES'): + import torch + _torch_available = True # pylint: disable=invalid-name + logger.info("PyTorch version {} available.".format(torch.__version__)) + else: + logger.info("USE_TORCH override through env variable, disabling PyTorch") + _torch_available = False except ImportError: _torch_available = False # pylint: disable=invalid-name +try: + os.environ.setdefault('USE_TF', 'YES') + if os.environ['USE_TF'].upper() in ('1', 'ON', 'YES'): + import tensorflow as tf + assert hasattr(tf, '__version__') and int(tf.__version__[0]) >= 2 + _tf_available = True # pylint: disable=invalid-name + logger.info("TensorFlow version {} available.".format(tf.__version__)) + else: + logger.info("USE_TF override through env variable, disabling Tensorflow") + _tf_available = False +except (ImportError, AssertionError): + _tf_available = False # pylint: disable=invalid-name try: from torch.hub import _get_torch_home @@ -71,11 +83,20 @@ WEIGHTS_NAME = "pytorch_model.bin" TF2_WEIGHTS_NAME = 'tf_model.h5' TF_WEIGHTS_NAME = 'model.ckpt' CONFIG_NAME = "config.json" +MODEL_CARD_NAME = "modelcard.json" + +DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] +DUMMY_MASK = [[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]] + +S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert" +CLOUDFRONT_DISTRIB_PREFIX = "https://d2ws9o8vfrpkyk.cloudfront.net" + def is_torch_available(): return _torch_available def is_tf_available(): + return _tf_available if not six.PY2: @@ -102,12 +123,25 @@ else: return fn return docstring_decorator + +def is_remote_url(url_or_filename): + parsed = urlparse(url_or_filename) + return parsed.scheme in ('http', 'https', 's3') + +def hf_bucket_url(identifier, postfix=None, cdn=False): + endpoint = CLOUDFRONT_DISTRIB_PREFIX if cdn else S3_BUCKET_PREFIX + if postfix is None: + return "/".join((endpoint, identifier)) + else: + return "/".join((endpoint, identifier, postfix)) + + def url_to_filename(url, etag=None): """ Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. - If the url ends with .h5 (Keras HDF5 weights) ands '.h5' to the name + If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can identify it as a HDF5 file (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) """ @@ -152,7 +186,7 @@ def filename_to_url(filename, cache_dir=None): return url, etag -def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None): +def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None, resume_download=False, user_agent=None): """ Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and @@ -161,6 +195,8 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N Args: cache_dir: specify a cache directory to save the file to (overwrite the default cache dir). force_download: if True, re-dowload the file even if it's already cached in the cache dir. + resume_download: if True, resume the download if incompletly recieved file is found. + user_agent: Optional string or dict that will be appended to the user-agent on remote requests. """ if cache_dir is None: cache_dir = TRANSFORMERS_CACHE @@ -169,15 +205,15 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N if sys.version_info[0] == 3 and isinstance(cache_dir, Path): cache_dir = str(cache_dir) - parsed = urlparse(url_or_filename) - - if parsed.scheme in ('http', 'https', 's3'): + if is_remote_url(url_or_filename): # URL, so get it from the cache (downloading if necessary) - return get_from_cache(url_or_filename, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + return get_from_cache(url_or_filename, cache_dir=cache_dir, + force_download=force_download, proxies=proxies, + resume_download=resume_download, user_agent=user_agent) elif os.path.exists(url_or_filename): # File, and it exists. return url_or_filename - elif parsed.scheme == '': + elif urlparse(url_or_filename).scheme == '': # File, but it doesn't exist. raise EnvironmentError("file {} not found".format(url_or_filename)) else: @@ -234,19 +270,34 @@ def s3_get(url, temp_file, proxies=None): s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) -def http_get(url, temp_file, proxies=None): - req = requests.get(url, stream=True, proxies=proxies) - content_length = req.headers.get('Content-Length') - total = int(content_length) if content_length is not None else None - progress = tqdm(unit="B", total=total) - for chunk in req.iter_content(chunk_size=1024): +def http_get(url, temp_file, proxies=None, resume_size=0, user_agent=None): + ua = "transformers/{}; python/{}".format(__version__, sys.version.split()[0]) + if isinstance(user_agent, dict): + ua += "; " + "; ".join( + "{}/{}".format(k, v) for k, v in user_agent.items() + ) + elif isinstance(user_agent, six.string_types): + ua += "; "+ user_agent + headers = { + "user-agent": ua + } + if resume_size > 0: + headers['Range'] = 'bytes=%d-' % (resume_size,) + response = requests.get(url, stream=True, proxies=proxies, headers=headers) + if response.status_code == 416: # Range not satisfiable + return + content_length = response.headers.get('Content-Length') + total = resume_size + int(content_length) if content_length is not None else None + progress = tqdm(unit="B", unit_scale=True, total=total, initial=resume_size, + desc="Downloading", disable=bool(logger.level<=logging.INFO)) + for chunk in response.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks progress.update(len(chunk)) temp_file.write(chunk) progress.close() -def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10): +def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10, resume_download=False, user_agent=None): """ Given a URL, look for the corresponding dataset in the local cache. If it's not there, download it. Then return the path to the cached file. @@ -284,41 +335,60 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag # If we don't have a connection (etag is None) and can't identify the file # try to get the last downloaded one if not os.path.exists(cache_path) and etag is None: - matching_files = fnmatch.filter(os.listdir(cache_dir), filename + '.*') - matching_files = list(filter(lambda s: not s.endswith('.json'), matching_files)) + matching_files = [ + file + for file in fnmatch.filter(os.listdir(cache_dir), filename + '.*') + if not file.endswith('.json') and not file.endswith('.lock') + ] if matching_files: cache_path = os.path.join(cache_dir, matching_files[-1]) - if not os.path.exists(cache_path) or force_download: - # Download to temporary file, then copy to cache dir once finished. - # Otherwise you get corrupt cache entries if the download gets interrupted. - with tempfile.NamedTemporaryFile() as temp_file: - logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name) + # Prevent parallel downloads of the same file with a lock. + lock_path = cache_path + '.lock' + with FileLock(lock_path): - # GET file object - if url.startswith("s3://"): - s3_get(url, temp_file, proxies=proxies) + if resume_download: + incomplete_path = cache_path + '.incomplete' + @contextmanager + def _resumable_file_manager(): + with open(incomplete_path,'a+b') as f: + yield f + temp_file_manager = _resumable_file_manager + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size else: - http_get(url, temp_file, proxies=proxies) + resume_size = 0 + else: + temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False) + resume_size = 0 - # we are copying the file before closing it, so flush to avoid truncation - temp_file.flush() - # shutil.copyfileobj() starts at the current position, so go to the start - temp_file.seek(0) + if etag is not None and (not os.path.exists(cache_path) or force_download): + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with temp_file_manager() as temp_file: + logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name) - logger.info("copying %s to cache at %s", temp_file.name, cache_path) - with open(cache_path, 'wb') as cache_file: - shutil.copyfileobj(temp_file, cache_file) + # GET file object + if url.startswith("s3://"): + if resume_download: + logger.warn('Warning: resumable downloads are not implemented for "s3://" urls') + s3_get(url, temp_file, proxies=proxies) + else: + http_get(url, temp_file, proxies=proxies, resume_size=resume_size, user_agent=user_agent) - logger.info("creating metadata file for %s", cache_path) - meta = {'url': url, 'etag': etag} - meta_path = cache_path + '.json' - with open(meta_path, 'w') as meta_file: - output_string = json.dumps(meta) - if sys.version_info[0] == 2 and isinstance(output_string, str): - output_string = unicode(output_string, 'utf-8') # The beauty of python 2 - meta_file.write(output_string) + # we are copying the file before closing it, so flush to avoid truncation + temp_file.flush() - logger.info("removing temp file %s", temp_file.name) + logger.info("storing %s in cache at %s", url, cache_path) + os.rename(temp_file.name, cache_path) + + logger.info("creating metadata file for %s", cache_path) + meta = {'url': url, 'etag': etag} + meta_path = cache_path + '.json' + with open(meta_path, 'w') as meta_file: + output_string = json.dumps(meta) + if sys.version_info[0] == 2 and isinstance(output_string, str): + output_string = unicode(output_string, 'utf-8') # The beauty of python 2 + meta_file.write(output_string) return cache_path diff --git a/transformers/hf_api.py b/transformers/hf_api.py new file mode 100644 index 00000000..17073233 --- /dev/null +++ b/transformers/hf_api.py @@ -0,0 +1,229 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function + +import os +from os.path import expanduser + +import requests +import six +from requests.exceptions import HTTPError +from tqdm import tqdm + +ENDPOINT = "https://huggingface.co" + +class S3Obj: + def __init__( + self, + filename, # type: str + LastModified, # type: str + ETag, # type: str + Size, # type: int + **kwargs + ): + self.filename = filename + self.LastModified = LastModified + self.ETag = ETag + self.Size = Size + + +class PresignedUrl: + def __init__( + self, + write, # type: str + access, # type: str + type, # type: str + **kwargs + ): + self.write = write + self.access = access + self.type = type # mime-type to send to S3. + + +class HfApi: + def __init__(self, endpoint=None): + self.endpoint = endpoint if endpoint is not None else ENDPOINT + + def login( + self, + username, # type: str + password, # type: str + ): + # type: (...) -> str + """ + Call HF API to sign in a user and get a token if credentials are valid. + + Outputs: + token if credentials are valid + + Throws: + requests.exceptions.HTTPError if credentials are invalid + """ + path = "{}/api/login".format(self.endpoint) + r = requests.post(path, json={"username": username, "password": password}) + r.raise_for_status() + d = r.json() + return d["token"] + + def whoami( + self, + token, # type: str + ): + # type: (...) -> str + """ + Call HF API to know "whoami" + """ + path = "{}/api/whoami".format(self.endpoint) + r = requests.get(path, headers={"authorization": "Bearer {}".format(token)}) + r.raise_for_status() + d = r.json() + return d["user"] + + def logout(self, token): + # type: (...) -> void + """ + Call HF API to log out. + """ + path = "{}/api/logout".format(self.endpoint) + r = requests.post(path, headers={"authorization": "Bearer {}".format(token)}) + r.raise_for_status() + + def presign(self, token, filename): + # type: (...) -> PresignedUrl + """ + Call HF API to get a presigned url to upload `filename` to S3. + """ + path = "{}/api/presign".format(self.endpoint) + r = requests.post( + path, + headers={"authorization": "Bearer {}".format(token)}, + json={"filename": filename}, + ) + r.raise_for_status() + d = r.json() + return PresignedUrl(**d) + + def presign_and_upload(self, token, filename, filepath): + # type: (...) -> str + """ + Get a presigned url, then upload file to S3. + + Outputs: + url: Read-only url for the stored file on S3. + """ + urls = self.presign(token, filename=filename) + # streaming upload: + # https://2.python-requests.org/en/master/user/advanced/#streaming-uploads + # + # Even though we presign with the correct content-type, + # the client still has to specify it when uploading the file. + with open(filepath, "rb") as f: + pf = TqdmProgressFileReader(f) + data = f if pf.total_size > 0 else "" + + r = requests.put(urls.write, data=data, headers={ + "content-type": urls.type, + }) + r.raise_for_status() + pf.close() + return urls.access + + def list_objs(self, token): + # type: (...) -> List[S3Obj] + """ + Call HF API to list all stored files for user. + """ + path = "{}/api/listObjs".format(self.endpoint) + r = requests.get(path, headers={"authorization": "Bearer {}".format(token)}) + r.raise_for_status() + d = r.json() + return [S3Obj(**x) for x in d] + + + +class TqdmProgressFileReader: + """ + Wrap an io.BufferedReader `f` (such as the output of `open(…, "rb")`) + and override `f.read()` so as to display a tqdm progress bar. + + see github.com/huggingface/transformers/pull/2078#discussion_r354739608 + for implementation details. + """ + def __init__( + self, + f # type: io.BufferedReader + ): + self.f = f + self.total_size = os.fstat(f.fileno()).st_size # type: int + self.pbar = tqdm(total=self.total_size, leave=False) + if six.PY3: + # does not work unless PY3 + # no big deal as the CLI does not currently support PY2 anyways. + self.read = f.read + f.read = self._read + + def _read(self, n=-1): + self.pbar.update(n) + return self.read(n) + + def close(self): + self.pbar.close() + + + +class HfFolder: + path_token = expanduser("~/.huggingface/token") + + @classmethod + def save_token(cls, token): + """ + Save token, creating folder as needed. + """ + if six.PY3: + os.makedirs(os.path.dirname(cls.path_token), exist_ok=True) + else: + # Python 2 + try: + os.makedirs(os.path.dirname(cls.path_token)) + except OSError as e: + if e.errno != os.errno.EEXIST: + raise e + pass + with open(cls.path_token, 'w+') as f: + f.write(token) + + @classmethod + def get_token(cls): + """ + Get token or None if not existent. + """ + try: + with open(cls.path_token, 'r') as f: + return f.read() + except: + # this is too wide. When Py2 is dead use: + # `except FileNotFoundError:` instead + return None + + @classmethod + def delete_token(cls): + """ + Delete token. + Do not fail if token does not exist. + """ + try: + os.remove(cls.path_token) + except: + return diff --git a/transformers/modelcard.py b/transformers/modelcard.py new file mode 100644 index 00000000..4a879235 --- /dev/null +++ b/transformers/modelcard.py @@ -0,0 +1,229 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Configuration base class and utilities.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import copy +import json +import logging +import os +from io import open + +from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP + +from .file_utils import CONFIG_NAME, MODEL_CARD_NAME, WEIGHTS_NAME, TF2_WEIGHTS_NAME, \ + cached_path, is_remote_url, hf_bucket_url + + +logger = logging.getLogger(__name__) + + +class ModelCard(object): + r""" Model Card class. + Store model card as well as methods for loading/downloading/saving model cards. + + Please read the following paper for details and explanation on the sections: + "Model Cards for Model Reporting" + by Margaret Mitchell, Simone Wu, + Andrew Zaldivar, Parker Barnes, Lucy Vasserman, Ben Hutchinson, Elena Spitzer, + Inioluwa Deborah Raji and Timnit Gebru for the proposal behind model cards. + Link: https://arxiv.org/abs/1810.03993 + + Note: + A model card can be loaded and saved to disk. + + Parameters: + """ + def __init__(self, **kwargs): + # Recomended attributes from https://arxiv.org/abs/1810.03993 (see papers) + self.model_details = kwargs.pop('model_details', {}) + self.intended_use = kwargs.pop('intended_use', {}) + self.factors = kwargs.pop('factors', {}) + self.metrics = kwargs.pop('metrics', {}) + self.evaluation_data = kwargs.pop('evaluation_data', {}) + self.training_data = kwargs.pop('training_data', {}) + self.quantitative_analyses = kwargs.pop('quantitative_analyses', {}) + self.ethical_considerations = kwargs.pop('ethical_considerations', {}) + self.caveats_and_recommendations = kwargs.pop('caveats_and_recommendations', {}) + + # Open additional attributes + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error("Can't set {} with value {} for {}".format(key, value, self)) + raise err + + def save_pretrained(self, save_directory_or_file): + """ Save a model card object to the directory or file `save_directory_or_file`. + """ + if os.path.isdir(save_directory_or_file): + # If we save using the predefined names, we can load using `from_pretrained` + output_model_card_file = os.path.join(save_directory_or_file, MODEL_CARD_NAME) + else: + output_model_card_file = save_directory_or_file + + self.to_json_file(output_model_card_file) + logger.info("Model card saved in {}".format(output_model_card_file)) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + r""" Instantiate a :class:`~transformers.ModelCard` from a pre-trained model model card. + + Parameters: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model card to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model card that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. + - a path to a `directory` containing a mode card file saved using the :func:`~transformers.ModelCard.save_pretrained` method, e.g.: ``./my_model_directory/``. + - a path or url to a saved model card JSON `file`, e.g.: ``./my_model_directory/modelcard.json``. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + card should be cached if the standard cache should not be used. + + kwargs: (`optional`) dict: key/value pairs with which to update the ModelCard object after loading. + + - The values in kwargs of any keys which are model card attributes will be used to override the loaded values. + - Behavior concerning key/value pairs whose keys are *not* model card attributes is controlled by the `return_unused_kwargs` keyword parameter. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + find_from_standard_name: (`optional`) boolean, default True: + If the pretrained_model_name_or_path ends with our standard model or config filenames, replace them with our standard modelcard filename. + Can be used to directly feed a model/config url and access the colocated modelcard. + + return_unused_kwargs: (`optional`) bool: + + - If False, then this function returns just the final model card object. + - If True, then this functions returns a tuple `(model card, unused_kwargs)` where `unused_kwargs` is a dictionary consisting of the key/value pairs whose keys are not model card attributes: ie the part of kwargs which has not been used to update `ModelCard` and is otherwise ignored. + + Examples:: + + modelcard = ModelCard.from_pretrained('bert-base-uncased') # Download model card from S3 and cache. + modelcard = ModelCard.from_pretrained('./test/saved_model/') # E.g. model card was saved using `save_pretrained('./test/saved_model/')` + modelcard = ModelCard.from_pretrained('./test/saved_model/modelcard.json') + modelcard = ModelCard.from_pretrained('bert-base-uncased', output_attention=True, foo=False) + + """ + cache_dir = kwargs.pop('cache_dir', None) + proxies = kwargs.pop('proxies', None) + find_from_standard_name = kwargs.pop('find_from_standard_name', True) + return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) + + if pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: + # For simplicity we use the same pretrained url than the configuration files + # but with a different suffix (modelcard.json). This suffix is replaced below. + model_card_file = ALL_PRETRAINED_CONFIG_ARCHIVE_MAP[pretrained_model_name_or_path] + elif os.path.isdir(pretrained_model_name_or_path): + model_card_file = os.path.join(pretrained_model_name_or_path, MODEL_CARD_NAME) + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): + model_card_file = pretrained_model_name_or_path + else: + model_card_file = hf_bucket_url(pretrained_model_name_or_path, postfix=MODEL_CARD_NAME) + + if find_from_standard_name or pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: + model_card_file = model_card_file.replace(CONFIG_NAME, MODEL_CARD_NAME) + model_card_file = model_card_file.replace(WEIGHTS_NAME, MODEL_CARD_NAME) + model_card_file = model_card_file.replace(TF2_WEIGHTS_NAME, MODEL_CARD_NAME) + + try: + # Load from URL or cache if already cached + resolved_model_card_file = cached_path(model_card_file, cache_dir=cache_dir, force_download=True, + proxies=proxies, resume_download=False) + if resolved_model_card_file == model_card_file: + logger.info("loading model card file {}".format(model_card_file)) + else: + logger.info("loading model card file {} from cache at {}".format( + model_card_file, resolved_model_card_file)) + # Load model card + modelcard = cls.from_json_file(resolved_model_card_file) + + except EnvironmentError: + if pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: + logger.warning("Couldn't reach server at '{}' to download model card file.".format( + model_card_file)) + else: + logger.warning("Model name '{}' was not found in model name list ({}). " \ + "We assumed '{}' was a path or url to a model card file named {} or " \ + "a directory containing such a file but couldn't find any such file at this path or url.".format( + pretrained_model_name_or_path, + ', '.join(ALL_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()), + model_card_file, MODEL_CARD_NAME)) + logger.warning("Creating an empty model card.") + + # We fall back on creating an empty model card + modelcard = cls() + + except json.JSONDecodeError: + logger.warning("Couldn't reach server at '{}' to download model card file or " + "model card file is not a valid JSON file. " + "Please check network or file content here: {}.".format(model_card_file, resolved_model_card_file)) + logger.warning("Creating an empty model card.") + + # We fall back on creating an empty model card + modelcard = cls() + + # Update model card with kwargs if needed + to_remove = [] + for key, value in kwargs.items(): + if hasattr(modelcard, key): + setattr(modelcard, key, value) + to_remove.append(key) + for key in to_remove: + kwargs.pop(key, None) + + logger.info("Model card: %s", str(modelcard)) + if return_unused_kwargs: + return modelcard, kwargs + else: + return modelcard + + @classmethod + def from_dict(cls, json_object): + """Constructs a `ModelCard` from a Python dictionary of parameters.""" + return cls(**json_object) + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `ModelCard` from a json file of parameters.""" + with open(json_file, "r", encoding='utf-8') as reader: + text = reader.read() + dict_obj = json.loads(text) + return cls(**dict_obj) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path): + """ Save this instance to a json file.""" + with open(json_file_path, "w", encoding='utf-8') as writer: + writer.write(self.to_json_string()) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py new file mode 100644 index 00000000..0f67bf8f --- /dev/null +++ b/transformers/modeling_albert.py @@ -0,0 +1,801 @@ + +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch ALBERT model. """ + +import os +import math +import logging +import torch +import torch.nn as nn +from torch.nn import CrossEntropyLoss, MSELoss +from transformers.modeling_utils import PreTrainedModel +from transformers.configuration_albert import AlbertConfig +from transformers.modeling_bert import BertEmbeddings, BertSelfAttention, prune_linear_layer, ACT2FN +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + + +ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-pytorch_model.bin", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-pytorch_model.bin", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-pytorch_model.bin", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-pytorch_model.bin", +} + + +def load_tf_weights_in_albert(model, config, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model.""" + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + for name, array in zip(names, arrays): + print(name) + + for name, array in zip(names, arrays): + original_name = name + + # If saved from the TF HUB module + name = name.replace("module/", "") + + # Renaming and simplifying + name = name.replace("ffn_1", "ffn") + name = name.replace("bert/", "albert/") + name = name.replace("attention_1", "attention") + name = name.replace("transform/", "") + name = name.replace("LayerNorm_1", "full_layer_layer_norm") + name = name.replace("LayerNorm", "attention/LayerNorm") + name = name.replace("transformer/", "") + + # The feed forward layer had an 'intermediate' step which has been abstracted away + name = name.replace("intermediate/dense/", "") + name = name.replace("ffn/intermediate/output/dense/", "ffn_output/") + + # ALBERT attention was split between self and output which have been abstracted away + name = name.replace("/output/", "/") + name = name.replace("/self/", "/") + + # The pooler is a linear layer + name = name.replace("pooler/dense", "pooler") + + # The classifier was simplified to predictions from cls/predictions + name = name.replace("cls/predictions", "predictions") + name = name.replace("predictions/attention", "predictions") + + # Naming was changed to be more explicit + name = name.replace("embeddings/attention", "embeddings") + name = name.replace("inner_group_", "albert_layers/") + name = name.replace("group_", "albert_layer_groups/") + + # Classifier + if len(name.split("/")) == 1 and ("output_bias" in name or "output_weights" in name): + name = "classifier/" + name + + # No ALBERT model currently handles the next sentence prediction task + if "seq_relationship" in name: + continue + + name = name.split('/') + + # Ignore the gradients applied by the LAMB/ADAM optimizers. + if "adam_m" in name or "adam_v" in name or "global_step" in name: + logger.info("Skipping {}".format("/".join(name))) + continue + + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + else: + l = [m_name] + + if l[0] == 'kernel' or l[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif l[0] == 'output_bias' or l[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif l[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif l[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, l[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + print("Initialize PyTorch weight {} from {}".format(name, original_name)) + pointer.data = torch.from_numpy(array) + + return model + + +class AlbertEmbeddings(BertEmbeddings): + """ + Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config): + super(AlbertEmbeddings, self).__init__(config) + + self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size) + self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) + + +class AlbertAttention(BertSelfAttention): + def __init__(self, config): + super(AlbertAttention, self).__init__(config) + + self.output_attentions = config.output_attentions + self.num_attention_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.attention_head_size = config.hidden_size // config.num_attention_heads + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.dense = prune_linear_layer(self.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward(self, input_ids, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(input_ids) + mixed_key_layer = self.key(input_ids) + mixed_value_layer = self.value(input_ids) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + reshaped_context_layer = context_layer.view(*new_context_layer_shape) + + + # Should find a better way to do this + w = self.dense.weight.t().view(self.num_attention_heads, self.attention_head_size, self.hidden_size).to(context_layer.dtype) + b = self.dense.bias.to(context_layer.dtype) + + projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b + projected_context_layer_dropout = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout) + return (layernormed_context_layer, attention_probs) if self.output_attentions else (layernormed_context_layer,) + + +class AlbertLayer(nn.Module): + def __init__(self, config): + super(AlbertLayer, self).__init__() + + self.config = config + self.full_layer_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.attention = AlbertAttention(config) + self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) + self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + self.activation = ACT2FN[config.hidden_act] + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + attention_output = self.attention(hidden_states, attention_mask, head_mask) + ffn_output = self.ffn(attention_output[0]) + ffn_output = self.activation(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.full_layer_layer_norm(ffn_output + attention_output[0]) + + return (hidden_states,) + attention_output[1:] # add attentions if we output them + + +class AlbertLayerGroup(nn.Module): + def __init__(self, config): + super(AlbertLayerGroup, self).__init__() + + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + layer_hidden_states = () + layer_attentions = () + + for layer_index, albert_layer in enumerate(self.albert_layers): + layer_output = albert_layer(hidden_states, attention_mask, head_mask[layer_index]) + hidden_states = layer_output[0] + + if self.output_attentions: + layer_attentions = layer_attentions + (layer_output[1],) + + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (layer_hidden_states,) + if self.output_attentions: + outputs = outputs + (layer_attentions,) + return outputs # last-layer hidden state, (layer hidden states), (layer attentions) + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config = config + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) + self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + + all_attentions = () + + if self.output_hidden_states: + all_hidden_states = (hidden_states,) + + for i in range(self.config.num_hidden_layers): + # Number of layers in a hidden group + layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups) + + # Index of the hidden group + group_idx = int(i / (self.config.num_hidden_layers / self.config.num_hidden_groups)) + + # Index of the layer inside the group + layer_idx = int(i - group_idx * layers_per_group) + + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]) + hidden_states = layer_group_output[0] + + if self.output_attentions: + all_attentions = all_attentions + layer_group_output[-1] + + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + + + +class AlbertPreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "albert" + + def _init_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if isinstance(module, (nn.Linear)) and module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + +ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in + `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ + by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents + two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`: + https://arxiv.org/abs/1909.11942 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ALBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.AlbertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertModel(AlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + """ + + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_albert + base_model_prefix = "albert" + + def __init__(self, config): + super(AlbertModel, self).__init__(config) + + self.config = config + self.embeddings = AlbertEmbeddings(config) + self.encoder = AlbertTransformer(config) + self.pooler = nn.Linear(config.hidden_size, config.hidden_size) + self.pooler_activation = nn.Tanh() + + self.init_weights() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _resize_token_embeddings(self, new_num_tokens): + old_embeddings = self.embeddings.word_embeddings + new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) + self.embeddings.word_embeddings = new_embeddings + return self.embeddings.word_embeddings + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + ALBERT has a different architecture in that its layers are shared across groups, which then has inner groups. + If an ALBERT model has 12 hidden layers and 2 hidden groups, with two inner groups, there + is a total of 4 different layers. + + These layers are flattened: the indices [0,1] correspond to the two inner groups of the first hidden layer, + while [2,3] correspond to the two inner groups of the second hidden layer. + + Any layer with in index other than [0,1,2,3] will result in an error. + See base class PreTrainedModel for more information about head pruning + """ + for layer, heads in heads_to_prune.items(): + group_idx = int(layer / self.config.inner_group_num) + inner_group_idx = int(layer - group_idx * self.config.inner_group_num) + self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads) + + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + inputs_embeds=None): + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + device = input_ids.device if input_ids is not None else inputs_embeds.device + + if attention_mask is None: + attention_mask = torch.ones(input_shape, device=device) + if token_type_ids is None: + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) + + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds) + encoder_outputs = self.encoder(embedding_output, + extended_attention_mask, + head_mask=head_mask) + + sequence_output = encoder_outputs[0] + + pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) + + outputs = (sequence_output, pooled_output) + encoder_outputs[1:] # add hidden_states and attentions if they are here + return outputs + +class AlbertMLMHead(nn.Module): + def __init__(self, config): + super(AlbertMLMHead, self).__init__() + + self.LayerNorm = nn.LayerNorm(config.embedding_size) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + self.dense = nn.Linear(config.hidden_size, config.embedding_size) + self.decoder = nn.Linear(config.embedding_size, config.vocab_size) + self.activation = ACT2FN[config.hidden_act] + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + hidden_states = self.decoder(hidden_states) + + prediction_scores = hidden_states + self.bias + + return prediction_scores + + +@add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForMaskedLM(AlbertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + """ + + def __init__(self, config): + super(AlbertForMaskedLM, self).__init__(config) + + self.albert = AlbertModel(config) + self.predictions = AlbertMLMHead(config) + + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.predictions.decoder, + self.albert.embeddings.word_embeddings) + + def get_output_embeddings(self): + return self.predictions.decoder + + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, + masked_lm_labels=None): + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) + sequence_outputs = outputs[0] + + prediction_scores = self.predictions(sequence_outputs) + + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + if masked_lm_labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + outputs = (masked_lm_loss,) + outputs + + return outputs + + +@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForSequenceClassification(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = AlbertForSequenceClassification.from_pretrained('albert-base-v2') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config): + super(AlbertForSequenceClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.albert = AlbertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, self.config.num_labels) + + self.init_weights() + + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): + + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), logits, (hidden_states), (attentions) + + + +@add_start_docstrings("""Albert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForQuestionAnswering(AlbertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = AlbertForQuestionAnswering.from_pretrained('albert-base-v2') + question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" + input_text = "[CLS] " + question + " [SEP] " + text + " [SEP]" + input_ids = tokenizer.encode(input_text) + token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] + start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids])) + all_tokens = tokenizer.convert_ids_to_tokens(input_ids) + print(' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])) + # a nice puppet + + + """ + def __init__(self, config): + super(AlbertForQuestionAnswering, self).__init__(config) + self.num_labels = config.num_labels + + self.albert = AlbertModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + inputs_embeds=None, start_positions=None, end_positions=None): + + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + outputs = (total_loss,) + outputs + + return outputs # (loss), start_logits, end_logits, (hidden_states), (attentions) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index d98110d4..6b49efd3 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -18,15 +18,31 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging -from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering -from .modeling_openai import OpenAIGPTModel, OpenAIGPTLMHeadModel -from .modeling_gpt2 import GPT2Model, GPT2LMHeadModel -from .modeling_ctrl import CTRLModel, CTRLLMHeadModel -from .modeling_transfo_xl import TransfoXLModel, TransfoXLLMHeadModel -from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering -from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering -from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification -from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification +from .configuration_auto import (AlbertConfig, BertConfig, CamembertConfig, CTRLConfig, + DistilBertConfig, GPT2Config, OpenAIGPTConfig, RobertaConfig, + TransfoXLConfig, XLMConfig, XLNetConfig, XLMRobertaConfig) + +from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering, \ + BertForTokenClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_openai import OpenAIGPTModel, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_gpt2 import GPT2Model, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_ctrl import CTRLModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_transfo_xl import TransfoXLModel, TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering, \ + XLNetForTokenClassification, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering, \ + XLM_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, \ + RobertaForTokenClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, \ + DistilBertForSequenceClassification, DistilBertForTokenClassification, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, \ + CamembertForMultipleChoice, CamembertForTokenClassification, CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, \ + AlbertForQuestionAnswering, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_t5 import T5Model, T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_xlm_roberta import XLMRobertaModel, XLMRobertaForMaskedLM, XLMRobertaForSequenceClassification, \ + XLMRobertaForMultipleChoice, XLMRobertaForTokenClassification, XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_utils import PreTrainedModel, SequenceSummary @@ -35,33 +51,105 @@ from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) +ALL_PRETRAINED_MODEL_ARCHIVE_MAP = dict((key, value) + for pretrained_map in [ + BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, + TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, + GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, + CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, + XLM_PRETRAINED_MODEL_ARCHIVE_MAP, + ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + T5_PRETRAINED_MODEL_ARCHIVE_MAP, + XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + ] + for key, value, in pretrained_map.items()) + + class AutoModel(object): r""" :class:`~transformers.AutoModel` is a generic model class that will be instantiated as one of the base model classes of the library when created with the `AutoModel.from_pretrained(pretrained_model_name_or_path)` - class method. + or the `AutoModel.from_config(config)` class methods. The `from_pretrained()` method takes care of returning the correct model class instance using pattern matching on the `pretrained_model_name_or_path` string. The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Model (T5 model) - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `albert`: AlbertModel (ALBERT model) + - contains `camembert`: CamembertModel (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaModel (XLM-RoBERTa model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) - - contains `ctrl`: CTRLModel (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) - contains `xlnet`: XLNetModel (XLNet model) - contains `xlm`: XLMModel (XLM model) + - contains `ctrl`: CTRLModel (Salesforce CTRL model) This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): raise EnvironmentError("AutoModel is designed to be instantiated " - "using the `AutoModel.from_pretrained(pretrained_model_name_or_path)` method.") + "using the `AutoModel.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModel.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `openai-gpt` configuration class: OpenAIGPTModel (OpenAI GPT model) + - isInstance of `gpt2` configuration class: GPT2Model (OpenAI GPT-2 model) + - isInstance of `ctrl` configuration class: CTRLModel (Salesforce CTRL model) + - isInstance of `transfo-xl` configuration class: TransfoXLModel (Transformer-XL model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModel.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return DistilBertModel(config) + elif isinstance(config, RobertaConfig): + return RobertaModel(config) + elif isinstance(config, BertConfig): + return BertModel(config) + elif isinstance(config, OpenAIGPTConfig): + return OpenAIGPTModel(config) + elif isinstance(config, GPT2Config): + return GPT2Model(config) + elif isinstance(config, TransfoXLConfig): + return TransfoXLModel(config) + elif isinstance(config, XLNetConfig): + return XLNetModel(config) + elif isinstance(config, XLMConfig): + return XLMModel(config) + elif isinstance(config, CTRLConfig): + return CTRLModel(config) + elif isinstance(config, AlbertConfig): + return AlbertModel(config) + elif isinstance(config, CamembertConfig): + return CamembertModel(config) + elif isinstance(config, XLMRobertaConfig): + return XLMRobertaModel(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -70,15 +158,19 @@ class AutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Model (T5 model) - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `albert`: AlbertModel (ALBERT model) + - contains `camembert`: CamembertModel (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaModel (XLM-RoBERTa model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) - - contains `ctrl`: CTRLModel (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) - contains `xlnet`: XLNetModel (XLNet model) - contains `xlm`: XLMModel (XLM model) + - contains `ctrl`: CTRLModel (Salesforce CTRL model) The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with `model.train()` @@ -87,6 +179,7 @@ class AutoModel(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. @@ -112,6 +205,9 @@ class AutoModel(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -136,8 +232,16 @@ class AutoModel(object): model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return T5Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -156,7 +260,7 @@ class AutoModel(object): return CTRLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta, 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm-roberta', 'xlm', 'roberta, 'ctrl', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) class AutoModelWithLMHead(object): @@ -171,21 +275,70 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5ModelWithLMHead (T5 model) - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `albert`: AlbertForMaskedLM (ALBERT model) + - contains `camembert`: CamembertForMaskedLM (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaForMaskedLM (XLM-RoBERTa model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) - contains `gpt2`: GPT2LMHeadModel (OpenAI GPT-2 model) - - contains `ctrl`: CTRLLMModel (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) - contains `xlnet`: XLNetLMHeadModel (XLNet model) - contains `xlm`: XLMWithLMHeadModel (XLM model) + - contains `ctrl`: CTRLLMHeadModel (Salesforce CTRL model) This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): raise EnvironmentError("AutoModelWithLMHead is designed to be instantiated " - "using the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + "using the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelWithLMHead.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `openai-gpt` configuration class: OpenAIGPTModel (OpenAI GPT model) + - isInstance of `gpt2` configuration class: GPT2Model (OpenAI GPT-2 model) + - isInstance of `ctrl` configuration class: CTRLModel (Salesforce CTRL model) + - isInstance of `transfo-xl` configuration class: TransfoXLModel (Transformer-XL model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelWithLMHead.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return DistilBertForMaskedLM(config) + elif isinstance(config, RobertaConfig): + return RobertaForMaskedLM(config) + elif isinstance(config, BertConfig): + return BertForMaskedLM(config) + elif isinstance(config, OpenAIGPTConfig): + return OpenAIGPTLMHeadModel(config) + elif isinstance(config, GPT2Config): + return GPT2LMHeadModel(config) + elif isinstance(config, TransfoXLConfig): + return TransfoXLLMHeadModel(config) + elif isinstance(config, XLNetConfig): + return XLNetLMHeadModel(config) + elif isinstance(config, XLMConfig): + return XLMWithLMHeadModel(config) + elif isinstance(config, CTRLConfig): + return CTRLLMHeadModel(config) + elif isinstance(config, XLMRobertaConfig): + return XLMRobertaForMaskedLM(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -197,7 +350,11 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5ModelWithLMHead (T5 model) - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `albert`: AlbertForMaskedLM (ALBERT model) + - contains `camembert`: CamembertForMaskedLM (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaForMaskedLM (XLM-RoBERTa model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -205,6 +362,7 @@ class AutoModelWithLMHead(object): - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) - contains `xlnet`: XLNetLMHeadModel (XLNet model) - contains `xlm`: XLMWithLMHeadModel (XLM model) + - contains `ctrl`: CTRLLMHeadModel (Salesforce CTRL model) The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with `model.train()` @@ -213,6 +371,7 @@ class AutoModelWithLMHead(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. @@ -237,6 +396,8 @@ class AutoModelWithLMHead(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. @@ -262,8 +423,16 @@ class AutoModelWithLMHead(object): model = AutoModelWithLMHead.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return T5WithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -282,7 +451,7 @@ class AutoModelWithLMHead(object): return CTRLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta','ctrl'".format(pretrained_model_name_or_path)) + "'xlm-roberta', 'xlm', 'roberta','ctrl', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) class AutoModelForSequenceClassification(object): @@ -298,6 +467,9 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `albert`: AlbertForSequenceClassification (ALBERT model) + - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaForSequenceClassification (XLM-RoBERTa model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -306,8 +478,45 @@ class AutoModelForSequenceClassification(object): This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): - raise EnvironmentError("AutoModelWithLMHead is designed to be instantiated " - "using the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + raise EnvironmentError("AutoModelForSequenceClassification is designed to be instantiated " + "using the `AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelForSequenceClassification.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForSequenceClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, AlbertConfig): + return AlbertForSequenceClassification(config) + elif isinstance(config, CamembertConfig): + return CamembertForSequenceClassification(config) + elif isinstance(config, DistilBertConfig): + return DistilBertForSequenceClassification(config) + elif isinstance(config, RobertaConfig): + return RobertaForSequenceClassification(config) + elif isinstance(config, BertConfig): + return BertForSequenceClassification(config) + elif isinstance(config, XLNetConfig): + return XLNetForSequenceClassification(config) + elif isinstance(config, XLMConfig): + return XLMForSequenceClassification(config) + elif isinstance(config, XLMRobertaConfig): + return XLMRobertaForSequenceClassification(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -320,6 +529,9 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `albert`: AlbertForSequenceClassification (ALBERT model) + - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaForSequenceClassification (XLM-RoBERTa model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -332,6 +544,7 @@ class AutoModelForSequenceClassification(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. @@ -357,6 +570,9 @@ class AutoModelForSequenceClassification(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -383,6 +599,12 @@ class AutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -393,7 +615,7 @@ class AutoModelForSequenceClassification(object): return XLMForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)) + "'bert', 'xlnet', 'xlm-roberta', 'xlm', 'roberta', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) class AutoModelForQuestionAnswering(object): @@ -409,6 +631,7 @@ class AutoModelForQuestionAnswering(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `albert`: AlbertForQuestionAnswering (ALBERT model) - contains `bert`: BertForQuestionAnswering (Bert model) - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) - contains `xlm`: XLMForQuestionAnswering (XLM model) @@ -416,8 +639,38 @@ class AutoModelForQuestionAnswering(object): This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): - raise EnvironmentError("AutoModelWithLMHead is designed to be instantiated " - "using the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + raise EnvironmentError("AutoModelForQuestionAnswering is designed to be instantiated " + "using the `AutoModelForQuestionAnswering.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelForQuestionAnswering.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForSequenceClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, AlbertConfig): + return AlbertForQuestionAnswering(config) + elif isinstance(config, DistilBertConfig): + return DistilBertForQuestionAnswering(config) + elif isinstance(config, BertConfig): + return BertForQuestionAnswering(config) + elif isinstance(config, XLNetConfig): + return XLNetForQuestionAnswering(config) + elif isinstance(config, XLMConfig): + return XLMForQuestionAnswering(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -430,6 +683,7 @@ class AutoModelForQuestionAnswering(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `albert`: AlbertForQuestionAnswering (ALBERT model) - contains `bert`: BertForQuestionAnswering (Bert model) - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) - contains `xlm`: XLMForQuestionAnswering (XLM model) @@ -441,6 +695,7 @@ class AutoModelForQuestionAnswering(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. @@ -492,6 +747,8 @@ class AutoModelForQuestionAnswering(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: return BertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlnet' in pretrained_model_name_or_path: @@ -500,4 +757,131 @@ class AutoModelForQuestionAnswering(object): return XLMForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) + "'bert', 'xlnet', 'xlm', 'distilbert', 'albert'".format(pretrained_model_name_or_path)) + + +class AutoModelForTokenClassification: + def __init__(self): + raise EnvironmentError("AutoModelForTokenClassification is designed to be instantiated " + "using the `AutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelForTokenClassification.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `camembert` configuration class: CamembertModel (Camembert model) + - isInstance of `roberta` configuration class: RobertaModel (Roberta model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForTokenClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, CamembertConfig): + return CamembertForTokenClassification(config) + elif isinstance(config, DistilBertConfig): + return DistilBertForTokenClassification(config) + elif isinstance(config, BertConfig): + return BertForTokenClassification(config) + elif isinstance(config, XLNetConfig): + return XLNetForTokenClassification(config) + elif isinstance(config, RobertaConfig): + return RobertaForTokenClassification(config) + elif isinstance(config, XLMRobertaConfig): + return XLMRobertaForTokenClassification(config) + raise ValueError("Unrecognized configuration class {}".format(config)) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the question answering model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForTokenClassification (DistilBERT model) + - contains `camembert`: CamembertForTokenClassification (Camembert model) + - contains `bert`: BertForTokenClassification (Bert model) + - contains `xlnet`: XLNetForTokenClassification (XLNet model) + - contains `roberta`: RobertaForTokenClassification (Roberta model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = AutoModelForTokenClassification.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = AutoModelForTokenClassification.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = AutoModelForTokenClassification.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = AutoModelForTokenClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'camembert' in pretrained_model_name_or_path: + return CamembertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: + return DistilBertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'roberta' in pretrained_model_name_or_path: + return RobertaForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'bert' in pretrained_model_name_or_path: + return BertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlnet' in pretrained_model_name_or_path: + return XLNetForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'xlnet', 'camembert', 'distilbert', 'xlm-roberta', 'roberta'".format(pretrained_model_name_or_path)) diff --git a/transformers/modeling_beam_search.py b/transformers/modeling_beam_search.py deleted file mode 100644 index 171dcb72..00000000 --- a/transformers/modeling_beam_search.py +++ /dev/null @@ -1,271 +0,0 @@ -# coding=utf-8 -# Copyright (c) 2019 Yang Liu - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -""" -A general wrapper around models with LM heads to generate sequences -using beam search. -""" -import torch -from torch import nn - - -class TransformerBeamSearch(nn.Module): - def __init__( - self, - model, - tokenizer, - batch_size, - beam_size, - min_length, - max_length, - alpha=0, - block_repeating_trigram=True, - ): - """ - Attributes: - mask_word_id: token id that corresponds to the mask - """ - super(TransformerBeamSearch, self).__init__() - self.model = model - self.tokenizer = tokenizer - - self.start_token_id = tokenizer.start_token_id - self.end_token_id = tokenizer.end_token_id - self.pad_token_id = tokenizer.pad_token_id - - self.beam_size = beam_size - self.min_length = min_length - self.max_length = max_length - - self.block_repeating_trigram = block_repeating_trigram - self.apply_length_penalty = False if alpha == 0 else True - self.alpha = alpha - - # State of the beam - self.hypotheses = [[] for _ in range(batch_size)] - self.batch_offset = torch.arange(batch_size, dtype=torch.long) - self.beam_offset = torch.arange( - 0, batch_size * self.beam_size, step=self.beam_size, dtype=torch.long - ) - self.growing_beam = torch.full( - (batch_size * self.beam_size, 1), self.start_token_id, dtype=torch.long - ) - self.topk_log_probabilities = torch.tensor( - [0.0] + [float("-inf")] * (self.beam_size - 1), dtype=torch.float - ).repeat(batch_size) - self.results = { - "prediction": [[] for _ in batch_size], - "scores": [[] for _ in batch_size], - } - self._step = 0 - self.is_done = False - - def step(self, log_probabilities): - """ Grows the beam by one step. """ - self._step += 1 - - # The batch size changes as some beams finish so we define _B - vocab_size = log_probabilities.size(-1) - _B = log_probabilities.size(0) // self.beam_size - - # Multiply each beam probability with the probability of the - # next token (conditioned on the words in the beam). - log_probabilities += self.topk_log_probabilities.view(-1, 1) - - self.enforce_min_length(log_probabilities) - if self.block_repeating_trigram: - self.remove_repeating_trigrams(log_probabilities, _B) - - # Find the `beam_size` (previous_beam + token) combinations with - # the highest score - topk_log_probabilities, topk_ids = log_probabilities.topk( - log_probabilities.view(_B, self.beam_size * vocab_size), - self.beam_size, - dim=1, - ) - - # Apply the length penalty. The +1 accounts for the [EOS] token - # that will be added if the beam ends. - topk_scores = topk_log_probabilities / self.length_penalty() - - # Retrieve the corresponding respective beam and token id - # topk_token_ids[i] will be added to topk_beam_ids[i] - topk_beam_ids = topk_ids.div(vocab_size) - topk_token_ids = topk_ids.fmod(vocab_size) - - # Retrieve the row index of the surviving beams in the original - # view of the log_probabilities tensor - surviving_beams_rows = (topk_beam_ids + self.beam_offset[:_B].view(-1, 1)).view( - -1 - ) - - # Append the last predictions - self.growing_beam = torch.cat( - [ - self.growing_beam.index_select(0, surviving_beams_rows), - topk_token_ids.view(-1, 1), - ], - 1, - ) - - # Check if any of the beam searches has ended during this - # growth step. Also if top beam (most probable) has ended - # for one element of the batch. - is_finished = topk_token_ids.eq(self.end_token_id) - self.enforce_max_length() - is_top_beam_finished = is_finished[:, 0].eq(1) - - # Save the finished searches - if is_finished.any(): - predictions = self.growing_beam.view( - -1, self.beam_size, self.growing_beam.size(1) - ) - for i in range(is_finished.size(0)): - if is_top_beam_finished[i]: - is_finished[i].fill_(1) - finished_hyp = is_finished[i].nonzero().view(-1) - - # Store finished hypotheses for this batch. - b = self.batch_offset[i] - for j in finished_hyp: - self.hypotheses[b].append((topk_scores[i, j], predictions[i, j, :])) - - # If the batch reached the end, save the best hypotheses - # in terms of length-penalized score. - if is_top_beam_finished[i]: - best_hyp = sorted( - self.hypotheses[b], key=lambda x: x[0], reverse=True - ) - best_score, best_prediction = best_hyp[0] - self.results["scores"][b].append(best_score) - self.results["predictions"][b].append(best_prediction) - - non_finished = is_top_beam_finished.eq(0).nonzero().view(-1) - if len(non_finished) == 0: - self.is_done = True - - # Remove finished batches for the next step. - topk_log_probabilities = topk_log_probabilities.index_select( - 0, non_finished - ) - self.batch_offset = self.batch_offset.index_select(0, non_finished) - self.growing_beam = predictions.index_select(0, non_finished).view( - -1, self.growing_beam.size(-1) - ) - - surviving_beams_rows = surviving_beams_rows.index_select(0, non_finished) - - return surviving_beams_rows - - def forward(self, encoder_input_ids, **kwargs): - # keyword arguments come in 3 flavors: encoder-specific (prefixed by - # `encoder_`), decoder-specific (prefixed by `decoder_`) and those - # that apply to the model as whole. - # We let the specific kwargs override the common ones in case of conflict. - kwargs_encoder = { - argument[len("encoder_"):]: value - for argument, value in kwargs.items() - if argument.startswith("encoder_") - } - kwargs_decoder = { - argument[len("decoder_"):]: value - for argument, value in kwargs.items() - if argument.startswith("decoder_") - } - kwargs_common = { - argument: value - for argument, value in kwargs.items() - if not (argument.startswith("encoder_") or argument.startswith("decoder_")) - } - kwargs_decoder = dict(kwargs_common, **kwargs_decoder) - kwargs_encoder = dict(kwargs_common, **kwargs_encoder) - - # forward pass on the encoder - encoder_outputs = self.model.encoder.forward(encoder_input_ids, kwargs_encoder) - kwargs_decoder["encoder_hidden_states"] = tile( - encoder_outputs, self.beam_size, dim=0 - ) - - # grow the beam by generating sequences in an autoregressive way - self.growing_beam = torch.full( - (self.batch_size * self.beam_size, 1), self.start_token_id, dtype=torch.long - ) - for step in range(self.max_length): - decoder_input = self.growing_beam[:, -1] - outputs = self.model.decoder(decoder_input, kwargs_decoder) - log_probabilities = torch.nn.functional.log_softmax(outputs[1]) - surviving_beams_rows = self.step(log_probabilities) - if self.is_done: - break - - kwargs_decoder["encoder_hidden_states"] = kwargs_decoder[ - "encoder_hidden_states" - ].index_select(0, surviving_beams_rows) - - return self.results - - def remove_repeating_trigrams(self, log_probabilities, _B): - if(self._step + 1 > 3): - for i in range(_B * self.beam_size): - tokens = [t for t in self.growing_beam[i]] - trigrams = [(tokens[i-1], tokens[i], tokens[i+1]) for i in range(1, len(words) - 1)] - last_trigram = tuple(trigrams[-1]) - if last_trigram in trigrams[:-1]: - log_probabilities[i] = -1e20 - - def enforce_min_length(self): - if self._step < self.min_length: - self.log_probabilities[self.end_token_id] = -1e20 - - def enforce_max_length(self): - if self._step + 1 == self.max_length: - self.is_finished.fill_(1) - - def length_penalty(self): - return ((5.0 + (self._step + 1)) / 6.0) ** self.alpha - - -def tile(x, count, dim=0): - """ - Tiles `x` along dimension `dim` `count` times. - - Example: - >> ex = torch.tensor([1,2],[3,4]) - >> tile(ex, 2, 0) - torch.Tensor([[1,2],[1,2],[3,4],[3,4]]) - """ - perm = list(range(len(x.size()))) - if dim != 0: - perm[0], perm[dim] = perm[dim], perm[0] - x = x.permute(perm).contiguous() - out_size = list(x.size()) - out_size[0] *= count - batch = x.size(0) - x = ( - x.view(batch, -1) - .transpose(0, 1) - .repeat(count, 1) - .transpose(0, 1) - .contiguous() - .view(*out_size) - ) - if dim != 0: - x = x.permute(perm).contiguous() - return x diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 7c2c6f46..afeb9d8e 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -48,6 +48,12 @@ BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-pytorch_model.bin", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-pytorch_model.bin", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-pytorch_model.bin", + 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-pytorch_model.bin", + 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-pytorch_model.bin", + 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-pytorch_model.bin", + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-pytorch_model.bin", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/pytorch_model.bin", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1/pytorch_model.bin", } @@ -138,7 +144,11 @@ def swish(x): return x * torch.sigmoid(x) -ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_new": gelu_new} +def mish(x): + return x * torch.tanh(nn.functional.softplus(x)) + + +ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_new": gelu_new, "mish": mish} BertLayerNorm = torch.nn.LayerNorm @@ -278,7 +288,7 @@ class BertAttention(nn.Module): if len(heads) == 0: return mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + heads = set(heads) - self.pruned_heads # Convert to set and remove already pruned heads for head in heads: # Compute how many pruned heads are before the head and move the index accordingly head = head - sum(1 if h < head else 0 for h in self.pruned_heads) @@ -597,7 +607,7 @@ class BertModel(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertModel.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -656,8 +666,6 @@ class BertModel(BertPreTrainedModel): if attention_mask is None: attention_mask = torch.ones(input_shape, device=device) - if encoder_attention_mask is None: - encoder_attention_mask = torch.ones(input_shape, device=device) if token_type_ids is None: token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) @@ -665,18 +673,20 @@ class BertModel(BertPreTrainedModel): # ourselves in which case we just need to make it broadcastable to all heads. if attention_mask.dim() == 3: extended_attention_mask = attention_mask[:, None, :, :] - - # Provided a padding mask of dimensions [batch_size, seq_length] - # - if the model is a decoder, apply a causal mask in addition to the padding mask - # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] - if attention_mask.dim() == 2: + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] if self.config.is_decoder: batch_size, seq_length = input_shape seq_ids = torch.arange(seq_length, device=device) causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] + causal_mask = causal_mask.to(torch.long) # not converting to long will cause errors with pytorch version < 1.3 extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] else: extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError("Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(input_shape, attention_mask.shape)) # Since attention_mask is 1.0 for positions we want to attend and 0.0 for # masked positions, this operation will create a tensor which is 0.0 for @@ -688,13 +698,24 @@ class BertModel(BertPreTrainedModel): # If a 2D ou 3D attention mask is provided for the cross-attention # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] - if encoder_attention_mask.dim() == 3: - encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] - if encoder_attention_mask.dim() == 2: - encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device) - encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility - encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + if encoder_attention_mask.dim() == 3: + encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] + elif encoder_attention_mask.dim() == 2: + encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + else: + raise ValueError("Wrong shape for encoder_hidden_shape (shape {}) or encoder_attention_mask (shape {})".format(encoder_hidden_shape, + encoder_attention_mask.shape)) + + encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + else: + encoder_extended_attention_mask = None # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head @@ -760,7 +781,7 @@ class BertForPreTraining(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForPreTraining.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) prediction_scores, seq_relationship_scores = outputs[:2] @@ -836,7 +857,7 @@ class BertForMaskedLM(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForMaskedLM.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) loss, prediction_scores = outputs[:2] @@ -919,7 +940,7 @@ class BertForNextSentencePrediction(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) seq_relationship_scores = outputs[0] @@ -984,7 +1005,7 @@ class BertForSequenceClassification(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForSequenceClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -1060,7 +1081,7 @@ class BertForMultipleChoice(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForMultipleChoice.from_pretrained('bert-base-uncased') choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] - input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices labels = torch.tensor(1).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, classification_scores = outputs[:2] @@ -1134,7 +1155,7 @@ class BertForTokenClassification(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForTokenClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, scores = outputs[:2] @@ -1218,9 +1239,9 @@ class BertForQuestionAnswering(BertPreTrainedModel): question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" input_text = "[CLS] " + question + " [SEP] " + text + " [SEP]" input_ids = tokenizer.encode(input_text) - token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] + token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids])) - all_tokens = tokenizer.convert_ids_to_tokens(input_ids) + all_tokens = tokenizer.convert_ids_to_tokens(input_ids) print(' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])) # a nice puppet diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py new file mode 100644 index 00000000..f302346f --- /dev/null +++ b/transformers/modeling_camembert.py @@ -0,0 +1,293 @@ +# coding=utf-8 +# Copyright 2019 Inria, Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch CamemBERT model. """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice, RobertaForTokenClassification +from .configuration_camembert import CamembertConfig +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin", +} + + +CAMEMBERT_START_DOCSTRING = r""" The CamemBERT model was proposed in + `CamemBERT: a Tasty French Language Model`_ + by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suárez, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah, and Benoît Sagot. It is based on Facebook's RoBERTa model released in 2019. + + It is a model trained on 138GB of French text. + + This implementation is the same as RoBERTa. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`CamemBERT: a Tasty French Language Model`: + https://arxiv.org/abs/1911.03894 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.CamembertConfig`): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +CAMEMBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, CamemBERT input sequence should be formatted with and tokens as follows: + + (a) For sequence pairs: + + ``tokens: Is this Jacksonville ? No it is not . `` + + (b) For single sequences: + + ``tokens: the dog is hairy . `` + + Fully encoded sequences or sequence pairs can be obtained using the CamembertTokenizer.encode function with + the ``add_special_tokens`` parameter set to ``True``. + + CamemBERT is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional` need to be trained) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Optional segment token indices to indicate first and second portions of the inputs. + This embedding matrice is not trained (not pretrained during CamemBERT pretraining), you will have to train it + during finetuning. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1[``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. +""" + +@add_start_docstrings("The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top.", + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertModel(RobertaModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + eo match pre-training, CamemBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertModel.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""CamemBERT Model with a `language modeling` head on top. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForMaskedLM(RobertaForMaskedLM): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForMaskedLM.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer + on top of the pooled output) e.g. for GLUE tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForSequenceClassification(RobertaForSequenceClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForSequenceClassification.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""CamemBERT Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForMultipleChoice(RobertaForMultipleChoice): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForMultipleChoice.from_pretrained('camembert-base') + choices = ["J'aime le camembert !", "Je deteste le camembert !"] + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""CamemBERT Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForTokenClassification(RobertaForTokenClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForTokenClassification.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !", add_special_tokens=True)).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py index 1ed9e6eb..af7f2c3a 100644 --- a/transformers/modeling_ctrl.py +++ b/transformers/modeling_ctrl.py @@ -63,7 +63,8 @@ def scaled_dot_product_attention(q, k, v, mask, attention_mask=None, head_mask=N scaled_attention_logits = matmul_qk / np.sqrt(dk) if mask is not None: - scaled_attention_logits += (mask * -1e4) + nd, ns = scaled_attention_logits.size(-2), scaled_attention_logits.size(-1) + scaled_attention_logits += (mask[ns-nd:ns, :ns] * -1e4) if attention_mask is not None: # Apply the attention mask @@ -251,7 +252,7 @@ class CTRLModel(CTRLPreTrainedModel): **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. @@ -267,7 +268,7 @@ class CTRLModel(CTRLPreTrainedModel): tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = CTRLModel.from_pretrained('ctrl') - input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -373,7 +374,7 @@ class CTRLModel(CTRLPreTrainedModel): inputs_embeds = self.w(input_ids) # inputs_embeds = embedded.unsqueeze(0) if len(input_ids.shape)<2 else embedded seq_len = input_shape[-1] - mask = torch.triu(torch.ones(seq_len, seq_len), 1).to(inputs_embeds.device) + mask = torch.triu(torch.ones(seq_len + past_length, seq_len + past_length), 1).to(inputs_embeds.device) inputs_embeds *= np.sqrt(self.d_model_size) @@ -437,7 +438,7 @@ class CTRLLMHeadModel(CTRLPreTrainedModel): **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. @@ -457,7 +458,7 @@ class CTRLLMHeadModel(CTRLPreTrainedModel): tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = CTRLLMHeadModel.from_pretrained('ctrl') - input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=input_ids) loss, logits = outputs[:2] diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index d30f493c..eb75efa9 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -42,7 +42,9 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-pytorch_model.bin", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin", + 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-pytorch_model.bin", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-pytorch_model.bin", } @@ -413,7 +415,7 @@ class DistilBertModel(DistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = DistilBertModel.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -509,7 +511,7 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = DistilBertForMaskedLM.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) loss, prediction_scores = outputs[:2] @@ -579,7 +581,7 @@ class DistilBertForSequenceClassification(DistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -654,7 +656,7 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = DistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) diff --git a/transformers/modeling_encoder_decoder.py b/transformers/modeling_encoder_decoder.py index a884abd0..c327bb91 100644 --- a/transformers/modeling_encoder_decoder.py +++ b/transformers/modeling_encoder_decoder.py @@ -59,12 +59,14 @@ class PreTrainedEncoderDecoder(nn.Module): encoder_pretrained_model_name_or_path: information necessary to initiate the encoder. Either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/encoder``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. decoder_pretrained_model_name_or_path: information necessary to initiate the decoder. Either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/decoder``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. @@ -164,7 +166,39 @@ class PreTrainedEncoderDecoder(nn.Module): We save the encoder' and decoder's parameters in two separate directories. """ + + # If the root output directory does not exist, create it + if not os.path.exists(save_directory): + os.mkdir(save_directory) + + # Check whether the output directory is empty or not + sub_directories = [directory for directory in os.listdir(save_directory) + if os.path.isdir(os.path.join(save_directory, directory))] + + if len(sub_directories) > 0: + if "encoder" in sub_directories and "decoder" in sub_directories: + print("WARNING: there is an older version of encoder-decoder saved in" +\ + " the output directory. The default behaviour is to overwrite them.") + + # Empty the output directory + for directory_to_remove in sub_directories: + # Remove all files into the subdirectory + files_to_remove = os.listdir(os.path.join(save_directory, directory_to_remove)) + for file_to_remove in files_to_remove: + os.remove(os.path.join(save_directory, directory_to_remove, file_to_remove)) + # Remove the subdirectory itself + os.rmdir(os.path.join(save_directory, directory_to_remove)) + + assert(len(os.listdir(save_directory)) == 0) # sanity check + + # Create the "encoder" directory inside the output directory and save the encoder into it + if not os.path.exists(os.path.join(save_directory, "encoder")): + os.mkdir(os.path.join(save_directory, "encoder")) self.encoder.save_pretrained(os.path.join(save_directory, "encoder")) + + # Create the "encoder" directory inside the output directory and save the decoder into it + if not os.path.exists(os.path.join(save_directory, "decoder")): + os.mkdir(os.path.join(save_directory, "decoder")) self.decoder.save_pretrained(os.path.join(save_directory, "decoder")) def forward(self, encoder_input_ids, decoder_input_ids, **kwargs): @@ -217,9 +251,7 @@ class PreTrainedEncoderDecoder(nn.Module): encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) if encoder_hidden_states is None: encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder) - encoder_hidden_states = encoder_outputs[ - 0 - ] # output the last layer hidden state + encoder_hidden_states = encoder_outputs[0] else: encoder_outputs = () diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index 35bc5c8d..e0dc6017 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -329,7 +329,7 @@ class GPT2Model(GPT2PreTrainedModel): **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. @@ -345,7 +345,7 @@ class GPT2Model(GPT2PreTrainedModel): tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2Model.from_pretrained('gpt2') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -503,7 +503,7 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. @@ -523,7 +523,7 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2LMHeadModel.from_pretrained('gpt2') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=input_ids) loss, logits = outputs[:2] @@ -596,7 +596,7 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): **mc_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` Prediction scores of the multiplechoice classification head (scores for each choice before SoftMax). **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. @@ -634,6 +634,7 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): """ def __init__(self, config): super(GPT2DoubleHeadsModel, self).__init__(config) + config.num_labels = 1 self.transformer = GPT2Model(config) self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) self.multiple_choice_head = SequenceSummary(config) diff --git a/transformers/modeling_openai.py b/transformers/modeling_openai.py index e88f55c3..4b726bca 100644 --- a/transformers/modeling_openai.py +++ b/transformers/modeling_openai.py @@ -50,8 +50,10 @@ def load_tf_weights_in_openai_gpt(model, config, openai_checkpoint_folder_path): logger.info("Loading weights from {}".format(openai_checkpoint_folder_path)) - names = json.load(open(openai_checkpoint_folder_path + '/parameters_names.json', "r", encoding='utf-8')) - shapes = json.load(open(openai_checkpoint_folder_path + '/params_shapes.json', "r", encoding='utf-8')) + with open(openai_checkpoint_folder_path + '/parameters_names.json', "r", encoding='utf-8') as names_handle: + names = json.load(names_handle) + with open(openai_checkpoint_folder_path + '/params_shapes.json', "r", encoding='utf-8') as shapes_handle: + shapes = json.load(shapes_handle) offsets = np.cumsum([np.prod(shape) for shape in shapes]) init_params = [np.load(openai_checkpoint_folder_path + '/params_{}.npy'.format(n)) for n in range(10)] init_params = np.split(np.concatenate(init_params, 0), offsets)[:-1] @@ -347,7 +349,7 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = OpenAIGPTModel.from_pretrained('openai-gpt') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -489,7 +491,7 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel): tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = OpenAIGPTLMHeadModel.from_pretrained('openai-gpt') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=input_ids) loss, logits = outputs[:2] @@ -588,6 +590,7 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): def __init__(self, config): super(OpenAIGPTDoubleHeadsModel, self).__init__(config) + config.num_labels = 1 self.transformer = OpenAIGPTModel(config) self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) self.multiple_choice_head = SequenceSummary(config) diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py index fc27353d..5ee646e6 100644 --- a/transformers/modeling_roberta.py +++ b/transformers/modeling_roberta.py @@ -51,24 +51,44 @@ class RobertaEmbeddings(BertEmbeddings): padding_idx=self.padding_idx) def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None): - if input_ids is not None: - input_shape = input_ids.size() - else: - input_shape = inputs_embeds.size()[:-1] - - seq_length = input_shape[1] - device = input_ids.device if input_ids is not None else inputs_embeds.device - if position_ids is None: - # Position numbers begin at padding_idx+1. Padding symbols are ignored. - # cf. fairseq's `utils.make_positions` - position_ids = torch.arange(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=torch.long, device=device) - position_ids = position_ids.unsqueeze(0).expand(input_shape) + if input_ids is not None: + # Create the position ids from the input token ids. Any padded tokens remain padded. + position_ids = self.create_position_ids_from_input_ids(input_ids).to(input_ids.device) + else: + position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds) + return super(RobertaEmbeddings, self).forward(input_ids, token_type_ids=token_type_ids, position_ids=position_ids, inputs_embeds=inputs_embeds) + def create_position_ids_from_input_ids(self, x): + """ Replace non-padding symbols with their position numbers. Position numbers begin at + padding_idx+1. Padding symbols are ignored. This is modified from fairseq's + `utils.make_positions`. + + :param torch.Tensor x: + :return torch.Tensor: + """ + mask = x.ne(self.padding_idx).long() + incremental_indicies = torch.cumsum(mask, dim=1) * mask + return incremental_indicies + self.padding_idx + + def create_position_ids_from_inputs_embeds(self, inputs_embeds): + """ We are provided embeddings directly. We cannot infer which are padded so just generate + sequential position ids. + + :param torch.Tensor inputs_embeds: + :return torch.Tensor: + """ + input_shape = inputs_embeds.size()[:-1] + sequence_length = input_shape[1] + + position_ids = torch.arange(self.padding_idx+1, sequence_length+self.padding_idx+1, dtype=torch.long, + device=inputs_embeds.device) + return position_ids.unsqueeze(0).expand(input_shape) + ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in `RoBERTa: A Robustly Optimized BERT Pretraining Approach`_ @@ -168,7 +188,7 @@ class RobertaModel(BertModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = RobertaModel.from_pretrained('roberta-base') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -216,7 +236,7 @@ class RobertaForMaskedLM(BertPreTrainedModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = RobertaForMaskedLM.from_pretrained('roberta-base') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) loss, prediction_scores = outputs[:2] @@ -307,7 +327,7 @@ class RobertaForSequenceClassification(BertPreTrainedModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = RobertaForSequenceClassification.from_pretrained('roberta-base') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py new file mode 100644 index 00000000..9baf69d0 --- /dev/null +++ b/transformers/modeling_t5.py @@ -0,0 +1,886 @@ +# coding=utf-8 +# Copyright 2018 Mesh TensorFlow authors, T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch T5 model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +import copy +import itertools +from io import open + +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn import CrossEntropyLoss, MSELoss + +from .modeling_utils import PreTrainedModel, prune_linear_layer +from .configuration_t5 import T5Config +from .file_utils import add_start_docstrings, DUMMY_INPUTS, DUMMY_MASK + +logger = logging.getLogger(__name__) + +#################################################### +# This dict contrains shortcut names and associated url +# for the pretrained weights provided with the models +#################################################### +T5_PRETRAINED_MODEL_ARCHIVE_MAP = { + 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-pytorch_model.bin", + 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-pytorch_model.bin", + 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-pytorch_model.bin", + 't5-3b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3b-pytorch_model.bin", + 't5-11b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11b-pytorch_model.bin", +} + +#################################################### +# This is a conversion method from TF 1.0 to PyTorch +# More details: https://medium.com/huggingface/from-tensorflow-to-pytorch-265f40ef2a28 +#################################################### +def load_tf_weights_in_t5(model, config, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model. + """ + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + tf_weights = {} + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + tf_weights[name] = array + + for txt_name in names: + name = txt_name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any(n in ["adam_v", "adam_m", "global_step"] for n in name): + logger.info("Skipping {}".format("/".join(name))) + tf_weights.pop(txt_name, None) + continue + if '_slot_' in name[-1]: + logger.info("Skipping {}".format("/".join(name))) + tf_weights.pop(txt_name, None) + continue + pointer = model + array = tf_weights[txt_name] + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + else: + l = [m_name] + if l[0] in ['kernel', 'scale', 'embedding']: + pointer = getattr(pointer, 'weight') + # elif l[0] == 'scale': + # pointer = getattr(pointer, 'weight') + # elif l[0] == 'output_bias' or l[0] == 'beta': + # pointer = getattr(pointer, 'bias') + # elif l[0] == 'squad': + # pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, l[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + if l[0] not in ['kernel', 'scale', 'embedding']: + pointer = getattr(pointer, 'weight') + if l[0] != 'embedding': + logger.info("Transposing numpy weight of shape {} for {}".format(array.shape, name)) + array = np.transpose(array) + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info("Initialize PyTorch weight {}".format(name)) + pointer.data = torch.from_numpy(array.astype(np.float32)) + tf_weights.pop(txt_name, None) + + logger.info("Weights not copied to PyTorch model: {}".format(', '.join(tf_weights.keys()))) + # logger.info("Weights not copied to PyTorch model: {}".format(', '.join(tf_weights.keys()))) + return model + + +#################################################### +# PyTorch Models are constructed by sub-classing +# - torch.nn.Module for the layers and +# - PreTrainedModel for the models (it-self a sub-class of torch.nn.Module) +#################################################### + +class T5LayerNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ Construct a layernorm module in the T5 style + No bias and no substraction of mean. + """ + super(T5LayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + variance = x.pow(2).mean(-1, keepdim=True) + x = x / torch.sqrt(variance + self.variance_epsilon) + return self.weight * x + + +class T5DenseReluDense(nn.Module): + def __init__(self, config): + super(T5DenseReluDense, self).__init__() + self.wi = nn.Linear(config.d_model, config.d_ff, bias=False) + self.wo = nn.Linear(config.d_ff, config.d_model, bias=False) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward(self, hidden_states): + h = self.wi(hidden_states) + h = F.relu(h) + h = self.dropout(h) + h = self.wo(h) + return h + + +class T5LayerFF(nn.Module): + def __init__(self, config): + super(T5LayerFF, self).__init__() + self.DenseReluDense = T5DenseReluDense(config) + self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward(self, hidden_states): + norm_x = self.layer_norm(hidden_states) + y = self.DenseReluDense(norm_x) + layer_output = hidden_states + self.dropout(y) + return layer_output + + +class T5Attention(nn.Module): + NEW_ID = itertools.count() + + def __init__(self, config, has_relative_attention_bias=False): + super(T5Attention, self).__init__() + self.layer_id = next(T5Attention.NEW_ID) + self.is_decoder = config.is_decoder + self.has_relative_attention_bias = has_relative_attention_bias + + self.output_attentions = config.output_attentions + self.relative_attention_num_buckets = config.relative_attention_num_buckets + self.d_model = config.d_model + self.d_kv = config.d_kv + self.n_heads = config.num_heads + self.dropout = config.dropout_rate + self.inner_dim = self.n_heads * self.d_kv + + # Mesh TensorFlow initialization to avoid scaling before softmax + self.q = nn.Linear(self.d_model, self.inner_dim, bias=False) + self.k = nn.Linear(self.d_model, self.inner_dim, bias=False) + self.v = nn.Linear(self.d_model, self.inner_dim, bias=False) + self.o = nn.Linear(self.inner_dim, self.d_model, bias=False) + + if self.has_relative_attention_bias: + self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, self.n_heads) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.n_heads, self.d_kv) + heads = set(heads) - self.pruned_heads + for head in heads: + head -= sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + # Prune linear layers + self.q = prune_linear_layer(self.q, index) + self.k = prune_linear_layer(self.k, index) + self.v = prune_linear_layer(self.v, index) + self.o = prune_linear_layer(self.o, index, dim=1) + # Update hyper params + self.n_heads = self.n_heads - len(heads) + self.inner_dim = self.d_kv * self.n_heads + self.pruned_heads = self.pruned_heads.union(heads) + + @staticmethod + def _relative_position_bucket(relative_position, + bidirectional=True, + num_buckets=32, + max_distance=128): + """ + Adapted from Mesh Tensorflow: + https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 + + Translate relative position to a bucket number for relative attention. + The relative position is defined as memory_position - query_position, i.e. + the distance in tokens from the attending position to the attended-to + position. If bidirectional=False, then positive relative positions are + invalid. + We use smaller buckets for small absolute relative_position and larger buckets + for larger absolute relative_positions. All relative positions >=max_distance + map to the same bucket. All relative positions <=-max_distance map to the + same bucket. This should allow for more graceful generalization to longer + sequences than the model has been trained on. + Args: + relative_position: an int32 Tensor + bidirectional: a boolean - whether the attention is bidirectional + num_buckets: an integer + max_distance: an integer + Returns: + a Tensor with the same shape as relative_position, containing int32 + values in the range [0, num_buckets) + """ + ret = 0 + n = -relative_position + if bidirectional: + num_buckets //= 2 + ret += (n < 0).to(torch.long) * num_buckets # mtf.to_int32(mtf.less(n, 0)) * num_buckets + n = torch.abs(n) + else: + n = torch.max(n, torch.zeros_like(n)) + # now n is in the range [0, inf) + + # half of the buckets are for exact increments in positions + max_exact = num_buckets // 2 + is_small = (n < max_exact) + + # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance + val_if_large = max_exact + ( + torch.log(n.float() / max_exact) + / math.log(max_distance / max_exact) * (num_buckets - max_exact)).to(torch.long) + val_if_large = torch.min(val_if_large, torch.full_like(val_if_large, num_buckets - 1)) + + ret += torch.where(is_small, n, val_if_large) + return ret + + def compute_bias(self, qlen, klen): + """ Compute binned relative position bias """ + context_position = torch.arange(qlen, dtype=torch.long)[:, None] + memory_position = torch.arange(klen, dtype=torch.long)[None, :] + relative_position = memory_position - context_position # shape (qlen, klen) + rp_bucket = self._relative_position_bucket(relative_position, # shape (qlen, klen) + bidirectional=not self.is_decoder, + num_buckets=self.relative_attention_num_buckets) + values = self.relative_attention_bias(rp_bucket) # shape (qlen, klen, num_heads) + values = values.permute([2, 0, 1]).unsqueeze(0) # shape (1, num_heads, qlen, klen) + return values + + def forward(self, input, mask=None, kv=None, position_bias=None, cache=None, head_mask=None): + """ + Self-attention (if kv is None) or attention over source sentence (provided by kv). + """ + # Input is (bs, qlen, dim) + # Mask is (bs, klen) (non-causal) or (bs, klen, klen) + bs, qlen, dim = input.size() + if kv is None: + klen = qlen if cache is None else cache['slen'] + qlen + else: + klen = kv.size(1) + + def shape(x): + """ projection """ + return x.view(bs, -1, self.n_heads, self.d_kv).transpose(1, 2) + + def unshape(x): + """ compute context """ + return x.transpose(1, 2).contiguous().view(bs, -1, self.inner_dim) + + q = shape(self.q(input)) # (bs, n_heads, qlen, dim_per_head) + if kv is None: + k = shape(self.k(input)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v(input)) # (bs, n_heads, qlen, dim_per_head) + elif cache is None or self.layer_id not in cache: + k = v = kv + k = shape(self.k(k)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v(v)) # (bs, n_heads, qlen, dim_per_head) + + if cache is not None: + if self.layer_id in cache: + if kv is None: + k_, v_ = cache[self.layer_id] + k = torch.cat([k_, k], dim=2) # (bs, n_heads, klen, dim_per_head) + v = torch.cat([v_, v], dim=2) # (bs, n_heads, klen, dim_per_head) + else: + k, v = cache[self.layer_id] + cache[self.layer_id] = (k, v) + + # q = q / math.sqrt(dim_per_head) # No scaling in T5 + scores = torch.einsum('bnqd,bnkd->bnqk', q, k) # (bs, n_heads, qlen, klen) + + if position_bias is None: + if not self.has_relative_attention_bias: + raise ValueError("No position_bias provided and no weights to compute position_bias") + position_bias = self.compute_bias(qlen, klen) + if mask is not None: + position_bias = position_bias + mask # (bs, n_heads, qlen, klen) + + scores += position_bias + weights = F.softmax(scores.float(), dim=-1).type_as(scores) # (bs, n_heads, qlen, klen) + weights = F.dropout(weights, p=self.dropout, training=self.training) # (bs, n_heads, qlen, klen) + + # Mask heads if we want to + if head_mask is not None: + weights = weights * head_mask + + context = torch.matmul(weights, v) # (bs, n_heads, qlen, dim_per_head) + context = unshape(context) # (bs, qlen, dim) + + context = self.o(context) + + outputs = (context,) + if self.output_attentions: + outputs = outputs + (weights,) + if self.has_relative_attention_bias: + outputs = outputs + (position_bias,) + return outputs + + +class T5LayerSelfAttention(nn.Module): + def __init__(self, config, has_relative_attention_bias=False): + super(T5LayerSelfAttention, self).__init__() + self.SelfAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias) + self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward(self, hidden_states, attention_mask=None, position_bias=None, head_mask=None): + norm_x = self.layer_norm(hidden_states) + attention_output = self.SelfAttention(norm_x, + mask=attention_mask, + position_bias=position_bias, + head_mask=head_mask) + y = attention_output[0] + layer_output = hidden_states + self.dropout(y) + outputs = (layer_output,) + attention_output[1:] # add attentions if we output them + return outputs + + +class T5LayerCrossAttention(nn.Module): + def __init__(self, config, has_relative_attention_bias=False): + super(T5LayerCrossAttention, self).__init__() + self.EncDecAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias) + self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward(self, hidden_states, kv, attention_mask=None, position_bias=None, head_mask=None): + norm_x = self.layer_norm(hidden_states) + attention_output = self.EncDecAttention(norm_x, + mask=attention_mask, + kv=kv, + position_bias=position_bias, + head_mask=head_mask) + y = attention_output[0] + layer_output = hidden_states + self.dropout(y) + outputs = (layer_output,) + attention_output[1:] # add attentions if we output them + return outputs + + +class T5Block(nn.Module): + def __init__(self, config, has_relative_attention_bias=False): + super(T5Block, self).__init__() + self.is_decoder = config.is_decoder + self.layer = nn.ModuleList() + self.layer.append(T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias)) + if self.is_decoder: + self.layer.append(T5LayerCrossAttention(config, has_relative_attention_bias=has_relative_attention_bias)) + self.layer.append(T5LayerFF(config)) + else: + self.layer.append(T5LayerFF(config)) + + def forward(self, hidden_states, attention_mask=None, position_bias=None, + encoder_hidden_states=None, encoder_attention_mask=None, encoder_decoder_position_bias=None, + head_mask=None): + self_attention_outputs = self.layer[0](hidden_states, + attention_mask=attention_mask, + position_bias=position_bias, + head_mask=head_mask) + hidden_states = self_attention_outputs[0] + outputs = self_attention_outputs[1:] # Keep self-attention outputs and relative position weights + + if not self.is_decoder: + hidden_states = self.layer[1](hidden_states) + else: + cross_attention_outputs = self.layer[1](hidden_states, + kv=encoder_hidden_states, + attention_mask=encoder_attention_mask, + position_bias=encoder_decoder_position_bias, + head_mask=head_mask) + hidden_states = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[1:] # Keep cross-attention outputs and relative position weights + hidden_states = self.layer[2](hidden_states) + + outputs = (hidden_states,) + outputs # add attentions if we output them + return outputs # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) + + +class T5PreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = T5Config + pretrained_model_archive_map = T5_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_t5 + base_model_prefix = "transformer" + + @property + def dummy_inputs(self): + input_ids = torch.tensor(DUMMY_INPUTS) + input_mask = torch.tensor(DUMMY_MASK) + dummy_inputs = {'decoder_input_ids': input_ids, + 'encoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + return dummy_inputs + + def _init_weights(self, module): + """ Initialize the weights """ + factor = self.config.initializer_factor # Used for testing weights initialization + if isinstance(module, T5LayerNorm): + module.weight.data.fill_(factor*1.0) + elif isinstance(module, (T5Model, T5WithLMHeadModel)): + # Mesh TensorFlow embeddings initialization + # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L1624 + module.shared.weight.data.normal_(mean=0.0, std=factor*1.0) + elif isinstance(module, T5DenseReluDense): + # Mesh TensorFlow FF initialization + # See https://github.com/tensorflow/mesh/blob/master/mesh_tensorflow/transformer/transformer_layers.py#L56 + # and https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L89 + module.wi.weight.data.normal_(mean=0.0, std=factor*((self.config.d_model) ** -0.5)) + if hasattr(module.wi, 'bias') and module.wi.bias is not None: + module.wi.bias.data.zero_() + module.wo.weight.data.normal_(mean=0.0, std=factor*((self.config.d_ff) ** -0.5)) + if hasattr(module.wo, 'bias') and module.wo.bias is not None: + module.wo.bias.data.zero_() + elif isinstance(module, T5Attention): + # Mesh TensorFlow attention initialization to avoid scaling before softmax + # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/attention.py#L136 + d_model = self.config.d_model + d_kv = self.config.d_kv + n_heads = self.config.num_heads + module.q.weight.data.normal_(mean=0.0, std=factor*((d_model * d_kv) ** -0.5)) + module.k.weight.data.normal_(mean=0.0, std=factor*(d_model ** -0.5)) + module.v.weight.data.normal_(mean=0.0, std=factor*(d_model ** -0.5)) + module.o.weight.data.normal_(mean=0.0, std=factor*((n_heads * d_kv) ** -0.5)) + if module.has_relative_attention_bias: + module.relative_attention_bias.weight.data.normal_(mean=0.0, std=factor*((d_model) ** -0.5)) + + +class T5Stack(T5PreTrainedModel): + def __init__(self, config): + super(T5Stack, self).__init__(config) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.is_decoder = config.is_decoder + + self.block = nn.ModuleList([T5Block(config, has_relative_attention_bias=bool(i == 0)) + for i in range(config.num_layers)]) + self.final_layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + self.init_weights() + + def forward(self, + hidden_states, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + head_mask=None): + + batch_size, seq_length = hidden_states.shape[0], hidden_states.shape[1] + if attention_mask is None: + attention_mask = torch.ones(batch_size, seq_length).to(hidden_states.device) + if self.is_decoder and encoder_attention_mask is None: + encoder_seq_length = encoder_hidden_states.shape[1] + encoder_attention_mask = torch.ones(batch_size, encoder_seq_length).to(hidden_states.device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder: + seq_ids = torch.arange(seq_length, device=hidden_states.device) + causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] + causal_mask = causal_mask.to(attention_mask) + extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] + else: + extended_attention_mask = attention_mask[:, None, None, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -1e9 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition + # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 + # extended_attention_mask = (extended_attention_mask == extended_attention_mask.transpose(-1, -2)) + + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -1e9 + + if self.is_decoder: + # If a 2D ou 3D attention mask is provided for the cross-attention + # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] + if encoder_attention_mask.dim() == 3: + encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] + if encoder_attention_mask.dim() == 2: + encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + + # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition + # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 + # encoder_extended_attention_mask = (encoder_extended_attention_mask == encoder_extended_attention_mask.transpose(-1, -2)) + + encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -1e9 + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_layers + + all_hidden_states = () + all_attentions = () + position_bias = None + encoder_decoder_position_bias = None + + hidden_states = self.dropout(hidden_states) + for i, layer_module in enumerate(self.block): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module(hidden_states, + attention_mask=extended_attention_mask, + position_bias=position_bias, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + encoder_decoder_position_bias=encoder_decoder_position_bias, + head_mask=head_mask[i]) + # layer_outputs is a tuple with: + # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) + hidden_states = layer_outputs[0] + if i == 0: + # We share the position biases between the layers - the first layer store them + # layer_outputs = hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) + position_bias = layer_outputs[2 if self.output_attentions else 1] + if self.is_decoder: + encoder_decoder_position_bias = layer_outputs[4 if self.output_attentions else 2] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) # We keep only self-attention weights for now + + hidden_states = self.final_layer_norm(hidden_states) + layer_output = self.dropout(hidden_states) + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + + +T5_START_DOCSTRING = r""" The T5 model was proposed in + `Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`_ + by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. + It's an encoder decoder transformer pre-trained in a text-to-text denoising generative setting. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`: + https://arxiv.org/abs/1910.10683 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.T5Config`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +T5_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, T5 input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + T5 is a model with relative position embeddings so you should be able to pad the inputs on + the right or the left. + + Indices can be obtained using :class:`transformers.T5Tokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare T5 Model transformer outputting raw hidden-states" + "without any specific head on top.", + T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) +class T5Model(T5PreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = T5Tokenizer.from_pretrained('t5-small') + model = T5Model.from_pretrained('t5-small') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids=input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config): + super(T5Model, self).__init__(config) + self.shared = nn.Embedding(config.vocab_size, config.d_model) + + encoder_config = copy.deepcopy(config) + self.encoder = T5Stack(encoder_config) + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + self.decoder = T5Stack(decoder_config) + + self.init_weights() + + def get_input_embeddings(self): + return self.shared + + def set_input_embeddings(self, new_embeddings): + self.shared = new_embeddings + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward(self, **kwargs): + # keyword arguments come in 3 flavors: encoder-specific (prefixed by + # `encoder_`), decoder-specific (prefixed by `decoder_`) and those + # that apply to the model as whole. + # We let the specific kwargs override the common ones in case of conflict. + kwargs_common = dict((k, v) for k, v in kwargs.items() + if not k.startswith("encoder_") and not k.startswith("decoder_")) + kwargs_encoder = kwargs_common.copy() + kwargs_decoder = kwargs_common.copy() + kwargs_encoder.update(dict((k[len("encoder_"):], v) for k, v in kwargs.items() if k.startswith("encoder_"))) + kwargs_decoder.update(dict((k[len("decoder_"):], v) for k, v in kwargs.items() if k.startswith("decoder_"))) + + # Encode if needed (training, first prediction pass) + encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) + encoder_attention_mask = kwargs_encoder.get("attention_mask", None) + if encoder_hidden_states is None: + # Convert encoder inputs in embeddings if needed + hidden_states = kwargs_encoder.pop("inputs_embeds", None) + if hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + + if encoder_attention_mask is not None: + # Apply masking + encoder_attention_mask = (encoder_attention_mask != 0).to(hidden_states) + hidden_states = hidden_states * encoder_attention_mask.unsqueeze(-1) + + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) + encoder_hidden_states = encoder_outputs[0] + else: + encoder_outputs = () + + # Decode + # Convert decoder inputs in embeddings if needed + hidden_states = kwargs_decoder.pop("inputs_embeds", None) + if hidden_states is None: + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) + + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states + kwargs_decoder["encoder_attention_mask"] = encoder_attention_mask + decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) + + return decoder_outputs + encoder_outputs + + +@add_start_docstrings("""T5 Model with a `language modeling` head on top. """, + T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) +class T5WithLMHeadModel(T5PreTrainedModel): + r""" + **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = T5Tokenizer.from_pretrained('t5-small') + model = T5WithLMHeadModel.from_pretrained('t5-small') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids=input_ids, lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + def __init__(self, config): + super(T5WithLMHeadModel, self).__init__(config) + self.model_dim = config.d_model + + self.shared = nn.Embedding(config.vocab_size, config.d_model) + + encoder_config = copy.deepcopy(config) + self.encoder = T5Stack(encoder_config) + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + self.decoder = T5Stack(decoder_config) + + self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False) + + self.init_weights() + + def get_input_embeddings(self): + return self.shared + + def set_input_embeddings(self, new_embeddings): + self.shared = new_embeddings + + def get_output_embeddings(self): + return self.lm_head + + def forward(self, **kwargs): + # keyword arguments come in 3 flavors: encoder-specific (prefixed by + # `encoder_`), decoder-specific (prefixed by `decoder_`) and those + # that apply to the model as whole. + # We let the specific kwargs override the common ones in case of conflict. + + lm_labels = kwargs.pop('decoder_lm_labels', None) + + kwargs_common = dict((k, v) for k, v in kwargs.items() + if not k.startswith("encoder_") and not k.startswith("decoder_")) + kwargs_encoder = kwargs_common.copy() + kwargs_decoder = kwargs_common.copy() + kwargs_encoder.update(dict((k[len("encoder_"):], v) for k, v in kwargs.items() if k.startswith("encoder_"))) + kwargs_decoder.update(dict((k[len("decoder_"):], v) for k, v in kwargs.items() if k.startswith("decoder_"))) + + # Encode if needed (training, first prediction pass) + encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) + if encoder_hidden_states is None: + # Convert encoder inputs in embeddings if needed + hidden_states = kwargs_encoder.pop("inputs_embeds", None) + if hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) + encoder_hidden_states = encoder_outputs[0] + else: + encoder_outputs = () + + # Decode + # Convert decoder inputs in embeddings if needed + hidden_states = kwargs_decoder.pop("inputs_embeds", None) + if hidden_states is None: + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) + + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states + kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) + decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) + + sequence_output = decoder_outputs[0] + # Rescale output before projecting on vocab + # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/transformer.py#L586 + sequence_output = sequence_output * (self.model_dim ** -0.5) + lm_logits = self.lm_head(sequence_output) + + decoder_outputs = (lm_logits,) + decoder_outputs[1:] # Add hidden states and attention if they are here + if lm_labels is not None: + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = lm_labels[..., 1:].contiguous() + loss_fct = CrossEntropyLoss(ignore_index=-1) + loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1)) + decoder_outputs = (loss,) + decoder_outputs # TODO(thom): Add z_loss https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666 + + return decoder_outputs + encoder_outputs diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py new file mode 100644 index 00000000..ac55a73f --- /dev/null +++ b/transformers/modeling_tf_albert.py @@ -0,0 +1,794 @@ +# coding=utf-8 +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 ALBERT model. """ +from __future__ import absolute_import, division, print_function, unicode_literals + +import logging +import sys + +import tensorflow as tf + +from .configuration_albert import AlbertConfig +from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list +from .modeling_tf_bert import ACT2FN, TFBertSelfAttention +from .file_utils import add_start_docstrings + +import logging + +logger = logging.getLogger(__name__) + +TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v1-tf_model.h5", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v1-tf_model.h5", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v1-tf_model.h5", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v1-tf_model.h5", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-tf_model.h5", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-tf_model.h5", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-tf_model.h5", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-tf_model.h5", +} + + +class TFAlbertEmbeddings(tf.keras.layers.Layer): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config, **kwargs): + super(TFAlbertEmbeddings, self).__init__(**kwargs) + + self.config = config + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, + config.embedding_size, + embeddings_initializer=get_initializer( + self.config.initializer_range), + name='position_embeddings') + self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, + config.embedding_size, + embeddings_initializer=get_initializer( + self.config.initializer_range), + name='token_type_embeddings') + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def build(self, input_shape): + """Build shared word embedding layer """ + with tf.name_scope("word_embeddings"): + # Create and initialize weights. The random normal initializer was chosen + # arbitrarily, and works well. + self.word_embeddings = self.add_weight( + "weight", + shape=[self.config.vocab_size, self.config.embedding_size], + initializer=get_initializer(self.config.initializer_range)) + super(TFAlbertEmbeddings, self).build(input_shape) + + def call(self, inputs, mode="embedding", training=False): + """Get token embeddings of inputs. + Args: + inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + if mode == "embedding": + return self._embedding(inputs, training=training) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, inputs, training=False): + """Applies embedding based on inputs tensor.""" + input_ids, position_ids, token_type_ids, inputs_embeds = inputs + + if input_ids is not None: + input_shape = shape_list(input_ids) + else: + input_shape = shape_list(inputs_embeds)[:-1] + + seq_length = input_shape[1] + if position_ids is None: + position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] + if token_type_ids is None: + token_type_ids = tf.fill(input_shape, 0) + + if inputs_embeds is None: + inputs_embeds = tf.gather(self.word_embeddings, input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = inputs_embeds + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings, training=training) + return embeddings + + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + Args: + inputs: A float32 tensor with shape [batch_size, length, embedding_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + batch_size = shape_list(inputs)[0] + length = shape_list(inputs)[1] + x = tf.reshape(inputs, [-1, self.config.embedding_size]) + logits = tf.matmul(x, self.word_embeddings, transpose_b=True) + return tf.reshape(logits, [batch_size, length, self.config.vocab_size]) + + +class TFAlbertSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertSelfAttention, self).__init__(**kwargs) + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + assert config.hidden_size % config.num_attention_heads == 0 + self.attention_head_size = int( + config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='query') + self.key = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='key') + self.value = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='value') + + self.dropout = tf.keras.layers.Dropout( + config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x, batch_size): + x = tf.reshape( + x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + batch_size = shape_list(hidden_states)[0] + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + # (batch size, num_heads, seq_len_q, seq_len_k) + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + # scale attention_scores + dk = tf.cast(shape_list(key_layer)[-1], tf.float32) + attention_scores = attention_scores / tf.math.sqrt(dk) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TFAlbertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs, training=training) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + outputs = (context_layer, attention_probs) if self.output_attentions else ( + context_layer,) + return outputs + + +class TFAlbertSelfOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertSelfOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFAlbertAttention(TFBertSelfAttention): + def __init__(self, config, **kwargs): + super(TFAlbertAttention, self).__init__(config, **kwargs) + + self.hidden_size = config.hidden_size + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.pruned_heads = set() + + def prune_heads(self, heads): + raise NotImplementedError + + def call(self, inputs, training=False): + input_tensor, attention_mask, head_mask = inputs + + batch_size = shape_list(input_tensor)[0] + mixed_query_layer = self.query(input_tensor) + mixed_key_layer = self.key(input_tensor) + mixed_value_layer = self.value(input_tensor) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + # (batch size, num_heads, seq_len_q, seq_len_k) + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + # scale attention_scores + dk = tf.cast(shape_list(key_layer)[-1], tf.float32) + attention_scores = attention_scores / tf.math.sqrt(dk) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TFBertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs, training=training) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + self_outputs = (context_layer, attention_probs) if self.output_attentions else ( + context_layer,) + + hidden_states = self_outputs[0] + + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + attention_output = self.LayerNorm(hidden_states + input_tensor) + + # add attentions if we output them + outputs = (attention_output,) + self_outputs[1:] + return outputs + + +class TFAlbertLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertLayer, self).__init__(**kwargs) + self.attention = TFAlbertAttention(config, name='attention') + + self.ffn = tf.keras.layers.Dense(config.intermediate_size, kernel_initializer=get_initializer( + config.initializer_range), name='ffn') + + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.activation = ACT2FN[config.hidden_act] + else: + self.activation = config.hidden_act + + self.ffn_output = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), name='ffn_output') + self.full_layer_layer_norm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='full_layer_layer_norm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + attention_outputs = self.attention( + [hidden_states, attention_mask, head_mask], training=training) + ffn_output = self.ffn(attention_outputs[0]) + ffn_output = self.activation(ffn_output) + ffn_output = self.ffn_output(ffn_output) + + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = self.full_layer_layer_norm( + ffn_output + attention_outputs[0]) + + # add attentions if we output them + outputs = (hidden_states,) + attention_outputs[1:] + return outputs + + +class TFAlbertLayerGroup(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertLayerGroup, self).__init__(**kwargs) + + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.albert_layers = [TFAlbertLayer(config, name="albert_layers_._{}".format( + i)) for i in range(config.inner_group_num)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + layer_hidden_states = () + layer_attentions = () + + for layer_index, albert_layer in enumerate(self.albert_layers): + layer_output = albert_layer( + [hidden_states, attention_mask, head_mask[layer_index]], training=training) + hidden_states = layer_output[0] + + if self.output_attentions: + layer_attentions = layer_attentions + (layer_output[1],) + + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (layer_hidden_states,) + if self.output_attentions: + outputs = outputs + (layer_attentions,) + # last-layer hidden state, (layer hidden states), (layer attentions) + return outputs + + +class TFAlbertTransformer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertTransformer, self).__init__(**kwargs) + + self.config = config + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), name='embedding_hidden_mapping_in') + self.albert_layer_groups = [TFAlbertLayerGroup( + config, name="albert_layer_groups_._{}".format(i)) for i in range(config.num_hidden_groups)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + all_attentions = () + + if self.output_hidden_states: + all_hidden_states = (hidden_states,) + + for i in range(self.config.num_hidden_layers): + # Number of layers in a hidden group + layers_per_group = int( + self.config.num_hidden_layers / self.config.num_hidden_groups) + + # Index of the hidden group + group_idx = int( + i / (self.config.num_hidden_layers / self.config.num_hidden_groups)) + + layer_group_output = self.albert_layer_groups[group_idx]( + [hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]], training=training) + hidden_states = layer_group_output[0] + + if self.output_attentions: + all_attentions = all_attentions + layer_group_output[-1] + + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + + # last-layer hidden state, (all hidden states), (all attentions) + return outputs + + +class TFAlbertPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "albert" + + +class TFAlbertMLMHead(tf.keras.layers.Layer): + def __init__(self, config, input_embeddings, **kwargs): + super(TFAlbertMLMHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + + self.dense = tf.keras.layers.Dense(config.embedding_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.activation = ACT2FN[config.hidden_act] + else: + self.activation = config.hidden_act + + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = input_embeddings + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + self.decoder_bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='decoder/bias') + super(TFAlbertMLMHead, self).build(input_shape) + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + hidden_states = self.decoder(hidden_states, mode="linear") + self.decoder_bias + hidden_states = hidden_states + self.bias + return hidden_states + + +ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in + `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ + by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents + two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`: + https://arxiv.org/abs/1909.11942 + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ALBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, ALBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.AlbertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Albert Model transformer outputing raw hidden-states without any specific head on top.", + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertModel(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Albert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertModel + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v1') + model = TFAlbertModel.from_pretrained('albert-base-v1') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + + def __init__(self, config, **kwargs): + super(TFAlbertModel, self).__init__(config, **kwargs) + self.num_hidden_layers = config.num_hidden_layers + + self.embeddings = TFAlbertEmbeddings(config, name="embeddings") + self.encoder = TFAlbertTransformer(config, name="encoder") + self.pooler = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), activation='tanh', name='pooler') + + def get_input_embeddings(self): + return self.embeddings + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + raise NotImplementedError + + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False): + if isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids + position_ids = inputs[3] if len(inputs) > 3 else position_ids + head_mask = inputs[4] if len(inputs) > 4 else head_mask + inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds + assert len(inputs) <= 6, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 6, "Too many inputs." + else: + input_ids = inputs + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = shape_list(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + if attention_mask is None: + attention_mask = tf.fill(input_shape, 1) + if token_type_ids is None: + token_type_ids = tf.fill(input_shape, 0) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + extended_attention_mask = tf.cast(extended_attention_mask, tf.float32) + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + embedding_output = self.embeddings( + [input_ids, position_ids, token_type_ids, inputs_embeds], training=training) + encoder_outputs = self.encoder( + [embedding_output, extended_attention_mask, head_mask], training=training) + + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output[:, 0]) + + # add hidden_states and attentions if they are here + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] + # sequence_output, pooled_output, (hidden_states), (attentions) + return outputs + + +@add_start_docstrings("""Albert Model with a `language modeling` head on top. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertForMaskedLM + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = TFAlbertForMaskedLM.from_pretrained('albert-base-v2') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + prediction_scores = outputs[0] + + """ + + def __init__(self, config, *inputs, **kwargs): + super(TFAlbertForMaskedLM, self).__init__(config, *inputs, **kwargs) + + self.albert = TFAlbertModel(config, name='albert') + self.predictions = TFAlbertMLMHead( + config, self.albert.embeddings, name='predictions') + + def get_output_embeddings(self): + return self.albert.embeddings + + def call(self, inputs, **kwargs): + outputs = self.albert(inputs, **kwargs) + + sequence_output = outputs[0] + prediction_scores = self.predictions( + sequence_output, training=kwargs.get('training', False)) + + # Add hidden states and attention if they are here + outputs = (prediction_scores,) + outputs[2:] + + return outputs # prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertForSequenceClassification(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **logits**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertForSequenceClassification + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = TFAlbertForSequenceClassification.from_pretrained('albert-base-v2') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFAlbertForSequenceClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.albert = TFAlbertModel(config, name='albert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + outputs = self.albert(inputs, **kwargs) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False)) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # logits, (hidden_states), (attentions) \ No newline at end of file diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index df0ad6e4..031ffea1 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -18,21 +18,48 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging -from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, TFBertForQuestionAnswering -from .modeling_tf_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel -from .modeling_tf_gpt2 import TFGPT2Model, TFGPT2LMHeadModel -from .modeling_tf_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel -from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnsweringSimple -from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple -from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification -from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification -from .modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel +from .configuration_auto import (BertConfig, CTRLConfig, DistilBertConfig, + GPT2Config, OpenAIGPTConfig, RobertaConfig, + TransfoXLConfig, XLMConfig, XLNetConfig) + +from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, \ + TFBertForQuestionAnswering, TFBertForTokenClassification, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_gpt2 import TFGPT2Model, TFGPT2LMHeadModel, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel, TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, \ + TFXLNetForQuestionAnsweringSimple, TFXLNetForTokenClassification, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, \ + TFXLMForQuestionAnsweringSimple, TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, \ + TFRobertaForTokenClassification, TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification, TFDistilBertForTokenClassification, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_albert import TFAlbertModel, TFAlbertForMaskedLM, TFAlbertForSequenceClassification, TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_t5 import TFT5Model, TFT5WithLMHeadModel, TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) +TF_ALL_PRETRAINED_MODEL_ARCHIVE_MAP = dict((key, value) + for pretrained_map in [ + TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP, + ] + for key, value, in pretrained_map.items()) + + class TFAutoModel(object): r""" :class:`~transformers.TFAutoModel` is a generic model class @@ -45,6 +72,7 @@ class TFAutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: TFT5Model (T5 model) - contains `distilbert`: TFDistilBertModel (DistilBERT model) - contains `roberta`: TFRobertaModel (RoBERTa model) - contains `bert`: TFBertModel (Bert model) @@ -59,7 +87,50 @@ class TFAutoModel(object): """ def __init__(self): raise EnvironmentError("TFAutoModel is designed to be instantiated " - "using the `TFAutoModel.from_pretrained(pretrained_model_name_or_path)` method.") + "using the `TFAutoModel.from_pretrained(pretrained_model_name_or_path)` or " + "`TFAutoModel.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: TFDistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: TFRobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: TFBertModel (Bert model) + - isInstance of `openai-gpt` configuration class: TFOpenAIGPTModel (OpenAI GPT model) + - isInstance of `gpt2` configuration class: TFGPT2Model (OpenAI GPT-2 model) + - isInstance of `ctrl` configuration class: TFCTRLModel (Salesforce CTRL model) + - isInstance of `transfo-xl` configuration class: TFTransfoXLModel (Transformer-XL model) + - isInstance of `xlnet` configuration class: TFXLNetModel (XLNet model) + - isInstance of `xlm` configuration class: TFXLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = TFAutoModel.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return TFDistilBertModel(config) + elif isinstance(config, RobertaConfig): + return TFRobertaModel(config) + elif isinstance(config, BertConfig): + return TFBertModel(config) + elif isinstance(config, OpenAIGPTConfig): + return TFOpenAIGPTModel(config) + elif isinstance(config, GPT2Config): + return TFGPT2Model(config) + elif isinstance(config, TransfoXLConfig): + return TFTransfoXLModel(config) + elif isinstance(config, XLNetConfig): + return TFXLNetModel(config) + elif isinstance(config, XLMConfig): + return TFXLMModel(config) + elif isinstance(config, CTRLConfig): + return TFCTRLModel(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -68,6 +139,7 @@ class TFAutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: TFT5Model (T5 model) - contains `distilbert`: TFDistilBertModel (DistilBERT model) - contains `roberta`: TFRobertaModel (RoBERTa model) - contains `bert`: TFTFBertModel (Bert model) @@ -81,6 +153,7 @@ class TFAutoModel(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. @@ -109,6 +182,9 @@ class TFAutoModel(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -133,8 +209,12 @@ class TFAutoModel(object): model = TFAutoModel.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return TFT5Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return TFDistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return TFAlbertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return TFRobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -153,7 +233,7 @@ class TFAutoModel(object): return TFCTRLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'distilbert', 'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) @@ -169,6 +249,7 @@ class TFAutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: TFT5WithLMHeadModel (T5 model) - contains `distilbert`: TFDistilBertForMaskedLM (DistilBERT model) - contains `roberta`: TFRobertaForMaskedLM (RoBERTa model) - contains `bert`: TFBertForMaskedLM (Bert model) @@ -183,7 +264,50 @@ class TFAutoModelWithLMHead(object): """ def __init__(self): raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " - "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` or " + "`TFAutoModelWithLMHead.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `openai-gpt` configuration class: OpenAIGPTModel (OpenAI GPT model) + - isInstance of `gpt2` configuration class: GPT2Model (OpenAI GPT-2 model) + - isInstance of `ctrl` configuration class: CTRLModel (Salesforce CTRL model) + - isInstance of `transfo-xl` configuration class: TransfoXLModel (Transformer-XL model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelWithLMHead.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return TFDistilBertForMaskedLM(config) + elif isinstance(config, RobertaConfig): + return TFRobertaForMaskedLM(config) + elif isinstance(config, BertConfig): + return TFBertForMaskedLM(config) + elif isinstance(config, OpenAIGPTConfig): + return TFOpenAIGPTLMHeadModel(config) + elif isinstance(config, GPT2Config): + return TFGPT2LMHeadModel(config) + elif isinstance(config, TransfoXLConfig): + return TFTransfoXLLMHeadModel(config) + elif isinstance(config, XLNetConfig): + return TFXLNetLMHeadModel(config) + elif isinstance(config, XLMConfig): + return TFXLMWithLMHeadModel(config) + elif isinstance(config, CTRLConfig): + return TFCTRLLMHeadModel(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -195,6 +319,7 @@ class TFAutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: TFT5WithLMHeadModel (T5 model) - contains `distilbert`: TFDistilBertForMaskedLM (DistilBERT model) - contains `roberta`: TFRobertaForMaskedLM (RoBERTa model) - contains `bert`: TFBertForMaskedLM (Bert model) @@ -209,6 +334,7 @@ class TFAutoModelWithLMHead(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. @@ -237,6 +363,9 @@ class TFAutoModelWithLMHead(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -261,8 +390,12 @@ class TFAutoModelWithLMHead(object): model = TFAutoModelWithLMHead.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return TFT5WithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return TFDistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return TFAlbertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return TFRobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -281,7 +414,7 @@ class TFAutoModelWithLMHead(object): return TFCTRLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'distilbert', 'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) @@ -306,8 +439,39 @@ class TFAutoModelForSequenceClassification(object): This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): - raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " - "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + raise EnvironmentError("TFAutoModelForSequenceClassification is designed to be instantiated " + "using the `TFAutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path)` or " + "`TFAutoModelForSequenceClassification.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForSequenceClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return TFDistilBertForSequenceClassification(config) + elif isinstance(config, RobertaConfig): + return TFRobertaForSequenceClassification(config) + elif isinstance(config, BertConfig): + return TFBertForSequenceClassification(config) + elif isinstance(config, XLNetConfig): + return TFXLNetForSequenceClassification(config) + elif isinstance(config, XLMConfig): + return TFXLMForSequenceClassification(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -332,6 +496,7 @@ class TFAutoModelForSequenceClassification(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. @@ -360,6 +525,9 @@ class TFAutoModelForSequenceClassification(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -386,6 +554,8 @@ class TFAutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return TFDistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return TFAlbertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return TFRobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -396,7 +566,7 @@ class TFAutoModelForSequenceClassification(object): return TFXLMForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)) + "'distilbert', 'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)) class TFAutoModelForQuestionAnswering(object): @@ -419,8 +589,36 @@ class TFAutoModelForQuestionAnswering(object): This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): - raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " - "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + raise EnvironmentError("TFAutoModelForQuestionAnswering is designed to be instantiated " + "using the `TFAutoModelForQuestionAnswering.from_pretrained(pretrained_model_name_or_path)` or " + "`TFAutoModelForQuestionAnswering.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForSequenceClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return TFDistilBertForQuestionAnswering(config) + elif isinstance(config, BertConfig): + return TFBertForQuestionAnswering(config) + elif isinstance(config, XLNetConfig): + return TFXLNetForQuestionAnswering(config) + elif isinstance(config, XLMConfig): + return TFXLMForQuestionAnswering(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -444,6 +642,7 @@ class TFAutoModelForQuestionAnswering(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. @@ -472,6 +671,9 @@ class TFAutoModelForQuestionAnswering(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -506,4 +708,121 @@ class TFAutoModelForQuestionAnswering(object): return TFXLMForQuestionAnsweringSimple.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) + "'distilbert', 'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) + + +class TFAutoModelForTokenClassification: + def __init__(self): + raise EnvironmentError("TFAutoModelForTokenClassification is designed to be instantiated " + "using the `TFAutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelForTokenClassification.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBert model) + - isInstance of `roberta` configuration class: RobteraModel (Roberta model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = TFAutoModelForTokenClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, BertConfig): + return TFBertForTokenClassification(config) + elif isinstance(config, XLNetConfig): + return TFXLNetForTokenClassification(config) + elif isinstance(config, DistilBertConfig): + return TFDistilBertForTokenClassification(config) + elif isinstance(config, RobertaConfig): + return TFRobertaForTokenClassification(config) + raise ValueError("Unrecognized configuration class {}".format(config)) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the question answering model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `bert`: BertForTokenClassification (Bert model) + - contains `xlnet`: XLNetForTokenClassification (XLNet model) + - contains `distilbert`: DistilBertForTokenClassification (DistilBert model) + - contains `roberta`: RobertaForTokenClassification (Roberta model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModelForTokenClassification.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModelForTokenClassification.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModelForTokenClassification.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModelForTokenClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'bert' in pretrained_model_name_or_path: + return TFBertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlnet' in pretrained_model_name_or_path: + return TFXLNetForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: + return TFDistilBertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'roberta' in pretrained_model_name_or_path: + return TFRobertaForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'xlnet', 'distilbert', 'roberta'".format(pretrained_model_name_or_path)) diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index ad0815e2..9caad53a 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -28,7 +28,7 @@ import numpy as np import tensorflow as tf from .configuration_bert import BertConfig -from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -48,6 +48,12 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-tf_model.h5", 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-tf_model.h5", 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-tf_model.h5", + 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-tf_model.h5", + 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-tf_model.h5", + 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-tf_model.h5", + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-tf_model.h5", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/tf_model.h5", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1/tf_model.h5", } @@ -129,7 +135,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): linear tensor, float32 with shape [batch_size, length, vocab_size]. Raises: ValueError: if mode is not valid. - + Shared weights logic adapted from https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 """ @@ -145,10 +151,10 @@ class TFBertEmbeddings(tf.keras.layers.Layer): input_ids, position_ids, token_type_ids, inputs_embeds = inputs if input_ids is not None: - input_shape = tf.shape(input_ids) + input_shape = shape_list(input_ids) else: - input_shape = tf.shape(inputs_embeds)[:-1] - + input_shape = shape_list(inputs_embeds)[:-1] + seq_length = input_shape[1] if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] @@ -172,8 +178,8 @@ class TFBertEmbeddings(tf.keras.layers.Layer): Returns: float32 tensor with shape [batch_size, length, vocab_size]. """ - batch_size = tf.shape(inputs)[0] - length = tf.shape(inputs)[1] + batch_size = shape_list(inputs)[0] + length = shape_list(inputs)[1] x = tf.reshape(inputs, [-1, self.hidden_size]) logits = tf.matmul(x, self.word_embeddings, transpose_b=True) @@ -214,7 +220,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer): def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs - batch_size = tf.shape(hidden_states)[0] + batch_size = shape_list(hidden_states)[0] mixed_query_layer = self.query(hidden_states) mixed_key_layer = self.key(hidden_states) mixed_value_layer = self.value(hidden_states) @@ -225,7 +231,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer): # Take the dot product between "query" and "key" to get the raw attention scores. attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) # (batch size, num_heads, seq_len_q, seq_len_k) - dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) # scale attention_scores + dk = tf.cast(shape_list(key_layer)[-1], tf.float32) # scale attention_scores attention_scores = attention_scores / tf.math.sqrt(dk) if attention_mask is not None: @@ -246,7 +252,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer): context_layer = tf.matmul(attention_probs, value_layer) context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) - context_layer = tf.reshape(context_layer, + context_layer = tf.reshape(context_layer, (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,) @@ -502,9 +508,9 @@ class TFBertMainLayer(tf.keras.layers.Layer): if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: - input_shape = input_ids.shape + input_shape = shape_list(input_ids) elif inputs_embeds is not None: - input_shape = inputs_embeds.shape[:-1] + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either input_ids or inputs_embeds") @@ -591,7 +597,7 @@ BERT_START_DOCSTRING = r""" The BERT model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ @@ -605,13 +611,13 @@ BERT_INPUTS_DOCSTRING = r""" (a) For sequence pairs: ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` - + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` (b) For single sequences: ``tokens: [CLS] the dog is hairy . [SEP]`` - + ``token_type_ids: 0 0 0 0 0 0 0`` Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on @@ -671,7 +677,7 @@ class TFBertModel(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertModel.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -710,7 +716,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForPreTraining.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores, seq_relationship_scores = outputs[:2] @@ -759,7 +765,7 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForMaskedLM.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores = outputs[0] @@ -806,7 +812,7 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) seq_relationship_scores = outputs[0] @@ -851,7 +857,7 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -939,11 +945,11 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): input_ids = inputs if input_ids is not None: - num_choices = tf.shape(input_ids)[1] - seq_length = tf.shape(input_ids)[2] + num_choices = shape_list(input_ids)[1] + seq_length = shape_list(input_ids)[2] else: - num_choices = tf.shape(inputs_embeds)[1] - seq_length = tf.shape(inputs_embeds)[2] + num_choices = shape_list(inputs_embeds)[1] + seq_length = shape_list(inputs_embeds)[2] flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None @@ -988,7 +994,7 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForTokenClassification.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) scores = outputs[0] @@ -1041,7 +1047,7 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForQuestionAnswering.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) start_scores, end_scores = outputs[:2] diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py index ae66dbc8..0f9b3492 100644 --- a/transformers/modeling_tf_ctrl.py +++ b/transformers/modeling_tf_ctrl.py @@ -95,7 +95,7 @@ class TFMultiHeadAttention(tf.keras.layers.Layer): def call(self, inputs, training=False): v, k, q, mask, layer_past, attention_mask, head_mask = inputs - batch_size = q.shape[0] + batch_size = shape_list(q)[0] q = self.Wq(q) k = self.Wk(k) @@ -400,7 +400,7 @@ class TFCTRLModel(TFCTRLPreTrainedModel): **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -418,7 +418,7 @@ class TFCTRLModel(TFCTRLPreTrainedModel): tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = TFCTRLModel.from_pretrained('ctrl') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -462,7 +462,7 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel): **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: - list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -481,7 +481,7 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel): tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = TFCTRLLMHeadModel.from_pretrained('ctrl') - input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=input_ids) loss, logits = outputs[:2] diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index 6d393bb9..afd88d7e 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -37,7 +37,8 @@ logger = logging.getLogger(__name__) TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-tf_model.h5", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-tf_model.h5" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-tf_model.h5", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-tf_model.h5", } @@ -137,9 +138,9 @@ class TFEmbeddings(tf.keras.layers.Layer): input_ids, position_ids = inputs if input_ids is not None: - seq_length = tf.shape(input_ids)[1] + seq_length = shape_list(input_ids)[1] else: - seq_length = tf.shape(inputs_embeds)[1] + seq_length = shape_list(inputs_embeds)[1] if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] @@ -160,8 +161,8 @@ class TFEmbeddings(tf.keras.layers.Layer): Returns: float32 tensor with shape [batch_size, length, vocab_size]. """ - batch_size = tf.shape(inputs)[0] - length = tf.shape(inputs)[1] + batch_size = shape_list(inputs)[0] + length = shape_list(inputs)[1] x = tf.reshape(inputs, [-1, self.dim]) logits = tf.matmul(x, self.word_embeddings, transpose_b=True) @@ -703,6 +704,53 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): return outputs # logits, (hidden_states), (attentions) +@add_start_docstrings("""DistilBert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) +class TFDistilBertForTokenClassification(TFDistilBertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + import tensorflow as tf + from transformers import DistilBertTokenizer, TFDistilBertForTokenClassification + tokenizer = DistilBertTokenizer.from_pretrained('bert-base-uncased') + model = TFDistilBertForTokenClassification.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + scores = outputs[0] + """ + def __init__(self, config, *inputs, **kwargs): + super(TFDistilBertForTokenClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.distilbert = TFDistilBertMainLayer(config, name='distilbert') + self.dropout = tf.keras.layers.Dropout(config.dropout) + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + outputs = self.distilbert(inputs, **kwargs) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output, training=kwargs.get('training', False)) + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # scores, (hidden_states), (attentions) + + @add_start_docstrings("""DistilBert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). """, DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py index 5e416a5e..718e8f60 100644 --- a/transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -92,7 +92,7 @@ class TFAttention(tf.keras.layers.Layer): # q, k, v have shape [batch, heads, sequence, features] w = tf.matmul(q, k, transpose_b=True) if self.scale: - dk = tf.cast(tf.shape(k)[-1], tf.float32) # scale attention_scores + dk = tf.cast(shape_list(k)[-1], tf.float32) # scale attention_scores w = w / tf.math.sqrt(dk) # w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst. @@ -436,7 +436,7 @@ class TFGPT2Model(TFGPT2PreTrainedModel): **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -454,7 +454,7 @@ class TFGPT2Model(TFGPT2PreTrainedModel): tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = TFGPT2Model.from_pretrained('gpt2') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -476,7 +476,7 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): **prediction_scores**: `tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: - list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -495,7 +495,7 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = TFGPT2LMHeadModel.from_pretrained('gpt2') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -535,7 +535,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): **mc_prediction_scores**: `tf.Tensor`` of shape ``(batch_size, num_choices)`` Prediction scores of the multiplechoice classification head (scores for each choice before SoftMax). **past**: - list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -574,6 +574,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFGPT2DoubleHeadsModel, self).__init__(config, *inputs, **kwargs) + config.num_labels = 1 self.transformer = TFGPT2MainLayer(config, name='transformer') self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head') diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py index c553d923..791c6dcc 100644 --- a/transformers/modeling_tf_openai.py +++ b/transformers/modeling_tf_openai.py @@ -98,7 +98,7 @@ class TFAttention(tf.keras.layers.Layer): # q, k, v have shape [batch, heads, sequence, features] w = tf.matmul(q, k, transpose_b=True) if self.scale: - dk = tf.cast(tf.shape(k)[-1], tf.float32) # scale attention_scores + dk = tf.cast(shape_list(k)[-1], tf.float32) # scale attention_scores w = w / tf.math.sqrt(dk) # w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst. @@ -431,7 +431,7 @@ class TFOpenAIGPTModel(TFOpenAIGPTPreTrainedModel): tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = TFOpenAIGPTModel.from_pretrained('openai-gpt') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -467,7 +467,7 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel): tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = TFOpenAIGPTLMHeadModel.from_pretrained('openai-gpt') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -538,6 +538,7 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFOpenAIGPTDoubleHeadsModel, self).__init__(config, *inputs, **kwargs) + config.num_labels = 1 self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head') diff --git a/transformers/modeling_tf_pytorch_utils.py b/transformers/modeling_tf_pytorch_utils.py index 88ce4d46..190caff1 100644 --- a/transformers/modeling_tf_pytorch_utils.py +++ b/transformers/modeling_tf_pytorch_utils.py @@ -78,6 +78,7 @@ def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_i logger.info("Loading PyTorch weights from {}".format(pt_path)) pt_state_dict = torch.load(pt_path, map_location='cpu') + logger.info("PyTorch checkpoint contains {:,} parameters".format(sum(t.numel() for t in pt_state_dict.values()))) return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys) @@ -131,7 +132,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a start_prefix_to_remove = tf_model.base_model_prefix + '.' symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights - + tf_loaded_numel = 0 weight_value_tuples = [] all_pytorch_weights = set(list(pt_state_dict.keys())) for symbolic_weight in symbolic_weights: @@ -139,7 +140,11 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a name, transpose = convert_tf_weight_name_to_pt_weight_name(sw_name, start_prefix_to_remove=start_prefix_to_remove) # Find associated numpy array in pytorch model state dict - assert name in pt_state_dict, "{} not found in PyTorch model".format(name) + if name not in pt_state_dict: + if allow_missing_keys: + continue + raise AttributeError("{} not found in PyTorch model".format(name)) + array = pt_state_dict[name].numpy() if transpose: @@ -156,7 +161,8 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a e.args += (symbolic_weight.shape, array.shape) raise e - logger.info("Initialize TF weight {}".format(symbolic_weight.name)) + tf_loaded_numel += array.size + # logger.warning("Initialize TF weight {}".format(symbolic_weight.name)) weight_value_tuples.append((symbolic_weight, array)) all_pytorch_weights.discard(name) @@ -166,6 +172,8 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a if tf_inputs is not None: tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + logger.info("Loaded {:,} parameters in the TF 2.0 model.".format(tf_loaded_numel)) + logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) return tf_model @@ -243,6 +251,7 @@ def load_tf2_weights_in_pytorch_model(pt_model, tf_weights, allow_missing_keys=F all_tf_weights = set(list(tf_weights_map.keys())) loaded_pt_weights_data_ptr = {} + missing_keys_pt = [] for pt_weight_name, pt_weight in current_pt_params_dict.items(): # Handle PyTorch shared weight ()not duplicated in TF 2.0 if pt_weight.data_ptr() in loaded_pt_weights_data_ptr: @@ -251,7 +260,10 @@ def load_tf2_weights_in_pytorch_model(pt_model, tf_weights, allow_missing_keys=F # Find associated numpy array in pytorch model state dict if pt_weight_name not in tf_weights_map: - raise ValueError("{} not found in TF 2.0 model".format(pt_weight_name)) + if allow_missing_keys: + missing_keys_pt.append(pt_weight_name) + continue + raise AttributeError("{} not found in TF 2.0 model".format(pt_weight_name)) array, transpose = tf_weights_map[pt_weight_name] @@ -269,13 +281,14 @@ def load_tf2_weights_in_pytorch_model(pt_model, tf_weights, allow_missing_keys=F e.args += (pt_weight.shape, array.shape) raise e - logger.info("Initialize PyTorch weight {}".format(pt_weight_name)) + # logger.warning("Initialize PyTorch weight {}".format(pt_weight_name)) new_pt_params_dict[pt_weight_name] = torch.from_numpy(array) loaded_pt_weights_data_ptr[pt_weight.data_ptr()] = torch.from_numpy(array) all_tf_weights.discard(pt_weight_name) missing_keys, unexpected_keys = pt_model.load_state_dict(new_pt_params_dict, strict=False) + missing_keys += missing_keys_pt if len(missing_keys) > 0: logger.info("Weights of {} not initialized from TF 2.0 model: {}".format( diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py index 450c0c72..ad730b58 100644 --- a/transformers/modeling_tf_roberta.py +++ b/transformers/modeling_tf_roberta.py @@ -24,7 +24,7 @@ import numpy as np import tensorflow as tf from .configuration_roberta import RobertaConfig -from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list from .file_utils import add_start_docstrings from .modeling_tf_bert import TFBertEmbeddings, TFBertMainLayer, gelu, gelu_new @@ -51,9 +51,9 @@ class TFRobertaEmbeddings(TFBertEmbeddings): input_ids, position_ids, token_type_ids, inputs_embeds = inputs if input_ids is not None: - seq_length = tf.shape(input_ids)[1] + seq_length = shape_list(input_ids)[1] else: - seq_length = tf.shape(inputs_embeds)[1] + seq_length = shape_list(inputs_embeds)[1] if position_ids is None: position_ids = tf.range(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=tf.int32)[tf.newaxis, :] @@ -199,7 +199,7 @@ class TFRobertaModel(TFRobertaPreTrainedModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = TFRobertaModel.from_pretrained('roberta-base') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -276,7 +276,7 @@ class TFRobertaForMaskedLM(TFRobertaPreTrainedModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = TFRobertaForMaskedLM.from_pretrained('roberta-base') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) prediction_scores = outputs[0] @@ -347,7 +347,7 @@ class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel): tokenizer = RoertaTokenizer.from_pretrained('roberta-base') model = TFRobertaForSequenceClassification.from_pretrained('roberta-base') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 labels = tf.constant([1])[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py new file mode 100644 index 00000000..e803e00c --- /dev/null +++ b/transformers/modeling_tf_t5.py @@ -0,0 +1,775 @@ +# coding=utf-8 +# Copyright 2018 T5 Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 T5 model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import logging +import math +import copy +import itertools + +import tensorflow as tf + +from .configuration_t5 import T5Config +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list +from .file_utils import add_start_docstrings, DUMMY_INPUTS, DUMMY_MASK + +logger = logging.getLogger(__name__) + +TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP = { + 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-tf_model.h5", + 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-tf_model.h5", + 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-tf_model.h5", + 't5-3b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3b-tf_model.h5", + 't5-11b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11b-tf_model.h5", +} + +#################################################### +# TF 2.0 Models are constructed using Keras imperative API by sub-classing +# - tf.keras.layers.Layer for the layers and +# - TFPreTrainedModel for the models (it-self a sub-class of tf.keras.Model) +#################################################### + +class TFT5LayerNorm(tf.keras.layers.Layer): + def __init__(self, epsilon=1e-6, **kwargs): + """ Construct a layernorm module in the T5 style + No bias and no substraction of mean. + """ + super(TFT5LayerNorm, self).__init__(**kwargs) + self.variance_epsilon = epsilon + + def build(self, input_shape): + """Build shared word embedding layer """ + self.weight = self.add_weight( + "weight", + shape=(input_shape[-1],), + initializer='ones') + super(TFT5LayerNorm, self).build(input_shape) + + def call(self, x): + variance = tf.math.reduce_mean(tf.math.square(x), axis=-1, keepdims=True) + x = x * tf.math.rsqrt(variance + self.variance_epsilon) + return self.weight * x + + +class TFT5DenseReluDense(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFT5DenseReluDense, self).__init__(**kwargs) + self.wi = tf.keras.layers.Dense(config.d_ff, use_bias=False, name='wi') + self.wo = tf.keras.layers.Dense(config.d_model, use_bias=False, name='wo') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + self.act = tf.keras.activations.relu + + def call(self, hidden_states, training=False): + h = self.wi(hidden_states) + h = self.act(h) + h = self.dropout(h, training=training) + h = self.wo(h) + return h + + +class TFT5LayerFF(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFT5LayerFF, self).__init__(**kwargs) + self.DenseReluDense = TFT5DenseReluDense(config, name='DenseReluDense') + self.layer_norm = TFT5LayerNorm(epsilon=config.layer_norm_epsilon, + name='layer_norm') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + + def call(self, hidden_states, training=False): + norm_x = self.layer_norm(hidden_states) + y = self.DenseReluDense(norm_x, training=training) + layer_output = hidden_states + self.dropout(y, training=training) + return layer_output + + +class TFT5Attention(tf.keras.layers.Layer): + NEW_ID = itertools.count() + + def __init__(self, config, has_relative_attention_bias=False, **kwargs): + super(TFT5Attention, self).__init__(**kwargs) + self.layer_id = next(TFT5Attention.NEW_ID) + self.is_decoder = config.is_decoder + self.has_relative_attention_bias = has_relative_attention_bias + + self.output_attentions = config.output_attentions + self.relative_attention_num_buckets = config.relative_attention_num_buckets + self.d_model = config.d_model + self.d_kv = config.d_kv + self.n_heads = config.num_heads + self.inner_dim = self.n_heads * self.d_kv + + # Mesh TensorFlow initialization to avoid scaling before softmax + self.q = tf.keras.layers.Dense(self.inner_dim, use_bias=False, name='q') + self.k = tf.keras.layers.Dense(self.inner_dim, use_bias=False, name='k') + self.v = tf.keras.layers.Dense(self.inner_dim, use_bias=False, name='v') + self.o = tf.keras.layers.Dense(self.d_model, use_bias=False, name='o') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + + if self.has_relative_attention_bias: + self.relative_attention_bias = tf.keras.layers.Embedding(self.relative_attention_num_buckets, + self.n_heads, + name='relative_attention_bias') + self.pruned_heads = set() + + def prune_heads(self, heads): + raise NotImplementedError + + @staticmethod + def _relative_position_bucket(relative_position, + bidirectional=True, + num_buckets=32, + max_distance=128): + """ + Adapted from Mesh Tensorflow: + https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 + + Translate relative position to a bucket number for relative attention. + The relative position is defined as memory_position - query_position, i.e. + the distance in tokens from the attending position to the attended-to + position. If bidirectional=False, then positive relative positions are + invalid. + We use smaller buckets for small absolute relative_position and larger buckets + for larger absolute relative_positions. All relative positions >=max_distance + map to the same bucket. All relative positions <=-max_distance map to the + same bucket. This should allow for more graceful generalization to longer + sequences than the model has been trained on. + Args: + relative_position: an int32 Tensor + bidirectional: a boolean - whether the attention is bidirectional + num_buckets: an integer + max_distance: an integer + Returns: + a Tensor with the same shape as relative_position, containing int32 + values in the range [0, num_buckets) + """ + ret = 0 + n = -relative_position + if bidirectional: + num_buckets //= 2 + ret += tf.dtypes.cast(tf.math.less(n, 0), tf.int32) * num_buckets + n = tf.math.abs(n) + else: + n = tf.math.maximum(n, 0) + # now n is in the range [0, inf) + max_exact = num_buckets // 2 + is_small = tf.math.less(n, max_exact) + val_if_large = max_exact + tf.dtypes.cast( + tf.math.log(tf.dtypes.cast(n, tf.float32) / max_exact) + / math.log(max_distance / max_exact) * (num_buckets - max_exact), tf.int32) + val_if_large = tf.math.minimum(val_if_large, num_buckets - 1) + ret += tf.where(is_small, n, val_if_large) + return ret + + def compute_bias(self, qlen, klen): + """ Compute binned relative position bias """ + context_position = tf.range(qlen)[:, None] + memory_position = tf.range(klen)[None, :] + relative_position = memory_position - context_position # shape (qlen, klen) + rp_bucket = self._relative_position_bucket(relative_position, + bidirectional=not self.is_decoder, + num_buckets=self.relative_attention_num_buckets) + values = self.relative_attention_bias(rp_bucket) # shape (qlen, klen, num_heads) + values = tf.expand_dims(tf.transpose(values, [2, 0, 1]), axis=0) # shape (1, num_heads, qlen, klen) + return values + + def call(self, input, mask=None, kv=None, position_bias=None, cache=None, head_mask=None, training=False): + """ + Self-attention (if kv is None) or attention over source sentence (provided by kv). + """ + # Input is (bs, qlen, dim) + # Mask is (bs, klen) (non-causal) or (bs, klen, klen) + bs, qlen, dim = shape_list(input) + if kv is None: + klen = qlen if cache is None else cache['slen'] + qlen + else: + klen = shape_list(kv)[1] + + def shape(x): + """ projection """ + return tf.transpose(tf.reshape(x, (bs, -1, self.n_heads, self.d_kv)), perm=(0, 2, 1, 3)) + + def unshape(x): + """ compute context """ + return tf.reshape(tf.transpose(x, perm=(0, 2, 1, 3)), (bs, -1, self.inner_dim)) + + q = shape(self.q(input)) # (bs, n_heads, qlen, dim_per_head) + if kv is None: + k = shape(self.k(input)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v(input)) # (bs, n_heads, qlen, dim_per_head) + elif cache is None or self.layer_id not in cache: + k = v = kv + k = shape(self.k(k)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v(v)) # (bs, n_heads, qlen, dim_per_head) + + if cache is not None: + if self.layer_id in cache: + if kv is None: + k_, v_ = cache[self.layer_id] + k = tf.concat([k_, k], axis=2) # (bs, n_heads, klen, dim_per_head) + v = tf.concat([v_, v], axis=2) # (bs, n_heads, klen, dim_per_head) + else: + k, v = cache[self.layer_id] + cache[self.layer_id] = (k, v) + + # q = q / math.sqrt(dim_per_head) # No scaling in T5 + # scores = tf.matmul(q, k, transpose_b=True) # (bs, n_heads, qlen, klen) + scores = tf.einsum('bnqd,bnkd->bnqk', q, k) # (bs, n_heads, qlen, klen) + + if position_bias is None: + if not self.has_relative_attention_bias: + raise ValueError("No position_bias provided and no weights to compute position_bias") + position_bias = self.compute_bias(qlen, klen) + if mask is not None: + position_bias = position_bias + mask + # mask = (mask == 0).expand_as(scores) # (bs, n_heads, qlen, klen) + # scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) + + scores += position_bias + weights = tf.nn.softmax(scores, axis=-1) # (bs, n_heads, qlen, klen) + weights = self.dropout(weights, training=training) # (bs, n_heads, qlen, klen) + + # Mask heads if we want to + if head_mask is not None: + weights = weights * head_mask + + context = tf.matmul(weights, v) # (bs, n_heads, qlen, dim_per_head) + context = unshape(context) # (bs, qlen, dim) + + context = self.o(context) + + outputs = (context,) + if self.output_attentions: + outputs = outputs + (weights,) + if self.has_relative_attention_bias: + outputs = outputs + (position_bias,) + return outputs + + +class TFT5LayerSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, has_relative_attention_bias=False, **kwargs): + super(TFT5LayerSelfAttention, self).__init__(**kwargs) + self.SelfAttention = TFT5Attention(config, + has_relative_attention_bias=has_relative_attention_bias, + name='SelfAttention') + self.layer_norm = TFT5LayerNorm(epsilon=config.layer_norm_epsilon, + name='layer_norm') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + + def call(self, hidden_states, attention_mask=None, position_bias=None, + head_mask=None, training=False): + norm_x = self.layer_norm(hidden_states) + attention_output = self.SelfAttention(norm_x, + mask=attention_mask, + position_bias=position_bias, + head_mask=head_mask, + training=training) + y = attention_output[0] + layer_output = hidden_states + self.dropout(y, training=training) + outputs = (layer_output,) + attention_output[1:] # add attentions if we output them + return outputs + + +class TFT5LayerCrossAttention(tf.keras.layers.Layer): + def __init__(self, config, has_relative_attention_bias=False, **kwargs): + super(TFT5LayerCrossAttention, self).__init__(**kwargs) + self.EncDecAttention = TFT5Attention(config, + has_relative_attention_bias=has_relative_attention_bias, + name='EncDecAttention') + self.layer_norm = TFT5LayerNorm(epsilon=config.layer_norm_epsilon, + name='layer_norm') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + + def call(self, hidden_states, kv, attention_mask=None, position_bias=None, + head_mask=None, training=False): + norm_x = self.layer_norm(hidden_states) + attention_output = self.EncDecAttention(norm_x, + mask=attention_mask, + kv=kv, + position_bias=position_bias, + head_mask=head_mask, + training=training) + y = attention_output[0] + layer_output = hidden_states + self.dropout(y, training=training) + outputs = (layer_output,) + attention_output[1:] # add attentions if we output them + return outputs + + +class TFT5Block(tf.keras.layers.Layer): + def __init__(self, config, has_relative_attention_bias=False, **kwargs): + super(TFT5Block, self).__init__(**kwargs) + self.is_decoder = config.is_decoder + self.layer = [] + self.layer.append(TFT5LayerSelfAttention(config, + has_relative_attention_bias=has_relative_attention_bias, + name='layer_._0')) + if self.is_decoder: + self.layer.append(TFT5LayerCrossAttention(config, + has_relative_attention_bias=has_relative_attention_bias, + name='layer_._1')) + self.layer.append(TFT5LayerFF(config, name='layer_._2')) + else: + self.layer.append(TFT5LayerFF(config, name='layer_._1')) + + def call(self, hidden_states, attention_mask=None, position_bias=None, + encoder_hidden_states=None, encoder_attention_mask=None, encoder_decoder_position_bias=None, + head_mask=None, training=False): + self_attention_outputs = self.layer[0](hidden_states, + attention_mask=attention_mask, + position_bias=position_bias, + head_mask=head_mask, + training=training) + hidden_states = self_attention_outputs[0] + outputs = self_attention_outputs[1:] + + if not self.is_decoder: + hidden_states = self.layer[1](hidden_states, training=training) + else: + cross_attention_outputs = self.layer[1](hidden_states, + kv=encoder_hidden_states, + attention_mask=encoder_attention_mask, + position_bias=encoder_decoder_position_bias, + head_mask=head_mask, + training=training) + hidden_states = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[1:] + hidden_states = self.layer[2](hidden_states, training=training) + + outputs = (hidden_states,) + outputs # add attentions if we output them + return outputs # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) + + +#################################################### +# The full model without a specific pretrained or finetuning head is +# provided as a tf.keras.layers.Layer usually called "TFT5MainLayer" +#################################################### +class TFT5MainLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFT5MainLayer, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.is_decoder = config.is_decoder + self.config = config + self.num_hidden_layers = config.num_layers + + self.block = [TFT5Block(config, + has_relative_attention_bias=bool(i == 0), + name='block_._{}'.format(i)) + for i in range(config.num_layers)] + self.final_layer_norm = TFT5LayerNorm(epsilon=config.layer_norm_epsilon, + name='final_layer_norm') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError # Not implemented yet in the library fr TF 2.0 models + + def _prune_heads(self, heads_to_prune): + raise NotImplementedError # Not implemented yet in the library fr TF 2.0 models + + def call(self, hidden_states, attention_mask=None, encoder_hidden_states=None, + encoder_attention_mask=None, head_mask=None, training=False): + + batch_size, seq_length = shape_list(hidden_states)[:2] + if attention_mask is None: + attention_mask = tf.fill((batch_size, seq_length), 1) + if self.is_decoder and encoder_attention_mask is None: + encoder_seq_length = encoder_hidden_states.shape[1] + encoder_attention_mask = tf.fill((batch_size, encoder_seq_length), 1) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + attention_mask = tf.cast(attention_mask, dtype=tf.float32) + num_dims_attention_mask = len(shape_list(attention_mask)) + if num_dims_attention_mask == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif num_dims_attention_mask == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder: + seq_ids = tf.range(seq_length) + causal_mask = tf.less_equal(tf.tile(seq_ids[None, None, :], (batch_size, seq_length, 1)), + seq_ids[None, :, None]) + causal_mask = tf.cast(causal_mask, dtype=tf.float32) + extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] + else: + extended_attention_mask = attention_mask[:, None, None, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + # T5 has a mask that can compare sequence ids, we can simulate this here with this transposistion + # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 + # extended_attention_mask = tf.math.equal(extended_attention_mask, + # tf.transpose(extended_attention_mask, perm=(-1, -2))) + + extended_attention_mask = (1.0 - extended_attention_mask) * -1e9 + + if self.is_decoder: + # If a 2D ou 3D attention mask is provided for the cross-attention + # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] + encoder_attention_mask = tf.cast(encoder_attention_mask, dtype=tf.float32) + num_dims_encoder_attention_mask = len(shape_list(encoder_attention_mask)) + if num_dims_encoder_attention_mask == 3: + encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] + if num_dims_encoder_attention_mask == 2: + encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + + # T5 has a mask that can compare sequence ids, we can simulate this here with this transposistion + # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 + # encoder_extended_attention_mask = tf.math.equal(encoder_extended_attention_mask, + # tf.transpose(encoder_extended_attention_mask, perm=(-1, -2))) + + encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -1e9 + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + all_hidden_states = () + all_attentions = () + position_bias = None + encoder_decoder_position_bias = None + for i, layer_module in enumerate(self.block): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module(hidden_states, + attention_mask=extended_attention_mask, + position_bias=position_bias, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + encoder_decoder_position_bias=encoder_decoder_position_bias, + head_mask=head_mask[i], + training=training) + hidden_states = layer_outputs[0] + if i == 0: + # We share the position biases between the layers - the first layer store them + # layer_outputs = hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) + position_bias = layer_outputs[2 if self.output_attentions else 1] + if self.is_decoder: + encoder_decoder_position_bias = layer_outputs[4 if self.output_attentions else 2] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + hidden_states = self.final_layer_norm(hidden_states) + layer_output = self.dropout(hidden_states, training=training) + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + + +#################################################### +# TFT5PreTrainedModel is a sub-class of tf.keras.Model +# which take care of loading and saving pretrained weights +# and various common utilities. +# Here you just need to specify a few (self-explanatory) +# pointers for your model. +#################################################### +class TFT5PreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = T5Config + pretrained_model_archive_map = TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "transformer" + + @property + def dummy_inputs(self): + input_ids = tf.constant(DUMMY_INPUTS) + input_mask = tf.constant(DUMMY_MASK) + dummy_inputs = {'decoder_input_ids': input_ids, + 'encoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + return dummy_inputs + + +T5_START_DOCSTRING = r""" The T5 model was proposed in + `Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`_ + by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. + It's an encoder decoder transformer pre-trained in a text-to-text denoising generative setting. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`: + https://arxiv.org/abs/1910.10683 + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~transformers.T5Config`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +T5_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, T5 input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + + T5 is a model with relative position embeddings so you should be able to pad the inputs on + the right or the left. + + Indices can be obtained using :class:`transformers.T5Tokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare T5 Model transformer outputting raw hidden-states" + "without any specific head on top.", + T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) +class TFT5Model(TFT5PreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import T5Tokenizer, TFT5Model + + tokenizer = T5Tokenizer.from_pretrained('t5-small') + model = TFT5Model.from_pretrained('t5-small') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids=input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFT5Model, self).__init__(config, *inputs, **kwargs) + self.shared = TFSharedEmbeddings(config.vocab_size, config.d_model, + name='shared') + + encoder_config = copy.deepcopy(config) + self.encoder = TFT5MainLayer(encoder_config, name='encoder') + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + self.decoder = TFT5MainLayer(decoder_config, name='decoder') + + def get_input_embeddings(self): + return self.shared + + def get_output_embeddings(self): + return self.shared + + def call(self, decoder_input_ids, **kwargs): + # We allow two types of multi-inputs: + # - traditional keyword arguments in the call method + # - all the arguments provided as a dict in the first positional argument of call + # The last option is useful to use the tf.keras fit() method. + + if isinstance(decoder_input_ids, dict): + kwargs.update(decoder_input_ids) + else: + kwargs['decoder_input_ids'] = decoder_input_ids + + kwargs_common = dict((k, v) for k, v in kwargs.items() + if not k.startswith("encoder_") and not k.startswith("decoder_")) + kwargs_encoder = kwargs_common.copy() + kwargs_decoder = kwargs_common.copy() + kwargs_encoder.update(dict((k[len("encoder_"):], v) for k, v in kwargs.items() if k.startswith("encoder_"))) + kwargs_decoder.update(dict((k[len("decoder_"):], v) for k, v in kwargs.items() if k.startswith("decoder_"))) + + # Encode if needed (training, first prediction pass) + encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) + if encoder_hidden_states is None: + # Convert encoder inputs in embeddings if needed + hidden_states = kwargs_encoder.pop("inputs_embeds", None) + if hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) + encoder_hidden_states = encoder_outputs[0] + else: + encoder_outputs = () + + # Decode + # Convert decoder inputs in embeddings if needed + hidden_states = kwargs_decoder.pop("inputs_embeds", None) + if hidden_states is None: + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) + + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states + kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) + decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) + + return decoder_outputs + encoder_outputs + + +@add_start_docstrings("""T5 Model with a `language modeling` head on top. """, + T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) +class TFT5WithLMHeadModel(TFT5PreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import T5Tokenizer, TFT5WithLMHeadModel + + tokenizer = T5Tokenizer.from_pretrained('t5-small') + model = TFT5WithLMHeadModel.from_pretrained('t5-small') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids=input_ids) + prediction_scores = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFT5WithLMHeadModel, self).__init__(config, *inputs, **kwargs) + self.model_dim = config.d_model + + self.shared = TFSharedEmbeddings(config.vocab_size, config.d_model, + name='shared') + + encoder_config = copy.deepcopy(config) + self.encoder = TFT5MainLayer(encoder_config, name='encoder') + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + self.decoder = TFT5MainLayer(decoder_config, name='decoder') + + def get_input_embeddings(self): + return self.shared + + def get_output_embeddings(self): + return self.shared + + def call(self, decoder_input_ids, **kwargs): + # We allow two types of multi-inputs: + # - traditional keyword arguments in the call method + # - all the arguments provided as a dict in the first positional argument of call + # The last option is useful to use the tf.keras fit() method. + + if isinstance(decoder_input_ids, dict): + kwargs.update(decoder_input_ids) + else: + kwargs['decoder_input_ids'] = decoder_input_ids + + kwargs_common = dict((k, v) for k, v in kwargs.items() + if not k.startswith("encoder_") and not k.startswith("decoder_")) + kwargs_encoder = kwargs_common.copy() + kwargs_decoder = kwargs_common.copy() + kwargs_encoder.update(dict((k[len("encoder_"):], v) for k, v in kwargs.items() if k.startswith("encoder_"))) + kwargs_decoder.update(dict((k[len("decoder_"):], v) for k, v in kwargs.items() if k.startswith("decoder_"))) + + # Encode if needed (training, first prediction pass) + encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) + if encoder_hidden_states is None: + # Convert encoder inputs in embeddings if needed + hidden_states = kwargs_encoder.pop("inputs_embeds", None) + if hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) + encoder_hidden_states = encoder_outputs[0] + else: + encoder_outputs = () + + # Decode + # Convert decoder inputs in embeddings if needed + hidden_states = kwargs_decoder.pop("inputs_embeds", None) + if hidden_states is None: + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) + + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states + kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) + decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) + + sequence_output = decoder_outputs[0] * (self.model_dim ** -0.5) + lm_logits = self.shared(sequence_output, mode="linear") + decoder_outputs = (lm_logits,) + decoder_outputs[1:] + + return decoder_outputs + encoder_outputs diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py index 8a8d11cf..08bbe740 100644 --- a/transformers/modeling_tf_transfo_xl.py +++ b/transformers/modeling_tf_transfo_xl.py @@ -337,7 +337,7 @@ class TFAdaptiveEmbedding(tf.keras.layers.Layer): emb_i = tf.einsum('id,de->ie', emb_i, self.emb_projs[i]) mask_idx = tf.cast(tf.where(mask_i), dtype=tf.int64) - emb_flat += tf.scatter_nd(mask_idx, emb_i, tf.cast(tf.shape(emb_flat), dtype=tf.int64)) + emb_flat += tf.scatter_nd(mask_idx, emb_i, tf.cast(shape_list(emb_flat), dtype=tf.int64)) embed_shape = shape_list(inp) + [self.d_proj] embed = tf.reshape(emb_flat, embed_shape) @@ -353,7 +353,7 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states - self.n_token = config.n_token + self.n_token = config.vocab_size self.d_embed = config.d_embed self.d_model = config.d_model @@ -361,7 +361,7 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): self.d_head = config.d_head self.untie_r = config.untie_r - self.word_emb = TFAdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, + self.word_emb = TFAdaptiveEmbedding(config.vocab_size, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val, init_std=config.init_std, name='word_emb') self.drop = tf.keras.layers.Dropout(config.dropout) @@ -673,7 +673,7 @@ class TFTransfoXLModel(TFTransfoXLPreTrainedModel): tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TFTransfoXLModel.from_pretrained('transfo-xl-wt103') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states, mems = outputs[:2] @@ -715,7 +715,7 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel): tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TFTransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores, mems = outputs[:2] @@ -729,7 +729,7 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel): raise NotImplementedError # use adaptive softmax (including standard softmax) else: - self.crit = TFAdaptiveSoftmaxMask(config.n_token, config.d_embed, config.d_model, + self.crit = TFAdaptiveSoftmaxMask(config.vocab_size, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val, name='crit') def reset_length(self, tgt_len, ext_len, mem_len): diff --git a/transformers/modeling_tf_transfo_xl_utilities.py b/transformers/modeling_tf_transfo_xl_utilities.py index d7666a65..f730af85 100644 --- a/transformers/modeling_tf_transfo_xl_utilities.py +++ b/transformers/modeling_tf_transfo_xl_utilities.py @@ -25,15 +25,15 @@ import tensorflow as tf from .modeling_tf_utils import shape_list class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): - def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, + def __init__(self, vocab_size, d_embed, d_proj, cutoffs, div_val=1, keep_order=False, **kwargs): super(TFAdaptiveSoftmaxMask, self).__init__(**kwargs) - self.n_token = n_token + self.vocab_size = vocab_size self.d_embed = d_embed self.d_proj = d_proj - self.cutoffs = cutoffs + [n_token] + self.cutoffs = cutoffs + [vocab_size] self.cutoff_ends = [0] + self.cutoffs self.div_val = div_val @@ -66,11 +66,11 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): self.out_projs.append(weight) else: self.out_projs.append(None) - weight = self.add_weight(shape=(self.n_token, self.d_embed,), + weight = self.add_weight(shape=(self.vocab_size, self.d_embed,), initializer='zeros', trainable=True, name='out_layers_._{}_._weight'.format(i)) - bias = self.add_weight(shape=(self.n_token,), + bias = self.add_weight(shape=(self.vocab_size,), initializer='zeros', trainable=True, name='out_layers_._{}_._bias'.format(i)) @@ -105,7 +105,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): @staticmethod def _gather_logprob(logprob, target): - lp_size = tf.shape(logprob) + lp_size = shape_list(logprob) r = tf.range(lp_size[0]) idx = tf.stack([r, target], 1) return tf.gather_nd(logprob, idx) @@ -114,7 +114,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): hidden, target = inputs head_logprob = 0 if self.n_clusters == 0: - softmax_b = tf.get_variable('bias', [n_token], initializer=tf.zeros_initializer()) + softmax_b = tf.get_variable('bias', [self.config.vocab_size], initializer=tf.zeros_initializer()) output = self._logit(hidden, self.out_layers[0][0], self.out_layers[0][1], self.out_projs[0]) if target is not None: loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, logits=output) @@ -159,7 +159,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): cur_logprob = self._gather_logprob(cur_tail_logprob, cur_target) cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1] if target is not None: - loss += tf.scatter_nd(mask_idx, -cur_logprob, tf.cast(tf.shape(loss), dtype=tf.int64)) + loss += tf.scatter_nd(mask_idx, -cur_logprob, tf.cast(shape_list(loss), dtype=tf.int64)) out = tf.concat(out, axis=-1) if target is not None: diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index e08605d1..0aa65a9f 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -22,15 +22,16 @@ import logging import os import tensorflow as tf +from tensorflow.python.keras.saving import hdf5_format +import h5py from .configuration_utils import PretrainedConfig -from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME +from .file_utils import (TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, WEIGHTS_NAME, DUMMY_INPUTS, + cached_path, hf_bucket_url, is_remote_url) from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model logger = logging.getLogger(__name__) -DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] - class TFPreTrainedModel(tf.keras.Model): r""" Base class for all TF models. @@ -51,7 +52,15 @@ class TFPreTrainedModel(tf.keras.Model): config_class = None pretrained_model_archive_map = {} base_model_prefix = "" - dummy_inputs = tf.constant(DUMMY_INPUTS) # dummy inputs to build the network + + @property + def dummy_inputs(self): + """ Dummy inputs to build the network. + + Returns: + tf.Tensor with dummy inputs + """ + return {'input_ids': tf.constant(DUMMY_INPUTS)} def __init__(self, config, *inputs, **kwargs): super(TFPreTrainedModel, self).__init__(*inputs, **kwargs) @@ -168,13 +177,16 @@ class TFPreTrainedModel(tf.keras.Model): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch state_dict save file` (e.g. `./pt_model/pytorch_model.bin`). In this case, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the PyTorch checkpoint in a TensorFlow model using the provided conversion scripts and loading the TensorFlow model afterwards. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + config: (`optional`) one of: + - an instance of a class derived from :class:`~transformers.PretrainedConfig`, or + - a string valid as input to :func:`~transformers.PretrainedConfig.from_pretrained()` Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or @@ -191,10 +203,16 @@ class TFPreTrainedModel(tf.keras.Model): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + kwargs: (`optional`) Remaining dictionary of keyword arguments: Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: @@ -216,14 +234,18 @@ class TFPreTrainedModel(tf.keras.Model): cache_dir = kwargs.pop('cache_dir', None) from_pt = kwargs.pop('from_pt', False) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) + output_loading_info = kwargs.pop('output_loading_info', False) - # Load config - if config is None: + # Load config if we don't provide a configuration + if not isinstance(config, PretrainedConfig): + config_path = config if config is not None else pretrained_model_name_or_path config, model_kwargs = cls.config_class.from_pretrained( - pretrained_model_name_or_path, *model_args, + config_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, + resume_download=resume_download, **kwargs ) else: @@ -244,14 +266,19 @@ class TFPreTrainedModel(tf.keras.Model): raise EnvironmentError("Error no file named {} found in directory {} or `from_pt` set to False".format( [WEIGHTS_NAME, TF2_WEIGHTS_NAME], pretrained_model_name_or_path)) - elif os.path.isfile(pretrained_model_name_or_path): + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path + elif os.path.isfile(pretrained_model_name_or_path + ".index"): + archive_file = pretrained_model_name_or_path + ".index" else: - raise EnvironmentError("Error file {} not found".format(pretrained_model_name_or_path)) + archive_file = hf_bucket_url(pretrained_model_name_or_path, postfix=TF2_WEIGHTS_NAME) + if from_pt: + raise EnvironmentError("Loading a TF model from a PyTorch checkpoint is not supported when using a model identifier name.") # redirect to the cache, if necessary try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, + resume_download=resume_download, proxies=proxies) except EnvironmentError as e: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: logger.error( @@ -279,17 +306,46 @@ class TFPreTrainedModel(tf.keras.Model): if from_pt: # Load from a PyTorch checkpoint - return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file) + return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, allow_missing_keys=True) ret = model(model.dummy_inputs, training=False) # build the network with dummy inputs assert os.path.isfile(resolved_archive_file), "Error retrieving file {}".format(resolved_archive_file) # 'by_name' allow us to do transfer learning by skipping/adding layers # see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1339-L1357 - model.load_weights(resolved_archive_file, by_name=True) + try: + model.load_weights(resolved_archive_file, by_name=True) + except OSError: + raise OSError("Unable to load weights from h5 file. " + "If you tried to load a TF 2.0 model from a PyTorch checkpoint, please set from_pt=True. ") ret = model(model.dummy_inputs, training=False) # Make sure restore ops are run + # Check if the models are the same to output loading informations + with h5py.File(resolved_archive_file, 'r') as f: + if 'layer_names' not in f.attrs and 'model_weights' in f: + f = f['model_weights'] + hdf5_layer_names = set(hdf5_format.load_attributes_from_hdf5_group(f, 'layer_names')) + model_layer_names = set(layer.name for layer in model.layers) + missing_keys = list(model_layer_names - hdf5_layer_names) + unexpected_keys = list(hdf5_layer_names - model_layer_names) + error_msgs = [] + + if len(missing_keys) > 0: + logger.info("Layers of {} not initialized from pretrained model: {}".format( + model.__class__.__name__, missing_keys)) + if len(unexpected_keys) > 0: + logger.info("Layers from pretrained model not used in {}: {}".format( + model.__class__.__name__, unexpected_keys)) + if len(error_msgs) > 0: + raise RuntimeError('Error(s) in loading weights for {}:\n\t{}'.format( + model.__class__.__name__, "\n\t".join(error_msgs))) + if output_loading_info: + loading_info = {"missing_keys": missing_keys, + "unexpected_keys": unexpected_keys, + "error_msgs": error_msgs} + return model, loading_info + return model class TFConv1D(tf.keras.layers.Layer): @@ -454,7 +510,7 @@ class TFSequenceSummary(tf.keras.layers.Layer): elif self.summary_type == 'first': output = hidden_states[:, 0] elif self.summary_type == 'mean': - output = tf.mean(hidden_states, axis=1) + output = tf.reduce_mean(hidden_states, axis=1) elif self.summary_type == 'cls_index': hidden_shape = shape_list(hidden_states) # e.g. [batch, num choices, seq length, hidden dims] if cls_index is None: diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py index 6f11b053..a7cc8ea4 100644 --- a/transformers/modeling_tf_xlm.py +++ b/transformers/modeling_tf_xlm.py @@ -460,7 +460,7 @@ class TFXLMPreTrainedModel(TFPreTrainedModel): langs_list = tf.constant([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) else: langs_list = None - return [inputs_list, attns_list, langs_list] + return {'input_ids': inputs_list, 'attention_mask': attns_list, 'langs': langs_list} XLM_START_DOCSTRING = r""" The XLM model was proposed in @@ -576,7 +576,7 @@ class TFXLMModel(TFXLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMModel.from_pretrained('xlm-mlm-en-2048') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -649,7 +649,7 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -695,7 +695,7 @@ class TFXLMForSequenceClassification(TFXLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMForSequenceClassification.from_pretrained('xlm-mlm-en-2048') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 labels = tf.constant([1])[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -743,7 +743,7 @@ class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMForQuestionAnsweringSimple.from_pretrained('xlm-mlm-en-2048') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) start_scores, end_scores = outputs[:2] diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py index 4733ea85..2f1fe150 100644 --- a/transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -112,8 +112,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): def prune_heads(self, heads): raise NotImplementedError - @staticmethod - def rel_shift(x, klen=-1): + def rel_shift(self, x, klen=-1): """perform relative shift to form the relative attention score.""" x_size = shape_list(x) @@ -135,7 +134,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): # position based attention score bd = tf.einsum('ibnd,jbnd->ijbn', q_head + self.r_r_bias, k_head_r) - bd = self.rel_shift(bd, klen=ac.shape[1]) + bd = self.rel_shift(bd, klen=shape_list(ac)[1]) # segment based attention score if seg_mat is None: @@ -192,7 +191,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): if g is not None: ###### Two-stream attention with relative positional encoding. # content based attention score - if mems is not None and mems.shape.ndims > 1: + if mems is not None and len(shape_list(mems)) > 1: cat = tf.concat([mems, h], axis=0) else: cat = h @@ -252,7 +251,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): else: ###### Multi-head attention with relative positional encoding - if mems is not None and mems.shape.ndims > 1: + if mems is not None and len(shape_list(mems)) > 1: cat = tf.concat([mems, h], axis=0) else: cat = h @@ -367,7 +366,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): self.use_bfloat16 = config.use_bfloat16 self.initializer_range = config.initializer_range - self.word_embedding = TFSharedEmbeddings(config.n_token, config.d_model, initializer_range=config.initializer_range, name='word_embedding') + self.word_embedding = TFSharedEmbeddings(config.vocab_size, config.d_model, initializer_range=config.initializer_range, name='word_embedding') self.layer = [TFXLNetLayer(config, name='layer_._{}'.format(i)) for i in range(config.n_layer)] self.dropout = tf.keras.layers.Dropout(config.dropout) @@ -553,7 +552,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): assert input_mask is None or attention_mask is None, "You can only use one of input_mask (uses 1 for padding) " \ "or attention_mask (uses 0 for padding, added for compatbility with BERT). Please choose one." if input_mask is None and attention_mask is not None: - input_mask = 1.0 - attention_mask + input_mask = 1.0 - tf.cast(attention_mask, dtype=dtype_float) if input_mask is not None and perm_mask is not None: data_mask = input_mask[None] + perm_mask elif input_mask is not None and perm_mask is None: @@ -565,7 +564,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): if data_mask is not None: # all mems can be attended to - mems_mask = tf.zeros([tf.shape(data_mask)[0], mlen, bsz], + mems_mask = tf.zeros([shape_list(data_mask)[0], mlen, bsz], dtype=dtype_float) data_mask = tf.concat([mems_mask, data_mask], axis=1) if attn_mask is None: @@ -590,7 +589,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): word_emb_k = self.word_embedding(input_ids) output_h = self.dropout(word_emb_k, training=training) if target_mapping is not None: - word_emb_q = tf.tile(self.mask_emb, [tf.shape(target_mapping)[0], bsz, 1]) + word_emb_q = tf.tile(self.mask_emb, [shape_list(target_mapping)[0], bsz, 1]) # else: # We removed the inp_q input which was same as target mapping # inp_q_ext = inp_q[:, :, None] # word_emb_q = inp_q_ext * self.mask_emb + (1 - inp_q_ext) * word_emb_k @@ -812,7 +811,7 @@ class TFXLNetModel(TFXLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = TFXLNetModel.from_pretrained('xlnet-large-cased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -856,7 +855,7 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel): model = TFXLNetLMHeadModel.from_pretrained('xlnet-large-cased') # We show how to setup inputs to predict a next token using a bi-directional context. - input_ids = tf.constant(tokenizer.encode("Hello, my dog is very "))[None, :] # We will predict the masked token + input_ids = tf.constant(tokenizer.encode("Hello, my dog is very ", add_special_tokens=True))[None, :] # We will predict the masked token perm_mask = tf.zeros((1, input_ids.shape[1], input_ids.shape[1])) perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token target_mapping = tf.zeros((1, 1, input_ids.shape[1])) # Shape [1, 1, seq_length] => let's predict one token @@ -912,7 +911,7 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = TFXLNetForSequenceClassification.from_pretrained('xlnet-large-cased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -939,6 +938,59 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): return outputs # return logits, (mems), (hidden states), (attentions) +@add_start_docstrings("""XLNet Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +class TFXLNetForTokenClassification(TFXLNetPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **mems**: (`optional`, returned when ``config.mem_len > 0``) + list of ``tf.Tensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. + See details in the docstring of the `mems` input above. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import XLNetTokenizer, TFXLNetForTokenClassification + + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') + model = TFXLNetForSequenceClassification.from_pretrained('xlnet-large-cased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + scores = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLNetForTokenClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.transformer = TFXLNetMainLayer(config, name='transformer') + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) + output = transformer_outputs[0] + + logits = self.classifier(output) + + outputs = (logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it + + return outputs # return logits, (mems), (hidden states), (attentions) + + # @add_start_docstrings("""XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of # the hidden-states output to compute `span start logits` and `span end logits`). """, # XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) @@ -970,7 +1022,7 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased') model = TFXLNetForQuestionAnsweringSimple.from_pretrained('xlnet-base-cased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) start_scores, end_scores = outputs[:2] @@ -1034,7 +1086,7 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): # tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') # model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') -# input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 +# input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 # start_positions = tf.constant([1]) # end_positions = tf.constant([3]) # outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) diff --git a/transformers/modeling_transfo_xl.py b/transformers/modeling_transfo_xl.py index a6a82f0d..45899900 100644 --- a/transformers/modeling_transfo_xl.py +++ b/transformers/modeling_transfo_xl.py @@ -582,7 +582,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel): tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TransfoXLModel.from_pretrained('transfo-xl-wt103') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states, mems = outputs[:2] @@ -592,14 +592,14 @@ class TransfoXLModel(TransfoXLPreTrainedModel): self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states - self.n_token = config.n_token + self.n_token = config.vocab_size self.d_embed = config.d_embed self.d_model = config.d_model self.n_head = config.n_head self.d_head = config.d_head - self.word_emb = AdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, + self.word_emb = AdaptiveEmbedding(config.vocab_size, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val) self.drop = nn.Dropout(config.dropout) @@ -825,7 +825,7 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) prediction_scores, mems = outputs[:2] @@ -836,11 +836,11 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): self.sample_softmax = config.sample_softmax # use sampled softmax if config.sample_softmax > 0: - self.out_layer = nn.Linear(config.d_model, config.n_token) - self.sampler = LogUniformSampler(config.n_token, config.sample_softmax) + self.out_layer = nn.Linear(config.d_model, config.vocab_size) + self.sampler = LogUniformSampler(config.vocab_size, config.sample_softmax) # use adaptive softmax (including standard softmax) else: - self.crit = ProjectedAdaptiveLogSoftmax(config.n_token, config.d_embed, config.d_model, + self.crit = ProjectedAdaptiveLogSoftmax(config.vocab_size, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val) self.init_weights() diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d51eefab..e3f4b9f3 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -31,11 +31,11 @@ from torch.nn import CrossEntropyLoss from torch.nn import functional as F from .configuration_utils import PretrainedConfig -from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME +from .file_utils import (TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, WEIGHTS_NAME, DUMMY_INPUTS, + cached_path, hf_bucket_url, is_remote_url) logger = logging.getLogger(__name__) - try: from torch.nn import Identity except ImportError: @@ -71,6 +71,15 @@ class PreTrainedModel(nn.Module): load_tf_weights = lambda model, config, path: None base_model_prefix = "" + @property + def dummy_inputs(self): + """ Dummy inputs to do a forward pass in the network. + + Returns: + torch.Tensor with dummy inputs + """ + return {'input_ids': torch.tensor(DUMMY_INPUTS)} + def __init__(self, config, *inputs, **kwargs): super(PreTrainedModel, self).__init__() if not isinstance(config, PretrainedConfig): @@ -160,8 +169,7 @@ class PreTrainedModel(nn.Module): base_model.vocab_size = new_num_tokens # Tie weights again if needed - if hasattr(self, 'tie_weights'): - self.tie_weights() + self.tie_weights() return model_embeds @@ -265,6 +273,7 @@ class PreTrainedModel(nn.Module): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. - None if you are both providing the configuration and state dictionary (resp. with keyword arguments ``config`` and ``state_dict``) @@ -272,7 +281,9 @@ class PreTrainedModel(nn.Module): model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + config: (`optional`) one of: + - an instance of a class derived from :class:`~transformers.PretrainedConfig`, or + - a string valid as input to :func:`~transformers.PretrainedConfig.from_pretrained()` Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or @@ -291,6 +302,9 @@ class PreTrainedModel(nn.Module): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -320,15 +334,18 @@ class PreTrainedModel(nn.Module): cache_dir = kwargs.pop('cache_dir', None) from_tf = kwargs.pop('from_tf', False) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) output_loading_info = kwargs.pop('output_loading_info', False) - # Load config - if config is None: + # Load config if we don't provide a configuration + if not isinstance(config, PretrainedConfig): + config_path = config if config is not None else pretrained_model_name_or_path config, model_kwargs = cls.config_class.from_pretrained( - pretrained_model_name_or_path, *model_args, + config_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, + resume_download=resume_download, proxies=proxies, **kwargs ) @@ -353,15 +370,21 @@ class PreTrainedModel(nn.Module): raise EnvironmentError("Error no file named {} found in directory {} or `from_tf` set to False".format( [WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME + ".index"], pretrained_model_name_or_path)) - elif os.path.isfile(pretrained_model_name_or_path): + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path - else: - assert from_tf, "Error finding file {}, no file or TF 1.X checkpoint found".format(pretrained_model_name_or_path) + elif os.path.isfile(pretrained_model_name_or_path + ".index"): + assert from_tf, "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format( + pretrained_model_name_or_path + ".index") archive_file = pretrained_model_name_or_path + ".index" + else: + archive_file = hf_bucket_url(pretrained_model_name_or_path, postfix=WEIGHTS_NAME) + if from_tf: + raise EnvironmentError("Loading a PyTorch model from a TF checkpoint is not supported when using a model identifier name.") # redirect to the cache, if necessary try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, + proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: msg = "Couldn't reach server at '{}' to download pretrained weights.".format( @@ -388,7 +411,11 @@ class PreTrainedModel(nn.Module): model = cls(config, *model_args, **model_kwargs) if state_dict is None and not from_tf: - state_dict = torch.load(resolved_archive_file, map_location='cpu') + try: + state_dict = torch.load(resolved_archive_file, map_location='cpu') + except: + raise OSError("Unable to load weights from pytorch checkpoint file. " + "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True. ") missing_keys = [] unexpected_keys = [] @@ -458,8 +485,7 @@ class PreTrainedModel(nn.Module): raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( model.__class__.__name__, "\n\t".join(error_msgs))) - if hasattr(model, 'tie_weights'): - model.tie_weights() # make sure word embedding weights are still tied + model.tie_weights() # make sure word embedding weights are still tied if needed # Set model in evaluation mode to desactivate DropOut modules by default model.eval() @@ -728,7 +754,7 @@ class SequenceSummary(nn.Module): def __init__(self, config): super(SequenceSummary, self).__init__() - self.summary_type = config.summary_type if hasattr(config, 'summary_use_proj') else 'last' + self.summary_type = config.summary_type if hasattr(config, 'summary_type') else 'last' if self.summary_type == 'attn': # We should use a standard multi-head attention module with absolute positional embedding for that. # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276 diff --git a/transformers/modeling_xlm.py b/transformers/modeling_xlm.py index 257f0da3..f39fe1d1 100644 --- a/transformers/modeling_xlm.py +++ b/transformers/modeling_xlm.py @@ -227,6 +227,16 @@ class XLMPreTrainedModel(PreTrainedModel): def __init__(self, *inputs, **kwargs): super(XLMPreTrainedModel, self).__init__(*inputs, **kwargs) + @property + def dummy_inputs(self): + inputs_list = torch.tensor([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) + attns_list = torch.tensor([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) + if self.config.use_lang_emb and self.config.n_langs > 1: + langs_list = torch.tensor([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) + else: + langs_list = None + return {'input_ids': inputs_list, 'attention_mask': attns_list, 'langs': langs_list} + def _init_weights(self, module): """ Initialize the weights. """ if isinstance(module, nn.Embedding): @@ -336,7 +346,7 @@ class XLMModel(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMModel.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -624,7 +634,7 @@ class XLMWithLMHeadModel(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -646,7 +656,7 @@ class XLMWithLMHeadModel(XLMPreTrainedModel): langs=langs, token_type_ids=token_type_ids, position_ids=position_ids, - lengths=lengths, + lengths=lengths, cache=cache, head_mask=head_mask, inputs_embeds=inputs_embeds) @@ -686,7 +696,7 @@ class XLMForSequenceClassification(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMForSequenceClassification.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -770,7 +780,7 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMForQuestionAnsweringSimple.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) @@ -866,7 +876,7 @@ class XLMForQuestionAnswering(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMForQuestionAnswering.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) diff --git a/transformers/modeling_xlm_roberta.py b/transformers/modeling_xlm_roberta.py new file mode 100644 index 00000000..0bdce941 --- /dev/null +++ b/transformers/modeling_xlm_roberta.py @@ -0,0 +1,298 @@ +# coding=utf-8 +# Copyright 2019 Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch XLM-RoBERTa model. """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice, RobertaForTokenClassification +from .configuration_xlm_roberta import XLMRobertaConfig +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'xlm-roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-base-pytorch_model.bin", + 'xlm-roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-pytorch_model.bin", + 'xlm-roberta-large-finetuned-conll02-dutch': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-dutch-pytorch_model.bin", + 'xlm-roberta-large-finetuned-conll02-spanish': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-spanish-pytorch_model.bin", + 'xlm-roberta-large-finetuned-conll03-english': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-english-pytorch_model.bin", + 'xlm-roberta-large-finetuned-conll03-german': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-german-pytorch_model.bin", +} + + +XLM_ROBERTA_START_DOCSTRING = r""" The XLM-RoBERTa model was proposed in + `Unsupervised Cross-lingual Representation Learning at Scale`_ + by Alexis Conneau, Kartikay Khandelwal, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. It is based on Facebook's RoBERTa model released in 2019. + + It is a large multi-lingual language model, trained on 2.5TB of filtered CommonCrawl data. + + This implementation is the same as RoBERTa. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`Unsupervised Cross-lingual Representation Learning at Scale`: + https://arxiv.org/abs/1911.02116 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.XLMRobertaConfig`): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +XLM_ROBERTA_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, XLM-RoBERTa input sequence should be formatted with and tokens as follows: + + (a) For sequence pairs: + + ``tokens: Is this Jacksonville ? No it is not . `` + + (b) For single sequences: + + ``tokens: the dog is hairy . `` + + Fully encoded sequences or sequence pairs can be obtained using the XLMRobertaTokenizer.encode function with + the ``add_special_tokens`` parameter set to ``True``. + + XLM-RoBERTa is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional` need to be trained) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Optional segment token indices to indicate first and second portions of the inputs. + This embedding matrice is not trained (not pretrained during XLM-RoBERTa pretraining), you will have to train it + during finetuning. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1[``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. +""" + +@add_start_docstrings("The bare XLM-RoBERTa Model transformer outputting raw hidden-states without any specific head on top.", + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaModel(RobertaModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + eo match pre-training, XLM-RoBERTa input sequence should be formatted with and tokens as follows: + + (a) For sequence pairs: + + ``tokens: is this jack ##son ##ville ? no it is not . `` + + ``token_type_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: the dog is hairy . `` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaModel.from_pretrained('xlm-roberta-large') + input_ids = torch.tensor(tokenizer.encode("Schloß Nymphenburg ist sehr schön .")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""XLM-RoBERTa Model with a `language modeling` head on top. """, + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaForMaskedLM(RobertaForMaskedLM): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaForMaskedLM.from_pretrained('xlm-roberta-large') + input_ids = torch.tensor(tokenizer.encode("Schloß Nymphenburg ist sehr schön .")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""XLM-RoBERTa Model transformer with a sequence classification/regression head on top (a linear layer + on top of the pooled output) e.g. for GLUE tasks. """, + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaForSequenceClassification(RobertaForSequenceClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaForSequenceClassification.from_pretrained('xlm-roberta-large') + input_ids = torch.tensor(tokenizer.encode("Schloß Nymphenburg ist sehr schön .")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""XLM-RoBERTa Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaForMultipleChoice(RobertaForMultipleChoice): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaForMultipleChoice.from_pretrained('xlm-roberta-large') + choices = ["Schloß Nymphenburg ist sehr schön .", "Der Schloßkanal auch !"] + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""XLM-RoBERTa Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaForTokenClassification(RobertaForTokenClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaForTokenClassification.from_pretrained('xlm-roberta-large') + input_ids = torch.tensor(tokenizer.encode("Schloß Nymphenburg ist sehr schön .", add_special_tokens=True)).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py index 658048a6..5a8df23c 100644 --- a/transformers/modeling_xlnet.py +++ b/transformers/modeling_xlnet.py @@ -583,12 +583,13 @@ class XLNetModel(XLNetPreTrainedModel): **attentions**: (`optional`, returned when ``config.output_attentions=True``) list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = XLNetModel.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -608,7 +609,7 @@ class XLNetModel(XLNetPreTrainedModel): self.clamp_len = config.clamp_len self.n_layer = config.n_layer - self.word_embedding = nn.Embedding(config.n_token, config.d_model) + self.word_embedding = nn.Embedding(config.vocab_size, config.d_model) self.mask_emb = nn.Parameter(torch.FloatTensor(1, 1, config.d_model)) self.layer = nn.ModuleList([XLNetLayer(config) for _ in range(config.n_layer)]) self.dropout = nn.Dropout(config.dropout) @@ -878,7 +879,11 @@ class XLNetModel(XLNetPreTrainedModel): hidden_states = tuple(hs.permute(1, 0, 2).contiguous() for hs in hidden_states) outputs = outputs + (hidden_states,) if self.output_attentions: - attentions = tuple(t.permute(2, 3, 0, 1).contiguous() for t in attentions) + if target_mapping is not None: + # when target_mapping is provided, there are 2-tuple of attentions + attentions = tuple(tuple(att_stream.permute(2, 3, 0, 1).contiguous() for att_stream in t) for t in attentions) + else: + attentions = tuple(t.permute(2, 3, 0, 1).contiguous() for t in attentions) outputs = outputs + (attentions,) return outputs # outputs, (new_mems), (hidden_states), (attentions) @@ -913,13 +918,14 @@ class XLNetLMHeadModel(XLNetPreTrainedModel): **attentions**: (`optional`, returned when ``config.output_attentions=True``) list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = XLNetLMHeadModel.from_pretrained('xlnet-large-cased') # We show how to setup inputs to predict a next token using a bi-directional context. - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ")).unsqueeze(0) # We will predict the masked token + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ", add_special_tokens=True)).unsqueeze(0) # We will predict the masked token perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token @@ -934,7 +940,7 @@ class XLNetLMHeadModel(XLNetPreTrainedModel): self.same_length = config.same_length self.transformer = XLNetModel(config) - self.lm_loss = nn.Linear(config.d_model, config.n_token, bias=True) + self.lm_loss = nn.Linear(config.d_model, config.vocab_size, bias=True) self.init_weights() @@ -995,12 +1001,13 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): **attentions**: (`optional`, returned when ``config.output_attentions=True``) list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = XLNetForSequenceClassification.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -1046,6 +1053,106 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): return outputs # return (loss), logits, (mems), (hidden states), (attentions) +@add_start_docstrings("""XLNet Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + XLNET_START_DOCSTRING, + XLNET_INPUTS_DOCSTRING) +class XLNetForTokenClassification(XLNetPreTrainedModel): + r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + The second dimension of the input (`num_choices`) indicates the number of choices to scores. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **mems**: (`optional`, returned when ``config.mem_len > 0``) + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. + See details in the docstring of the `mems` input above. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') + model = XLNetForSequenceClassification.from_pretrained('xlnet-large-cased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + scores = outputs[0] + + """ + def __init__(self, config): + super(XLNetForTokenClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.transformer = XLNetModel(config) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, + token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, labels=None): + + outputs = self.transformer(input_ids, + attention_mask=attention_mask, + mems=mems, + perm_mask=perm_mask, + target_mapping=target_mapping, + token_type_ids=token_type_ids, + input_mask=input_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds) + + sequence_output = outputs[0] + + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[1:] # Keep mems, hidden states, attentions if there are in it + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels)[active_loss] + active_labels = labels.view(-1)[active_loss] + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # return (loss), logits, (mems), (hidden states), (attentions) + + @add_start_docstrings("""XLNet Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RACE/SWAG tasks. """, XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) @@ -1095,6 +1202,7 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel): **attentions**: (`optional`, returned when ``config.output_attentions=True``) list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: @@ -1180,12 +1288,13 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): **attentions**: (`optional`, returned when ``config.output_attentions=True``) list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) @@ -1294,12 +1403,13 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): **attentions**: (`optional`, returned when ``config.output_attentions=True``) list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) diff --git a/transformers/optimization_tf.py b/transformers/optimization_tf.py new file mode 100644 index 00000000..c5fa2480 --- /dev/null +++ b/transformers/optimization_tf.py @@ -0,0 +1,254 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions and classes related to optimization (weight updates).""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re + +import tensorflow as tf + + +class WarmUp(tf.keras.optimizers.schedules.LearningRateSchedule): + """Applys a warmup schedule on a given learning rate decay schedule.""" + + def __init__( + self, + initial_learning_rate, + decay_schedule_fn, + warmup_steps, + power=1.0, + name=None): + super(WarmUp, self).__init__() + self.initial_learning_rate = initial_learning_rate + self.warmup_steps = warmup_steps + self.power = power + self.decay_schedule_fn = decay_schedule_fn + self.name = name + + def __call__(self, step): + with tf.name_scope(self.name or 'WarmUp') as name: + # Implements polynomial warmup. i.e., if global_step < warmup_steps, the + # learning rate will be `global_step/num_warmup_steps * init_lr`. + global_step_float = tf.cast(step, tf.float32) + warmup_steps_float = tf.cast(self.warmup_steps, tf.float32) + warmup_percent_done = global_step_float / warmup_steps_float + warmup_learning_rate = ( + self.initial_learning_rate * + tf.math.pow(warmup_percent_done, self.power)) + return tf.cond(global_step_float < warmup_steps_float, + lambda: warmup_learning_rate, + lambda: self.decay_schedule_fn(step), + name=name) + + def get_config(self): + return { + 'initial_learning_rate': self.initial_learning_rate, + 'decay_schedule_fn': self.decay_schedule_fn, + 'warmup_steps': self.warmup_steps, + 'power': self.power, + 'name': self.name + } + + +def create_optimizer(init_lr, num_train_steps, num_warmup_steps): + """Creates an optimizer with learning rate schedule.""" + # Implements linear decay of the learning rate. + learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay( + initial_learning_rate=init_lr, + decay_steps=num_train_steps, + end_learning_rate=0.0) + if num_warmup_steps: + learning_rate_fn = WarmUp(initial_learning_rate=init_lr, + decay_schedule_fn=learning_rate_fn, + warmup_steps=num_warmup_steps) + optimizer = AdamWeightDecay( + learning_rate=learning_rate_fn, + weight_decay_rate=0.01, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=['layer_norm', 'bias']) + return optimizer + + +class AdamWeightDecay(tf.keras.optimizers.Adam): + """Adam enables L2 weight decay and clip_by_global_norm on gradients. + + Just adding the square of the weights to the loss function is *not* the + correct way of using L2 regularization/weight decay with Adam, since that will + interact with the m and v parameters in strange ways. + + Instead we want ot decay the weights in a manner that doesn't interact with + the m/v parameters. This is equivalent to adding the square of the weights to + the loss with plain (non-momentum) SGD. + """ + + def __init__(self, + learning_rate=0.001, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-7, + amsgrad=False, + weight_decay_rate=0.0, + include_in_weight_decay=None, + exclude_from_weight_decay=None, + name='AdamWeightDecay', + **kwargs): + super(AdamWeightDecay, self).__init__( + learning_rate, beta_1, beta_2, epsilon, amsgrad, name, **kwargs) + self.weight_decay_rate = weight_decay_rate + self._include_in_weight_decay = include_in_weight_decay + self._exclude_from_weight_decay = exclude_from_weight_decay + + @classmethod + def from_config(cls, config): + """Creates an optimizer from its config with WarmUp custom object.""" + custom_objects = {'WarmUp': WarmUp} + return super(AdamWeightDecay, cls).from_config( + config, custom_objects=custom_objects) + + def _prepare_local(self, var_device, var_dtype, apply_state): + super(AdamWeightDecay, self)._prepare_local(var_device, var_dtype, + apply_state) + apply_state['weight_decay_rate'] = tf.constant( + self.weight_decay_rate, name='adam_weight_decay_rate') + + def _decay_weights_op(self, var, learning_rate, apply_state): + do_decay = self._do_use_weight_decay(var.name) + if do_decay: + return var.assign_sub( + learning_rate * var * + apply_state['weight_decay_rate'], + use_locking=self._use_locking) + return tf.no_op() + + def apply_gradients(self, grads_and_vars, clip_norm, name=None): + grads, tvars = list(zip(*grads_and_vars)) + (grads, _) = tf.clip_by_global_norm(grads, clip_norm=clip_norm) + return super(AdamWeightDecay, self).apply_gradients(zip(grads, tvars)) + + def _get_lr(self, var_device, var_dtype, apply_state): + """Retrieves the learning rate with the given state.""" + if apply_state is None: + return self._decayed_lr_t[var_dtype], {} + + apply_state = apply_state or {} + coefficients = apply_state.get((var_device, var_dtype)) + if coefficients is None: + coefficients = self._fallback_apply_state(var_device, var_dtype) + apply_state[(var_device, var_dtype)] = coefficients + + return coefficients['lr_t'], dict(apply_state=apply_state) + + def _resource_apply_dense(self, grad, var, apply_state=None): + lr_t, kwargs = self._get_lr(var.device, var.dtype.base_dtype, apply_state) + decay = self._decay_weights_op(var, lr_t, apply_state) + with tf.control_dependencies([decay]): + return super(AdamWeightDecay, self)._resource_apply_dense( + grad, var, **kwargs) + + def _resource_apply_sparse(self, grad, var, indices, apply_state=None): + lr_t, kwargs = self._get_lr(var.device, var.dtype.base_dtype, apply_state) + decay = self._decay_weights_op(var, lr_t, apply_state) + with tf.control_dependencies([decay]): + return super(AdamWeightDecay, self)._resource_apply_sparse( + grad, var, indices, **kwargs) + + def get_config(self): + config = super(AdamWeightDecay, self).get_config() + config.update({ + 'weight_decay_rate': self.weight_decay_rate, + }) + return config + + def _do_use_weight_decay(self, param_name): + """Whether to use L2 weight decay for `param_name`.""" + if self.weight_decay_rate == 0: + return False + + if self._include_in_weight_decay: + for r in self._include_in_weight_decay: + if re.search(r, param_name) is not None: + return True + + if self._exclude_from_weight_decay: + for r in self._exclude_from_weight_decay: + if re.search(r, param_name) is not None: + return False + return True + + +## Inspired from https://github.com/OpenNMT/OpenNMT-tf/blob/master/opennmt/optimizers/utils.py +class GradientAccumulator(object): + """Distribution strategies-aware gradient accumulation utility.""" + + def __init__(self): + """Initializes the accumulator.""" + self._gradients = [] + self._accum_steps = tf.Variable( + initial_value=0, + dtype=tf.int64, + trainable=False, + aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA) + + @property + def step(self): + """Number of accumulated steps.""" + return self._accum_steps.value() + + @property + def gradients(self): + """The accumulated gradients.""" + return list(gradient.value() if gradient is not None else gradient for gradient in self._get_replica_gradients()) + + def __call__(self, gradients): + """Accumulates :obj:`gradients`.""" + if not self._gradients: + self._gradients.extend([tf.Variable(tf.zeros_like(gradient), trainable=False) if gradient is not None else gradient for gradient in gradients]) + + if len(gradients) != len(self._gradients): + raise ValueError("Expected %s gradients, but got %d" % (len(self._gradients), len(gradients))) + + for accum_gradient, gradient in zip(self._get_replica_gradients(), gradients): + if accum_gradient is not None: + accum_gradient.assign_add(gradient) + + self._accum_steps.assign_add(1) + + def reset(self): + """Resets the accumulated gradients.""" + if self._gradients: + self._accum_steps.assign(0) + + for gradient in self._get_replica_gradients(): + if gradient is not None: + gradient.assign(tf.zeros_like(gradient)) + + def _get_replica_gradients(self): + if tf.distribute.has_strategy(): + # In a replica context, we want to accumulate gradients on each replica + # without synchronization, so we directly assign the value of the + # current replica. + replica_context = tf.distribute.get_replica_context() + + if replica_context is None or tf.distribute.get_strategy().num_replicas_in_sync == 1: + return self._gradients + + return (gradient.device_map.select_for_current_replica(gradient.values, replica_context) for gradient in self._gradients) + else: + return self._gradients diff --git a/transformers/pipelines.py b/transformers/pipelines.py new file mode 100755 index 00000000..f4bf3da6 --- /dev/null +++ b/transformers/pipelines.py @@ -0,0 +1,907 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import sys +import csv +import json +import os +import pickle +import logging +import six + +from abc import ABC, abstractmethod +from contextlib import contextmanager +from itertools import groupby +from os.path import abspath, exists +from typing import Union, Optional, Tuple, List, Dict + +import numpy as np + +from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer, + PretrainedConfig, ModelCard, SquadExample, + squad_convert_examples_to_features, is_tf_available, + is_torch_available, BasicTokenizer, + ALL_PRETRAINED_CONFIG_ARCHIVE_MAP) + +if is_tf_available(): + import tensorflow as tf + from transformers import TFAutoModel, TFAutoModelForSequenceClassification, \ + TFAutoModelForQuestionAnswering, TFAutoModelForTokenClassification + +if is_torch_available(): + import torch + from transformers import AutoModel, AutoModelForSequenceClassification, \ + AutoModelForQuestionAnswering, AutoModelForTokenClassification + + +logger = logging.getLogger(__name__) + +def get_framework(model=None): + """ Select framework (TensorFlow/PyTorch) to use. + If both frameworks are installed and no specific model is provided, defaults to using PyTorch. + """ + if is_tf_available() and is_torch_available() and model is not None and not isinstance(model, str): + # Both framework are available but the use supplied a model class instance. + # Try to guess which framework to use from the model classname + framework = 'tf' if model.__class__.__name__.startswith('TF') else 'pt' + elif not is_tf_available() and not is_torch_available(): + raise ImportError("At least one of TensorFlow 2.0 or PyTorch should be installed. " + "To install TensorFlow 2.0, read the instructions at https://www.tensorflow.org/install/ " + "To install PyTorch, read the instructions at https://pytorch.org/.") + else: + # framework = 'tf' if is_tf_available() else 'pt' + framework = 'pt' if is_torch_available() else 'tf' + return framework + +class ArgumentHandler(ABC): + """ + Base interface for handling varargs for each Pipeline + """ + @abstractmethod + def __call__(self, *args, **kwargs): + raise NotImplementedError() + + +class DefaultArgumentHandler(ArgumentHandler): + """ + Default varargs argument parser handling parameters for each Pipeline + """ + def __call__(self, *args, **kwargs): + if 'X' in kwargs: + return kwargs['X'] + elif 'data' in kwargs: + return kwargs['data'] + elif len(args) == 1: + if isinstance(args[0], list): + return args[0] + else: + return [args[0]] + elif len(args) > 1: + return list(args) + raise ValueError('Unable to infer the format of the provided data (X=, data=, ...)') + + +class PipelineDataFormat: + """ + Base class for all the pipeline supported data format both for reading and writing. + Supported data formats currently includes: + - JSON + - CSV + - stdin/stdout (pipe) + + PipelineDataFormat also includes some utilities to work with multi-columns like mapping from datasets columns + to pipelines keyword arguments through the `dataset_kwarg_1=dataset_column_1` format. + """ + SUPPORTED_FORMATS = ['json', 'csv', 'pipe'] + + def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False): + self.output_path = output_path + self.input_path = input_path + self.column = column.split(',') if column is not None else [''] + self.is_multi_columns = len(self.column) > 1 + + if self.is_multi_columns: + self.column = [tuple(c.split('=')) if '=' in c else (c, c) for c in self.column] + + if output_path is not None and not overwrite: + if exists(abspath(self.output_path)): + raise OSError('{} already exists on disk'.format(self.output_path)) + + if input_path is not None: + if not exists(abspath(self.input_path)): + raise OSError('{} doesnt exist on disk'.format(self.input_path)) + + @abstractmethod + def __iter__(self): + raise NotImplementedError() + + @abstractmethod + def save(self, data: dict): + """ + Save the provided data object with the representation for the current `DataFormat`. + :param data: data to store + :return: + """ + raise NotImplementedError() + + def save_binary(self, data: Union[dict, List[dict]]) -> str: + """ + Save the provided data object as a pickle-formatted binary data on the disk. + :param data: data to store + :return: (str) Path where the data has been saved + """ + path, _ = os.path.splitext(self.output_path) + binary_path = os.path.extsep.join((path, 'pickle')) + + with open(binary_path, 'wb+') as f_output: + pickle.dump(data, f_output) + + return binary_path + + @staticmethod + def from_str(format: str, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False): + if format == 'json': + return JsonPipelineDataFormat(output_path, input_path, column, overwrite=overwrite) + elif format == 'csv': + return CsvPipelineDataFormat(output_path, input_path, column, overwrite=overwrite) + elif format == 'pipe': + return PipedPipelineDataFormat(output_path, input_path, column, overwrite=overwrite) + else: + raise KeyError('Unknown reader {} (Available reader are json/csv/pipe)'.format(format)) + + +class CsvPipelineDataFormat(PipelineDataFormat): + def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False): + super().__init__(output_path, input_path, column, overwrite=overwrite) + + def __iter__(self): + with open(self.input_path, 'r') as f: + reader = csv.DictReader(f) + for row in reader: + if self.is_multi_columns: + yield {k: row[c] for k, c in self.column} + else: + yield row[self.column[0]] + + def save(self, data: List[dict]): + with open(self.output_path, 'w') as f: + if len(data) > 0: + writer = csv.DictWriter(f, list(data[0].keys())) + writer.writeheader() + writer.writerows(data) + + +class JsonPipelineDataFormat(PipelineDataFormat): + def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False): + super().__init__(output_path, input_path, column, overwrite=overwrite) + + with open(input_path, 'r') as f: + self._entries = json.load(f) + + def __iter__(self): + for entry in self._entries: + if self.is_multi_columns: + yield {k: entry[c] for k, c in self.column} + else: + yield entry[self.column[0]] + + def save(self, data: dict): + with open(self.output_path, 'w') as f: + json.dump(data, f) + + +class PipedPipelineDataFormat(PipelineDataFormat): + """ + Read data from piped input to the python process. + For multi columns data, columns should separated by \t + + If columns are provided, then the output will be a dictionary with {column_x: value_x} + """ + def __iter__(self): + for line in sys.stdin: + # Split for multi-columns + if '\t' in line: + + line = line.split('\t') + if self.column: + # Dictionary to map arguments + yield {kwargs: l for (kwargs, _), l in zip(self.column, line)} + else: + yield tuple(line) + + # No dictionary to map arguments + else: + yield line + + def save(self, data: dict): + print(data) + + def save_binary(self, data: Union[dict, List[dict]]) -> str: + if self.output_path is None: + raise KeyError( + 'When using piped input on pipeline outputting large object requires an output file path. ' + 'Please provide such output path through --output argument.' + ) + + return super().save_binary(data) + + +class _ScikitCompat(ABC): + """ + Interface layer for the Scikit and Keras compatibility. + """ + + @abstractmethod + def transform(self, X): + raise NotImplementedError() + + @abstractmethod + def predict(self, X): + raise NotImplementedError() + + +class Pipeline(_ScikitCompat): + """ + Base class implementing pipelined operations. + Pipeline workflow is defined as a sequence of the following operations: + Input -> Tokenization -> Model Inference -> Post-Processing (Task dependent) -> Output + + Pipeline supports running on CPU or GPU through the device argument. Users can specify + device argument as an integer, -1 meaning "CPU", >= 0 referring the CUDA device ordinal. + + Some pipeline, like for instance FeatureExtractionPipeline ('feature-extraction') outputs large + tensor object as nested-lists. In order to avoid dumping such large structure as textual data we + provide the binary_output constructor argument. If set to True, the output will be stored in the + pickle format. + + Arguments: + **model**: ``(str, PretrainedModel, TFPretrainedModel)``: + Reference to the model to use through this pipeline. + + **tokenizer**: ``(str, PreTrainedTokenizer)``: + Reference to the tokenizer to use through this pipeline. + + **args_parser**: ``ArgumentHandler``: + Reference to the object in charge of parsing supplied pipeline parameters. + + **device**: ``int``: + Device ordinal for CPU/GPU supports. Setting this to -1 will leverage CPU, >=0 will run the model + on the associated CUDA device id. + + **binary_output** ``bool`` (default: False): + Flag indicating if the output the pipeline should happen in a binary format (i.e. pickle) or as raw text. + + Return: + Pipeline returns list or dictionary depending on: + - Does the user provided multiple sample + - The pipeline expose multiple fields in the output object + + Examples: + nlp = pipeline('ner') + nlp = pipeline('ner', model='...', config='...', tokenizer='...') + nlp = NerPipeline(model='...', config='...', tokenizer='...') + nlp = QuestionAnsweringPipeline(model=AutoModel.from_pretrained('...'), tokenizer='...') + """ + + default_input_names = None + + def __init__(self, model, tokenizer: PreTrainedTokenizer = None, + modelcard: ModelCard = None, framework: Optional[str] = None, + args_parser: ArgumentHandler = None, device: int = -1, + binary_output: bool = False): + + if framework is None: + framework = get_framework() + + self.model = model + self.tokenizer = tokenizer + self.modelcard = modelcard + self.framework = framework + self.device = device + self.binary_output = binary_output + self._args_parser = args_parser or DefaultArgumentHandler() + + # Special handling + if self.device >= 0 and self.framework == 'pt': + self.model = self.model.to('cuda:{}'.format(self.device)) + + def save_pretrained(self, save_directory): + """ + Save the pipeline's model and tokenizer to the specified save_directory + """ + if not os.path.isdir(save_directory): + logger.error("Provided path ({}) should be a directory".format(save_directory)) + return + + self.model.save_pretrained(save_directory) + self.tokenizer.save_pretrained(save_directory) + self.modelcard.save_pretrained(save_directory) + + def transform(self, X): + """ + Scikit / Keras interface to transformers' pipelines. This method will forward to __call__(). + """ + return self(X=X) + + def predict(self, X): + """ + Scikit / Keras interface to transformers' pipelines. This method will forward to __call__(). + Se + """ + return self(X=X) + + @contextmanager + def device_placement(self): + """ + Context Manager allowing tensor allocation on the user-specified device in framework agnostic way. + example: + # Explicitly ask for tensor allocation on CUDA device :0 + nlp = pipeline(..., device=0) + with nlp.device_placement(): + # Every framework specific tensor allocation will be done on the request device + output = nlp(...) + Returns: + Context manager + """ + if self.framework == 'tf': + with tf.device('/CPU:0' if self.device == -1 else '/device:GPU:{}'.format(self.device)): + yield + else: + if self.device >= 0: + torch.cuda.set_device(self.device) + + yield + + def inputs_for_model(self, features: Union[dict, List[dict]]) -> Dict: + """ + Generates the input dictionary with model-specific parameters. + + Returns: + dict holding all the required parameters for model's forward + """ + args = ['input_ids', 'attention_mask'] + model_type = type(self.model).__name__.lower() + + if 'distilbert' not in model_type and 'xlm' not in model_type: + args += ['token_type_ids'] + + # PR #1548 (CLI) There is an issue with attention_mask + # if 'xlnet' in model_type or 'xlm' in model_type: + # args += ['cls_index', 'p_mask'] + + if isinstance(features, dict): + return {k: features[k] for k in args} + else: + return {k: [feature[k] for feature in features] for k in args} + + def __call__(self, *texts, **kwargs): + # Parse arguments + inputs = self._args_parser(*texts, **kwargs) + + # Encode for forward + with self.device_placement(): + inputs = self.tokenizer.batch_encode_plus( + inputs, add_special_tokens=True, + return_tensors=self.framework, + max_length=self.tokenizer.max_len + ) + + # Filter out features not available on specific models + inputs = self.inputs_for_model(inputs) + return self._forward(inputs) + + def _forward(self, inputs): + """ + Internal framework specific forward dispatching. + Args: + inputs: dict holding all the keyworded arguments for required by the model forward method. + Returns: + Numpy array + """ + if self.framework == 'tf': + # TODO trace model + predictions = self.model(inputs, training=False)[0] + else: + with torch.no_grad(): + predictions = self.model(**inputs)[0].cpu() + + return predictions.numpy() + + +class FeatureExtractionPipeline(Pipeline): + """ + Feature extraction pipeline using Model head. + """ + + def __init__(self, model, + tokenizer: PreTrainedTokenizer = None, + modelcard: ModelCard = None, + framework: Optional[str] = None, + args_parser: ArgumentHandler = None, + device: int = -1): + super().__init__(model=model, + tokenizer=tokenizer, + modelcard=modelcard, + framework=framework, + args_parser=args_parser, + device=device, + binary_output=True) + + def __call__(self, *args, **kwargs): + return super().__call__(*args, **kwargs).tolist() + + +class TextClassificationPipeline(Pipeline): + """ + Text classification pipeline using ModelForTextClassification head. + """ + + def __call__(self, *args, **kwargs): + outputs = super().__call__(*args, **kwargs) + scores = np.exp(outputs) / np.exp(outputs).sum(-1) + return [{'label': self.model.config.id2label[item.argmax()], 'score': item.max()} for item in scores] + + +class NerPipeline(Pipeline): + """ + Named Entity Recognition pipeline using ModelForTokenClassification head. + """ + + default_input_names = 'sequences' + + def __init__(self, model, tokenizer: PreTrainedTokenizer = None, + modelcard: ModelCard = None, framework: Optional[str] = None, + args_parser: ArgumentHandler = None, device: int = -1, + binary_output: bool = False, ignore_labels=['O']): + super().__init__(model=model, + tokenizer=tokenizer, + modelcard=modelcard, + framework=framework, + args_parser=args_parser, + device=device, + binary_output=binary_output) + + self._basic_tokenizer = BasicTokenizer(do_lower_case=False) + self.ignore_labels = ignore_labels + + def __call__(self, *texts, **kwargs): + inputs, answers = self._args_parser(*texts, **kwargs), [] + for sentence in inputs: + + # Manage correct placement of the tensors + with self.device_placement(): + + tokens = self.tokenizer.encode_plus( + sentence, return_attention_mask=False, + return_tensors=self.framework, + max_length=self.tokenizer.max_len + ) + + # Forward + if self.framework == 'tf': + entities = self.model(tokens)[0][0].numpy() + input_ids = tokens['input_ids'].numpy()[0] + else: + with torch.no_grad(): + entities = self.model(**tokens)[0][0].cpu().numpy() + input_ids = tokens['input_ids'].cpu().numpy()[0] + + score = np.exp(entities) / np.exp(entities).sum(-1, keepdims=True) + labels_idx = score.argmax(axis=-1) + + answer = [] + for idx, label_idx in enumerate(labels_idx): + if self.model.config.id2label[label_idx] not in self.ignore_labels: + answer += [{ + 'word': self.tokenizer.decode([int(input_ids[idx])]), + 'score': score[idx][label_idx].item(), + 'entity': self.model.config.id2label[label_idx] + }] + + # Append + answers += [answer] + if len(answers) == 1: + return answers[0] + return answers + + +class QuestionAnsweringArgumentHandler(ArgumentHandler): + """ + QuestionAnsweringPipeline requires the user to provide multiple arguments (i.e. question & context) to be mapped + to internal SquadExample / SquadFeature structures. + + QuestionAnsweringArgumentHandler manages all the possible to create SquadExample from the command-line supplied + arguments. + """ + def __call__(self, *args, **kwargs): + # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating + if args is not None and len(args) > 0: + if len(args) == 1: + kwargs['X'] = args[0] + else: + kwargs['X'] = list(args) + + # Generic compatibility with sklearn and Keras + # Batched data + if 'X' in kwargs or 'data' in kwargs: + inputs = kwargs['X'] if 'X' in kwargs else kwargs['data'] + + if isinstance(inputs, dict): + inputs = [inputs] + else: + # Copy to avoid overriding arguments + inputs = [i for i in inputs] + + for i, item in enumerate(inputs): + if isinstance(item, dict): + if any(k not in item for k in ['question', 'context']): + raise KeyError('You need to provide a dictionary with keys {question:..., context:...}') + + inputs[i] = QuestionAnsweringPipeline.create_sample(**item) + + elif not isinstance(item, SquadExample): + raise ValueError( + '{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)' + .format('X' if 'X' in kwargs else 'data') + ) + + # Tabular input + elif 'question' in kwargs and 'context' in kwargs: + if isinstance(kwargs['question'], str): + kwargs['question'] = [kwargs['question']] + + if isinstance(kwargs['context'], str): + kwargs['context'] = [kwargs['context']] + + inputs = [QuestionAnsweringPipeline.create_sample(q, c) for q, c in zip(kwargs['question'], kwargs['context'])] + else: + raise ValueError('Unknown arguments {}'.format(kwargs)) + + if not isinstance(inputs, list): + inputs = [inputs] + + return inputs + + +class QuestionAnsweringPipeline(Pipeline): + """ + Question Answering pipeline using ModelForQuestionAnswering head. + """ + + default_input_names = 'question,context' + + def __init__(self, model, + tokenizer: Optional[PreTrainedTokenizer], + modelcard: Optional[ModelCard], + framework: Optional[str] = None, + device: int = -1, **kwargs): + super().__init__(model=model, + tokenizer=tokenizer, + modelcard=modelcard, + framework=framework, + args_parser=QuestionAnsweringArgumentHandler(), + device=device, **kwargs) + + @staticmethod + def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[SquadExample, List[SquadExample]]: + """ + QuestionAnsweringPipeline leverages the SquadExample/SquadFeatures internally. + This helper method encapsulate all the logic for converting question(s) and context(s) to SquadExample(s). + We currently support extractive question answering. + Arguments: + question: (str, List[str]) The question to be ask for the associated context + context: (str, List[str]) The context in which we will look for the answer. + + Returns: + SquadExample initialized with the corresponding question and context. + """ + if isinstance(question, list): + return [SquadExample(None, q, c, None, None, None) for q, c in zip(question, context)] + else: + return SquadExample(None, question, context, None, None, None) + + def __call__(self, *texts, **kwargs): + """ + Args: + We support multiple use-cases, the following are exclusive: + X: sequence of SquadExample + data: sequence of SquadExample + question: (str, List[str]), batch of question(s) to map along with context + context: (str, List[str]), batch of context(s) associated with the provided question keyword argument + Returns: + dict: {'answer': str, 'score": float, 'start": int, "end": int} + answer: the textual answer in the intial context + score: the score the current answer scored for the model + start: the character index in the original string corresponding to the beginning of the answer' span + end: the character index in the original string corresponding to the ending of the answer' span + """ + # Set defaults values + kwargs.setdefault('topk', 1) + kwargs.setdefault('doc_stride', 128) + kwargs.setdefault('max_answer_len', 15) + kwargs.setdefault('max_seq_len', 384) + kwargs.setdefault('max_question_len', 64) + + if kwargs['topk'] < 1: + raise ValueError('topk parameter should be >= 1 (got {})'.format(kwargs['topk'])) + + if kwargs['max_answer_len'] < 1: + raise ValueError('max_answer_len parameter should be >= 1 (got {})'.format(kwargs['max_answer_len'])) + + # Convert inputs to features + examples = self._args_parser(*texts, **kwargs) + features = squad_convert_examples_to_features(examples, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) + fw_args = self.inputs_for_model([f.__dict__ for f in features]) + + # Manage tensor allocation on correct device + with self.device_placement(): + if self.framework == 'tf': + fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()} + start, end = self.model(fw_args) + start, end = start.numpy(), end.numpy() + else: + with torch.no_grad(): + # Retrieve the score for the context tokens only (removing question tokens) + fw_args = {k: torch.tensor(v) for (k, v) in fw_args.items()} + start, end = self.model(**fw_args) + start, end = start.cpu().numpy(), end.cpu().numpy() + + answers = [] + for (example, feature, start_, end_) in zip(examples, features, start, end): + # Normalize logits and spans to retrieve the answer + start_ = np.exp(start_) / np.sum(np.exp(start_)) + end_ = np.exp(end_) / np.sum(np.exp(end_)) + + # Mask padding and question + start_, end_ = start_ * np.abs(np.array(feature.p_mask) - 1), end_ * np.abs(np.array(feature.p_mask) - 1) + + # TODO : What happens if not possible + # Mask CLS + start_[0] = end_[0] = 0 + + starts, ends, scores = self.decode(start_, end_, kwargs['topk'], kwargs['max_answer_len']) + char_to_word = np.array(example.char_to_word_offset) + + # Convert the answer (tokens) back to the original text + answers += [ + { + 'score': score.item(), + 'start': np.where(char_to_word == feature.token_to_orig_map[s])[0][0].item(), + 'end': np.where(char_to_word == feature.token_to_orig_map[e])[0][-1].item(), + 'answer': ' '.join(example.doc_tokens[feature.token_to_orig_map[s]:feature.token_to_orig_map[e] + 1]) + } + for s, e, score in zip(starts, ends, scores) + ] + if len(answers) == 1: + return answers[0] + return answers + + def decode(self, start: np.ndarray, end: np.ndarray, topk: int, max_answer_len: int) -> Tuple: + """ + Take the output of any QuestionAnswering head and will generate probalities for each span to be + the actual answer. + In addition, it filters out some unwanted/impossible cases like answer len being greater than + max_answer_len or answer end position being before the starting position. + The method supports output the k-best answer through the topk argument. + + Args: + start: numpy array, holding individual start probabilities for each token + end: numpy array, holding individual end probabilities for each token + topk: int, indicates how many possible answer span(s) to extract from the model's output + max_answer_len: int, maximum size of the answer to extract from the model's output + """ + # Ensure we have batch axis + if start.ndim == 1: + start = start[None] + + if end.ndim == 1: + end = end[None] + + # Compute the score of each tuple(start, end) to be the real answer + outer = np.matmul(np.expand_dims(start, -1), np.expand_dims(end, 1)) + + # Remove candidate with end < start and end - start > max_answer_len + candidates = np.tril(np.triu(outer), max_answer_len - 1) + + # Inspired by Chen & al. (https://github.com/facebookresearch/DrQA) + scores_flat = candidates.flatten() + if topk == 1: + idx_sort = [np.argmax(scores_flat)] + elif len(scores_flat) < topk: + idx_sort = np.argsort(-scores_flat) + else: + idx = np.argpartition(-scores_flat, topk)[0:topk] + idx_sort = idx[np.argsort(-scores_flat[idx])] + + start, end = np.unravel_index(idx_sort, candidates.shape)[1:] + return start, end, candidates[0, start, end] + + def span_to_answer(self, text: str, start: int, end: int): + """ + When decoding from token probalities, this method maps token indexes to actual word in + the initial context. + + Args: + text: str, the actual context to extract the answer from + start: int, starting answer token index + end: int, ending answer token index + + Returns: + dict: {'answer': str, 'start': int, 'end': int} + """ + words = [] + token_idx = char_start_idx = char_end_idx = chars_idx = 0 + + for i, word in enumerate(text.split(" ")): + token = self.tokenizer.tokenize(word) + + # Append words if they are in the span + if start <= token_idx <= end: + if token_idx == start: + char_start_idx = chars_idx + + if token_idx == end: + char_end_idx = chars_idx + len(word) + + words += [word] + + # Stop if we went over the end of the answer + if token_idx > end: + break + + # Append the subtokenization length to the running index + token_idx += len(token) + chars_idx += len(word) + 1 + + # Join text with spaces + return {'answer': ' '.join(words), 'start': max(0, char_start_idx), 'end': min(len(text), char_end_idx)} + + +# Register all the supported task here +SUPPORTED_TASKS = { + 'feature-extraction': { + 'impl': FeatureExtractionPipeline, + 'tf': TFAutoModel if is_tf_available() else None, + 'pt': AutoModel if is_torch_available() else None, + 'default': { + 'model': { + 'pt': 'distilbert-base-uncased', + 'tf': 'distilbert-base-uncased', + }, + 'config': None, + 'tokenizer': 'distilbert-base-uncased' + } + }, + 'sentiment-analysis': { + 'impl': TextClassificationPipeline, + 'tf': TFAutoModelForSequenceClassification if is_tf_available() else None, + 'pt': AutoModelForSequenceClassification if is_torch_available() else None, + 'default': { + 'model': { + 'pt': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin', + 'tf': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-tf_model.h5', + }, + 'config': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json', + 'tokenizer': 'distilbert-base-uncased' + } + }, + 'ner': { + 'impl': NerPipeline, + 'tf': TFAutoModelForTokenClassification if is_tf_available() else None, + 'pt': AutoModelForTokenClassification if is_torch_available() else None, + 'default': { + 'model': { + 'pt':'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-pytorch_model.bin', + 'tf': 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-tf_model.h5', + }, + 'config': 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json', + 'tokenizer': 'bert-large-cased' + } + }, + 'question-answering': { + 'impl': QuestionAnsweringPipeline, + 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, + 'pt': AutoModelForQuestionAnswering if is_torch_available() else None, + 'default': { + 'model': { + 'pt': 'distilbert-base-uncased-distilled-squad', + 'tf': 'distilbert-base-uncased-distilled-squad', + }, + 'config': None, + 'tokenizer': 'distilbert-base-uncased' + } + } +} + + +def pipeline(task: str, model: Optional = None, + config: Optional[Union[str, PretrainedConfig]] = None, + tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, + modelcard: Optional[Union[str, ModelCard]] = None, + **kwargs) -> Pipeline: + """ + Utility factory method to build a pipeline. + Pipeline are made of: + A Tokenizer instance in charge of mapping raw textual input to token + A Model instance + Some (optional) post processing for enhancing model's output + + Examples: + pipeline('sentiment-analysis') + pipeline('question-answering', model='distilbert-base-uncased-distilled-squad', tokenizer='bert-base-cased') + pipeline('ner', model=AutoModel.from_pretrained(...), tokenizer=AutoTokenizer.from_pretrained(...) + pipeline('ner', model='https://...pytorch-model.bin', config='https://...config.json', tokenizer='bert-base-cased') + """ + # Retrieve the task + if task not in SUPPORTED_TASKS: + raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys()))) + + framework = get_framework(model) + + targeted_task = SUPPORTED_TASKS[task] + task, model_class = targeted_task['impl'], targeted_task[framework] + + # Use default model/config/tokenizer for the task if no model is provided + if model is None: + models, config, tokenizer = tuple(targeted_task['default'].values()) + model = models[framework] + + # Try to infer tokenizer from model or config name (if provided as str) + if tokenizer is None: + if isinstance(model, str) and model in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: + tokenizer = model + elif isinstance(config, str) and config in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: + tokenizer = config + else: + # Impossible to guest what is the right tokenizer here + raise Exception("Impossible to guess which tokenizer to use. " + "Please provided a PretrainedTokenizer class or a path/url/shortcut name to a pretrained tokenizer.") + + # Try to infer modelcard from model or config name (if provided as str) + if modelcard is None: + # Try to fallback on one of the provided string for model or config (will replace the suffix) + if isinstance(model, str): + modelcard = model + elif isinstance(config, str): + modelcard = config + + # Instantiate tokenizer if needed + if isinstance(tokenizer, six.string_types): + tokenizer = AutoTokenizer.from_pretrained(tokenizer) + + # Instantiate config if needed + if isinstance(config, str): + config = AutoConfig.from_pretrained(config) + + # Instantiate modelcard if needed + if isinstance(modelcard, str): + modelcard = ModelCard.from_pretrained(modelcard) + + # Instantiate model if needed + if isinstance(model, str): + # Handle transparent TF/PT model conversion + model_kwargs = {} + if framework == 'pt' and model.endswith('.h5'): + model_kwargs['from_tf'] = True + logger.warning('Model might be a TensorFlow model (ending with `.h5`) but TensorFlow is not available. ' + 'Trying to load the model with PyTorch.') + elif framework == 'tf' and model.endswith('.bin'): + model_kwargs['from_pt'] = True + logger.warning('Model might be a PyTorch model (ending with `.bin`) but PyTorch is not available. ' + 'Trying to load the model with Tensorflow.') + model = model_class.from_pretrained(model, config=config, **model_kwargs) + + return task(model=model, tokenizer=tokenizer, modelcard=modelcard, framework=framework, **kwargs) diff --git a/transformers/tests/configuration_common_test.py b/transformers/tests/configuration_common_test.py index 8ee75115..376d110d 100644 --- a/transformers/tests/configuration_common_test.py +++ b/transformers/tests/configuration_common_test.py @@ -16,15 +16,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import copy import os -import shutil import json -import random -import uuid +import tempfile import unittest -import logging +from .tokenization_tests_commons import TemporaryDirectory class ConfigTester(object): @@ -48,16 +45,28 @@ class ConfigTester(object): def create_and_test_config_to_json_file(self): config_first = self.config_class(**self.inputs_dict) - json_file_path = os.path.join(os.getcwd(), "config_" + str(uuid.uuid4()) + ".json") - config_first.to_json_file(json_file_path) - config_second = self.config_class.from_json_file(json_file_path) - os.remove(json_file_path) + + with TemporaryDirectory() as tmpdirname: + json_file_path = os.path.join(tmpdirname, "config.json") + config_first.to_json_file(json_file_path) + config_second = self.config_class.from_json_file(json_file_path) + + self.parent.assertEqual(config_second.to_dict(), config_first.to_dict()) + + def create_and_test_config_from_and_save_pretrained(self): + config_first = self.config_class(**self.inputs_dict) + + with TemporaryDirectory() as tmpdirname: + config_first.save_pretrained(tmpdirname) + config_second = self.config_class.from_pretrained(tmpdirname) + self.parent.assertEqual(config_second.to_dict(), config_first.to_dict()) def run_common_tests(self): self.create_and_test_config_common_properties() self.create_and_test_config_to_json_string() self.create_and_test_config_to_json_file() + self.create_and_test_config_from_and_save_pretrained() if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/transformers/tests/conftest.py b/transformers/tests/conftest.py deleted file mode 100644 index f809234c..00000000 --- a/transformers/tests/conftest.py +++ /dev/null @@ -1,31 +0,0 @@ -# content of conftest.py - -import pytest - - -def pytest_addoption(parser): - parser.addoption( - "--runslow", action="store_true", default=False, help="run slow tests" - ) - parser.addoption( - "--use_cuda", action="store_true", default=False, help="run tests on gpu" - ) - - -def pytest_configure(config): - config.addinivalue_line("markers", "slow: mark test as slow to run") - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--runslow"): - # --runslow given in cli: do not skip slow tests - return - skip_slow = pytest.mark.skip(reason="need --runslow option to run") - for item in items: - if "slow" in item.keywords: - item.add_marker(skip_slow) - -@pytest.fixture -def use_cuda(request): - """ Run test on gpu """ - return request.config.getoption("--use_cuda") diff --git a/transformers/tests/fixtures/empty.txt b/transformers/tests/fixtures/empty.txt new file mode 100644 index 00000000..e69de29b diff --git a/transformers/tests/fixtures/spiece.model b/transformers/tests/fixtures/spiece.model new file mode 100644 index 00000000..c91b8acf Binary files /dev/null and b/transformers/tests/fixtures/spiece.model differ diff --git a/transformers/tests/hf_api_test.py b/transformers/tests/hf_api_test.py new file mode 100644 index 00000000..b45f5ace --- /dev/null +++ b/transformers/tests/hf_api_test.py @@ -0,0 +1,120 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function + +import os +import time +import unittest + +import requests +import six + +from transformers.hf_api import HfApi, HfFolder, HTTPError, PresignedUrl, S3Obj + +USER = "__DUMMY_TRANSFORMERS_USER__" +PASS = "__DUMMY_TRANSFORMERS_PASS__" +FILES = [ + ( + "Test-{}.txt".format(int(time.time())), + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "fixtures/input.txt" + ) + ), + ( + "yoyo {}.txt".format(int(time.time())), # space is intentional + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "fixtures/empty.txt" + ) + ), +] + + + +class HfApiCommonTest(unittest.TestCase): + _api = HfApi(endpoint="https://moon-staging.huggingface.co") + + +class HfApiLoginTest(HfApiCommonTest): + def test_login_invalid(self): + with self.assertRaises(HTTPError): + self._api.login(username=USER, password="fake") + + def test_login_valid(self): + token = self._api.login(username=USER, password=PASS) + self.assertIsInstance(token, six.string_types) + + +class HfApiEndpointsTest(HfApiCommonTest): + @classmethod + def setUpClass(cls): + """ + Share this valid token in all tests below. + """ + cls._token = cls._api.login(username=USER, password=PASS) + + def test_whoami(self): + user = self._api.whoami(token=self._token) + self.assertEqual(user, USER) + + def test_presign(self): + for FILE_KEY, FILE_PATH in FILES: + urls = self._api.presign(token=self._token, filename=FILE_KEY) + self.assertIsInstance(urls, PresignedUrl) + self.assertEqual(urls.type, "text/plain") + + def test_presign_and_upload(self): + for FILE_KEY, FILE_PATH in FILES: + access_url = self._api.presign_and_upload( + token=self._token, filename=FILE_KEY, filepath=FILE_PATH + ) + self.assertIsInstance(access_url, six.string_types) + with open(FILE_PATH, 'r') as f: + body = f.read() + r = requests.get(access_url) + self.assertEqual(r.text, body) + + def test_list_objs(self): + objs = self._api.list_objs(token=self._token) + self.assertIsInstance(objs, list) + if len(objs) > 0: + o = objs[-1] + self.assertIsInstance(o, S3Obj) + + + +class HfFolderTest(unittest.TestCase): + def test_token_workflow(self): + """ + Test the whole token save/get/delete workflow, + with the desired behavior with respect to non-existent tokens. + """ + token = "token-{}".format(int(time.time())) + HfFolder.save_token(token) + self.assertEqual( + HfFolder.get_token(), + token + ) + HfFolder.delete_token() + HfFolder.delete_token() + # ^^ not an error, we test that the + # second call does not fail. + self.assertEqual( + HfFolder.get_token(), + None + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/transformers/tests/model_card_test.py b/transformers/tests/model_card_test.py new file mode 100644 index 00000000..b293b572 --- /dev/null +++ b/transformers/tests/model_card_test.py @@ -0,0 +1,89 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import json +import unittest + +from transformers.modelcard import ModelCard +from .tokenization_tests_commons import TemporaryDirectory + +class ModelCardTester(unittest.TestCase): + + def setUp(self): + self.inputs_dict = {'model_details': { + 'Organization': 'testing', + 'Model date': 'today', + 'Model version': 'v2.1, Developed by Test Corp in 2019.', + 'Architecture': 'Convolutional Neural Network.', + }, + 'metrics': 'BLEU and ROUGE-1', + 'evaluation_data':{ + 'Datasets':{ + 'BLEU': 'My-great-dataset-v1', + 'ROUGE-1': 'My-short-dataset-v2.1', + }, + 'Preprocessing': 'See details on https://arxiv.org/pdf/1810.03993.pdf' + }, + 'training_data':{ + 'Dataset': 'English Wikipedia dump dated 2018-12-01', + 'Preprocessing': 'Using SentencePiece vocabulary of size 52k tokens. See details on https://arxiv.org/pdf/1810.03993.pdf' + }, + 'quantitative_analyses': { + 'BLEU': 55.1, + 'ROUGE-1': 76, + }, + } + + def test_model_card_common_properties(self): + modelcard = ModelCard.from_dict(self.inputs_dict) + self.assertTrue(hasattr(modelcard, 'model_details')) + self.assertTrue(hasattr(modelcard, 'intended_use')) + self.assertTrue(hasattr(modelcard, 'factors')) + self.assertTrue(hasattr(modelcard, 'metrics')) + self.assertTrue(hasattr(modelcard, 'evaluation_data')) + self.assertTrue(hasattr(modelcard, 'training_data')) + self.assertTrue(hasattr(modelcard, 'quantitative_analyses')) + self.assertTrue(hasattr(modelcard, 'ethical_considerations')) + self.assertTrue(hasattr(modelcard, 'caveats_and_recommendations')) + + def test_model_card_to_json_string(self): + modelcard = ModelCard.from_dict(self.inputs_dict) + obj = json.loads(modelcard.to_json_string()) + for key, value in self.inputs_dict.items(): + self.assertEqual(obj[key], value) + + def test_model_card_to_json_file(self): + model_card_first = ModelCard.from_dict(self.inputs_dict) + + with TemporaryDirectory() as tmpdirname: + filename = os.path.join(tmpdirname, u"modelcard.json") + model_card_first.to_json_file(filename) + model_card_second = ModelCard.from_json_file(filename) + + self.assertEqual(model_card_second.to_dict(), model_card_first.to_dict()) + + def test_model_card_from_and_save_pretrained(self): + model_card_first = ModelCard.from_dict(self.inputs_dict) + + with TemporaryDirectory() as tmpdirname: + model_card_first.save_pretrained(tmpdirname) + model_card_second = ModelCard.from_pretrained(tmpdirname) + + self.assertEqual(model_card_second.to_dict(), model_card_first.to_dict()) + +if __name__ == "__main__": + unittest.main() diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py new file mode 100644 index 00000000..b726fd92 --- /dev/null +++ b/transformers/tests/modeling_albert_test.py @@ -0,0 +1,237 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +from transformers import is_torch_available + +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device + +if is_torch_available(): + from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM, + AlbertForSequenceClassification, AlbertForQuestionAnswering, + ) + from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +@require_torch +class AlbertModelTest(CommonTestCases.CommonModelTester): + + all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () + + class AlbertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + embedding_size=16, + hidden_size=36, + num_hidden_layers=6, + num_hidden_groups=6, + num_attention_heads=6, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.embedding_size = embedding_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + self.num_hidden_groups = num_hidden_groups + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = AlbertConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range, + num_hidden_groups=self.num_hidden_groups) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def check_loss_output(self, result): + self.parent.assertListEqual( + list(result["loss"].size()), + []) + + def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertModel(config=config) + model.to(torch_device) + model.eval() + sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output, + "pooled_output": pooled_output, + } + self.parent.assertListEqual( + list(result["sequence_output"].size()), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size]) + + + def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertForMaskedLM(config=config) + model.to(torch_device) + model.eval() + loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) + result = { + "loss": loss, + "prediction_scores": prediction_scores, + } + self.parent.assertListEqual( + list(result["prediction_scores"].size()), + [self.batch_size, self.seq_length, self.vocab_size]) + self.check_loss_output(result) + + def create_and_check_albert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertForQuestionAnswering(config=config) + model.to(torch_device) + model.eval() + loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, + start_positions=sequence_labels, end_positions=sequence_labels) + result = { + "loss": loss, + "start_logits": start_logits, + "end_logits": end_logits, + } + self.parent.assertListEqual( + list(result["start_logits"].size()), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].size()), + [self.batch_size, self.seq_length]) + self.check_loss_output(result) + + + def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = AlbertForSequenceClassification(config) + model.to(torch_device) + model.eval() + loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) + result = { + "loss": loss, + "logits": logits, + } + self.parent.assertListEqual( + list(result["logits"].size()), + [self.batch_size, self.num_labels]) + self.check_loss_output(result) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = AlbertModelTest.AlbertModelTester(self) + self.config_tester = ConfigTester(self, config_class=AlbertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_albert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_masked_lm(*config_and_inputs) + + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_question_answering(*config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_sequence_classification(*config_and_inputs) + + @slow + def test_model_from_pretrained(self): + for model_name in list(ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = AlbertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() diff --git a/transformers/tests/modeling_auto_test.py b/transformers/tests/modeling_auto_test.py index 6d2c7ec9..871a262f 100644 --- a/transformers/tests/modeling_auto_test.py +++ b/transformers/tests/modeling_auto_test.py @@ -18,11 +18,12 @@ from __future__ import print_function import unittest import shutil -import pytest import logging from transformers import is_torch_available +from .utils import require_torch, slow, SMALL_MODEL_IDENTIFIER + if is_torch_available(): from transformers import (AutoConfig, BertConfig, AutoModel, BertModel, @@ -33,12 +34,11 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -else: - pytestmark = pytest.mark.skip("Require Torch") +@require_torch class AutoModelTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -53,7 +53,7 @@ class AutoModelTest(unittest.TestCase): for value in loading_info.values(): self.assertEqual(len(value), 0) - @pytest.mark.slow + @slow def test_lmhead_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -66,7 +66,7 @@ class AutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, BertForMaskedLM) - @pytest.mark.slow + @slow def test_sequence_classification_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -79,7 +79,7 @@ class AutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, BertForSequenceClassification) - @pytest.mark.slow + @slow def test_question_answering_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -92,6 +92,11 @@ class AutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, BertForQuestionAnswering) + def test_from_pretrained_identifier(self): + logging.basicConfig(level=logging.INFO) + model = AutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER) + self.assertIsInstance(model, BertForMaskedLM) + if __name__ == "__main__": unittest.main() diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py index 6c93c9a1..a5adff8f 100644 --- a/transformers/tests/modeling_bert_test.py +++ b/transformers/tests/modeling_bert_test.py @@ -17,13 +17,12 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor, floats_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device if is_torch_available(): from transformers import (BertConfig, BertModel, BertForMaskedLM, @@ -31,11 +30,9 @@ if is_torch_available(): BertForQuestionAnswering, BertForSequenceClassification, BertForTokenClassification, BertForMultipleChoice) from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") -@pytest.mark.usefixtures("use_cuda") +@require_torch class BertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (BertModel, BertForMaskedLM, BertForNextSentencePrediction, @@ -67,7 +64,6 @@ class BertModelTest(CommonTestCases.CommonModelTester): num_labels=3, num_choices=4, scope=None, - device='cpu', ): self.parent = parent self.batch_size = batch_size @@ -91,29 +87,28 @@ class BertModelTest(CommonTestCases.CommonModelTester): self.num_labels = num_labels self.num_choices = num_choices self.scope = scope - self.device = device def prepare_config_and_inputs(self): - input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size).to(self.device) + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2).to(self.device) + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) token_type_ids = None if self.use_token_type_ids: - token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size).to(self.device) + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) sequence_labels = None token_labels = None choice_labels = None if self.use_labels: - sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size).to(self.device) - token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels).to(self.device) - choice_labels = ids_tensor([self.batch_size], self.num_choices).to(self.device) + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) config = BertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, @@ -144,7 +139,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertModel(config=config) - model.to(input_ids.device) + model.to(torch_device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) @@ -161,6 +156,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_model_as_decoder(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels, encoder_hidden_states, encoder_attention_mask): model = BertModel(config) + model.to(torch_device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_attention_mask) sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, encoder_hidden_states=encoder_hidden_states) @@ -177,6 +173,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) result = { @@ -190,6 +187,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_model_for_masked_lm_as_decoder(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels, encoder_hidden_states, encoder_attention_mask): model = BertForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_attention_mask) loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels, encoder_hidden_states=encoder_hidden_states) @@ -204,6 +202,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForNextSentencePrediction(config=config) + model.to(torch_device) model.eval() loss, seq_relationship_score = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, next_sentence_label=sequence_labels) result = { @@ -217,6 +216,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForPreTraining(config=config) + model.to(torch_device) model.eval() loss, prediction_scores, seq_relationship_score = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels, next_sentence_label=sequence_labels) @@ -235,6 +235,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForQuestionAnswering(config=config) + model.to(torch_device) model.eval() loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, start_positions=sequence_labels, end_positions=sequence_labels) @@ -254,6 +255,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = BertForSequenceClassification(config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) result = { @@ -268,6 +270,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = BertForTokenClassification(config=config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) result = { @@ -282,6 +285,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_multiple_choice(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_choices = self.num_choices model = BertForMultipleChoice(config=config) + model.to(torch_device) model.eval() multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() @@ -313,10 +317,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def test_config(self): self.config_tester.run_common_tests() - def test_bert_model(self, use_cuda=False): - # ^^ This could be a real fixture - if use_cuda: - self.model_tester.device = "cuda" + def test_bert_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_model(*config_and_inputs) @@ -356,12 +357,10 @@ class BertModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = BertModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = BertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index baf15314..2116651f 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -18,7 +18,7 @@ from __future__ import print_function import copy import sys -import os +import os.path import shutil import tempfile import json @@ -27,10 +27,11 @@ import uuid import unittest import logging -import pytest from transformers import is_torch_available +from .utils import CACHE_DIR, require_torch, slow, torch_device + if is_torch_available(): import torch import numpy as np @@ -38,8 +39,6 @@ if is_torch_available(): from transformers import (AdaptiveEmbedding, PretrainedConfig, PreTrainedModel, BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require Torch") if sys.version_info[0] == 2: import cPickle as pickle @@ -59,12 +58,13 @@ else: def _config_zero_init(config): configs_no_init = copy.deepcopy(config) for key in configs_no_init.__dict__.keys(): - if '_range' in key or '_std' in key: + if '_range' in key or '_std' in key or 'initializer_factor' in key: setattr(configs_no_init, key, 0.0) return configs_no_init class CommonTestCases: + @require_torch class CommonModelTester(unittest.TestCase): model_tester = None @@ -73,27 +73,30 @@ class CommonTestCases: test_pruning = True test_resize_embeddings = True test_head_masking = True + is_encoder_decoder = False def test_save_load(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() for model_class in self.all_model_classes: model = model_class(config) + model.to(torch_device) model.eval() with torch.no_grad(): outputs = model(**inputs_dict) + out_2 = outputs[0].numpy() + out_2[np.isnan(out_2)] = 0 with TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model = model_class.from_pretrained(tmpdirname) + model.to(torch_device) with torch.no_grad(): after_outputs = model(**inputs_dict) # Make sure we don't have nans - out_1 = after_outputs[0].numpy() - out_2 = outputs[0].numpy() - out_1 = out_1[~np.isnan(out_1)] - out_2 = out_2[~np.isnan(out_2)] + out_1 = after_outputs[0].cpu().numpy() + out_1[np.isnan(out_1)] = 0 max_diff = np.amax(np.abs(out_1 - out_2)) self.assertLessEqual(max_diff, 1e-5) @@ -113,20 +116,34 @@ class CommonTestCases: for model_class in self.all_model_classes: model = model_class(config) + model.to(torch_device) model.eval() - first, second = model(inputs_dict["input_ids"])[0], model(inputs_dict["input_ids"])[0] - self.assertEqual(first.ne(second).sum().item(), 0) - + with torch.no_grad(): + first = model(**inputs_dict)[0] + second = model(**inputs_dict)[0] + out_1 = first.cpu().numpy() + out_2 = second.cpu().numpy() + out_1 = out_1[~np.isnan(out_1)] + out_2 = out_2[~np.isnan(out_2)] + max_diff = np.amax(np.abs(out_1 - out_2)) + self.assertLessEqual(max_diff, 1e-5) def test_attention_outputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + decoder_seq_length = self.model_tester.decoder_seq_length if hasattr(self.model_tester, 'decoder_seq_length') else self.model_tester.seq_length + encoder_seq_length = self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length + decoder_key_length = self.model_tester.key_length if hasattr(self.model_tester, 'key_length') else decoder_seq_length + encoder_key_length = self.model_tester.key_length if hasattr(self.model_tester, 'key_length') else encoder_seq_length + for model_class in self.all_model_classes: config.output_attentions = True config.output_hidden_states = False model = model_class(config) + model.to(torch_device) model.eval() - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, False) @@ -134,27 +151,42 @@ class CommonTestCases: self.assertListEqual( list(attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + encoder_seq_length , + encoder_key_length]) out_len = len(outputs) + if self.is_encoder_decoder: + self.assertEqual(out_len % 2, 0) + decoder_attentions = outputs[(out_len // 2)-1] + self.assertEqual(model.config.output_attentions, True) + self.assertEqual(model.config.output_hidden_states, False) + self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers) + self.assertListEqual( + list(decoder_attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, + decoder_seq_length, + decoder_key_length + ]) + # Check attention is always last and order is fine config.output_attentions = True config.output_hidden_states = True model = model_class(config) + model.to(torch_device) model.eval() - outputs = model(**inputs_dict) - self.assertEqual(out_len+1, len(outputs)) + with torch.no_grad(): + outputs = model(**inputs_dict) + self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1), len(outputs)) self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, True) - attentions = outputs[-1] - self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + self_attentions = outputs[-1] + self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers) self.assertListEqual( - list(attentions[0].shape[-3:]), + list(self_attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + encoder_seq_length, + encoder_key_length]) def test_torchscript(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -181,27 +213,32 @@ class CommonTestCases: configs_no_init.torchscript = True for model_class in self.all_model_classes: model = model_class(config=configs_no_init) + model.to(torch_device) model.eval() inputs = inputs_dict['input_ids'] # Let's keep only input_ids try: - torch.jit.trace(model, inputs) + traced_gpt2 = torch.jit.trace(model, inputs) except RuntimeError: self.fail("Couldn't trace module.") - try: - traced_gpt2 = torch.jit.trace(model, inputs) - torch.jit.save(traced_gpt2, "traced_model.pt") - except RuntimeError: - self.fail("Couldn't save module.") + with TemporaryDirectory() as tmp_dir_name: + pt_file_name = os.path.join(tmp_dir_name, "traced_model.pt") - try: - loaded_model = torch.jit.load("traced_model.pt") - os.remove("traced_model.pt") - except ValueError: - self.fail("Couldn't load module.") + try: + torch.jit.save(traced_gpt2, pt_file_name) + except Exception: + self.fail("Couldn't save module.") + try: + loaded_model = torch.jit.load(pt_file_name) + except Exception: + self.fail("Couldn't load module.") + + model.to(torch_device) model.eval() + + loaded_model.to(torch_device) loaded_model.eval() model_params = model.parameters() @@ -214,7 +251,6 @@ class CommonTestCases: self.assertTrue(models_equal) - def test_headmasking(self): if not self.test_head_masking: return @@ -228,11 +264,12 @@ class CommonTestCases: configs_no_init = _config_zero_init(config) # To be sure we have no Nan for model_class in self.all_model_classes: model = model_class(config=configs_no_init) + model.to(torch_device) model.eval() # Prepare head_mask # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior) - head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads) + head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads, device=torch_device) head_mask[0, 0] = 0 head_mask[-1, :-1] = 0 head_mask.requires_grad_(requires_grad=True) @@ -268,7 +305,6 @@ class CommonTestCases: self.assertNotEqual( attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) - def test_head_pruning(self): if not self.test_pruning: return @@ -282,11 +318,13 @@ class CommonTestCases: config.output_attentions = True config.output_hidden_states = False model = model_class(config=config) + model.to(torch_device) model.eval() heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), -1: [0]} model.prune_heads(heads_to_prune) - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] @@ -310,23 +348,24 @@ class CommonTestCases: config.output_attentions = True config.output_hidden_states = False model = model_class(config=config) + model.to(torch_device) model.eval() heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), -1: [0]} model.prune_heads(heads_to_prune) - directory = "pruned_model" - if not os.path.exists(directory): - os.makedirs(directory) - model.save_pretrained(directory) - model = model_class.from_pretrained(directory) - outputs = model(**inputs_dict) + with TemporaryDirectory() as temp_dir_name: + model.save_pretrained(temp_dir_name) + model = model_class.from_pretrained(temp_dir_name) + model.to(torch_device) + + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], 1) self.assertEqual(attentions[1].shape[-3], self.model_tester.num_attention_heads) self.assertEqual(attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) - shutil.rmtree(directory) def test_head_pruning_save_load_from_config_init(self): if not self.test_pruning: @@ -346,9 +385,11 @@ class CommonTestCases: config.pruned_heads = heads_to_prune model = model_class(config=config) + model.to(torch_device) model.eval() - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], 1) @@ -372,9 +413,11 @@ class CommonTestCases: config.pruned_heads = heads_to_prune model = model_class(config=config) + model.to(torch_device) model.eval() - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], self.model_tester.num_attention_heads - 1) @@ -382,15 +425,13 @@ class CommonTestCases: self.assertEqual(attentions[2].shape[-3], self.model_tester.num_attention_heads) self.assertEqual(attentions[3].shape[-3], self.model_tester.num_attention_heads) - directory = "pruned_model" + with TemporaryDirectory() as temp_dir_name: + model.save_pretrained(temp_dir_name) + model = model_class.from_pretrained(temp_dir_name) + model.to(torch_device) - if not os.path.exists(directory): - os.makedirs(directory) - model.save_pretrained(directory) - model = model_class.from_pretrained(directory) - shutil.rmtree(directory) - - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], self.model_tester.num_attention_heads - 1) @@ -401,7 +442,8 @@ class CommonTestCases: heads_to_prune = {0: [0], 2: [1, 2]} model.prune_heads(heads_to_prune) - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], self.model_tester.num_attention_heads -1) @@ -411,7 +453,6 @@ class CommonTestCases: self.assertDictEqual(model.config.pruned_heads, {0: [0], 1: [1, 2], 2: [1, 2]}) - def test_hidden_states_output(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -419,15 +460,18 @@ class CommonTestCases: config.output_hidden_states = True config.output_attentions = False model = model_class(config) + model.to(torch_device) model.eval() - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) hidden_states = outputs[-1] self.assertEqual(model.config.output_attentions, False) self.assertEqual(model.config.output_hidden_states, True) self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) self.assertListEqual( list(hidden_states[0].shape[-2:]), - [self.model_tester.seq_length, self.model_tester.hidden_size]) + [self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length, + self.model_tester.hidden_size]) def test_resize_tokens_embeddings(self): original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -533,17 +577,29 @@ class CommonTestCases: def test_inputs_embeds(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - input_ids = inputs_dict["input_ids"] - del inputs_dict["input_ids"] + if not self.is_encoder_decoder: + input_ids = inputs_dict["input_ids"] + del inputs_dict["input_ids"] + else: + encoder_input_ids = inputs_dict["encoder_input_ids"] + decoder_input_ids = inputs_dict["decoder_input_ids"] + del inputs_dict["encoder_input_ids"] + del inputs_dict["decoder_input_ids"] for model_class in self.all_model_classes: model = model_class(config) + model.to(torch_device) model.eval() wte = model.get_input_embeddings() - inputs_dict["inputs_embeds"] = wte(input_ids) - outputs = model(**inputs_dict) + if not self.is_encoder_decoder: + inputs_dict["inputs_embeds"] = wte(input_ids) + else: + inputs_dict["encoder_inputs_embeds"] = wte(encoder_input_ids) + inputs_dict["decoder_inputs_embeds"] = wte(decoder_input_ids) + with torch.no_grad(): + outputs = model(**inputs_dict) class GPTModelTester(CommonModelTester): @@ -615,7 +671,7 @@ class CommonTestCases: mc_token_ids = ids_tensor([self.batch_size, self.n_choices], self.seq_length) config = self.config_class( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_positions=self.n_positions, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, @@ -628,11 +684,13 @@ class CommonTestCases: def create_and_check_base_model(self, config, input_ids, token_type_ids, position_ids, mc_labels, lm_labels, mc_token_ids): model = self.base_model_class(config) + model.to(torch_device) model.eval() - outputs = model(input_ids, position_ids, token_type_ids) - outputs = model(input_ids, position_ids) - outputs = model(input_ids) + with torch.no_grad(): + outputs = model(input_ids, position_ids, token_type_ids) + outputs = model(input_ids, position_ids) + outputs = model(input_ids) hidden_state = outputs[0] self.parent.assertListEqual( @@ -643,8 +701,10 @@ class CommonTestCases: def create_and_check_lm_head(self, config, input_ids, token_type_ids, position_ids, mc_labels, lm_labels, mc_token_ids): model = self.lm_head_model_class(config) + model.to(torch_device) model.eval() - outputs = model(input_ids, position_ids, token_type_ids, lm_labels) + with torch.no_grad(): + outputs = model(input_ids, position_ids, token_type_ids, lm_labels) loss, lm_logits = outputs[:2] total_voc = self.vocab_size @@ -659,8 +719,10 @@ class CommonTestCases: mc_labels, lm_labels, mc_token_ids): for model_class in self.all_model_classes: model = model_class(config) + model.to(torch_device) model.eval() - outputs = model(input_ids) + with torch.no_grad(): + outputs = model(input_ids) presents = outputs[-1] self.parent.assertEqual(self.num_hidden_layers, len(presents)) self.parent.assertListEqual( @@ -671,8 +733,10 @@ class CommonTestCases: def create_and_check_double_heads(self, config, input_ids, token_type_ids, position_ids, mc_labels, lm_labels, mc_token_ids): model = self.double_head_model_class(config) + model.to(torch_device) model.eval() - outputs = model(input_ids, mc_token_ids, lm_labels=lm_labels, mc_labels=mc_labels, + with torch.no_grad(): + outputs = model(input_ids, mc_token_ids, lm_labels=lm_labels, mc_labels=mc_labels, token_type_ids=token_type_ids, position_ids=position_ids) lm_loss, mc_loss, lm_logits, mc_logits = outputs[:4] loss = [lm_loss, mc_loss] @@ -689,10 +753,8 @@ class CommonTestCases: [[], []]) def create_and_check_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(self.base_model_class.pretrained_model_archive_map.keys())[:1]: - model = self.base_model_class.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = self.base_model_class.from_pretrained(model_name, cache_dir=CACHE_DIR) self.parent.assertIsNotNone(model) def prepare_config_and_inputs_for_common(self): @@ -716,7 +778,7 @@ class CommonTestCases: config_and_inputs = self.prepare_config_and_inputs() self.create_and_check_presents(*config_and_inputs) - @pytest.mark.slow + @slow def run_slow_tests(self): self.create_and_check_model_from_pretrained() @@ -770,7 +832,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None): for _ in range(total_dims): values.append(rng.randint(0, vocab_size - 1)) - return torch.tensor(data=values, dtype=torch.long).view(shape).contiguous() + return torch.tensor(data=values, dtype=torch.long, device=torch_device).view(shape).contiguous() def floats_tensor(shape, scale=1.0, rng=None, name=None): @@ -786,11 +848,12 @@ def floats_tensor(shape, scale=1.0, rng=None, name=None): for _ in range(total_dims): values.append(rng.random() * scale) - return torch.tensor(data=values, dtype=torch.float).view(shape).contiguous() + return torch.tensor(data=values, dtype=torch.float, device=torch_device).view(shape).contiguous() +@require_torch class ModelUtilsTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_ctrl_test.py b/transformers/tests/modeling_ctrl_test.py index 47ff8d8d..ed0d62d1 100644 --- a/transformers/tests/modeling_ctrl_test.py +++ b/transformers/tests/modeling_ctrl_test.py @@ -16,8 +16,6 @@ from __future__ import division from __future__ import print_function import unittest -import pytest -import shutil import pdb from transformers import is_torch_available @@ -25,13 +23,13 @@ from transformers import is_torch_available if is_torch_available(): from transformers import (CTRLConfig, CTRLModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRLLMHeadModel) -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device +@require_torch class CTRLModelTest(CommonTestCases.CommonModelTester): all_model_classes = (CTRLModel, CTRLLMHeadModel) if is_torch_available() else () @@ -115,7 +113,7 @@ class CTRLModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = CTRLConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, @@ -140,6 +138,7 @@ class CTRLModelTest(CommonTestCases.CommonModelTester): def create_and_check_ctrl_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = CTRLModel(config=config) + model.to(torch_device) model.eval() model(input_ids, token_type_ids=token_type_ids, head_mask=head_mask) @@ -157,6 +156,7 @@ class CTRLModelTest(CommonTestCases.CommonModelTester): def create_and_check_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = CTRLLMHeadModel(config) + model.to(torch_device) model.eval() loss, lm_logits, _ = model(input_ids, token_type_ids=token_type_ids, labels=input_ids) @@ -202,12 +202,10 @@ class CTRLModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_lm_head_model(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(CTRL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = CTRLModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = CTRLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_distilbert_test.py b/transformers/tests/modeling_distilbert_test.py index 8099c035..ac6f5d24 100644 --- a/transformers/tests/modeling_distilbert_test.py +++ b/transformers/tests/modeling_distilbert_test.py @@ -17,7 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import pytest from transformers import is_torch_available @@ -25,13 +24,13 @@ if is_torch_available(): from transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, DistilBertForTokenClassification, DistilBertForQuestionAnswering, DistilBertForSequenceClassification) -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device +@require_torch class DistilBertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (DistilBertModel, DistilBertForMaskedLM, DistilBertForQuestionAnswering, @@ -106,7 +105,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = DistilBertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, dim=self.hidden_size, n_layers=self.num_hidden_layers, n_heads=self.num_attention_heads, @@ -126,6 +125,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_model(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): model = DistilBertModel(config=config) + model.to(torch_device) model.eval() (sequence_output,) = model(input_ids, input_mask) (sequence_output,) = model(input_ids) @@ -139,6 +139,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_for_masked_lm(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): model = DistilBertForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, masked_lm_labels=token_labels) result = { @@ -152,6 +153,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_for_question_answering(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): model = DistilBertForQuestionAnswering(config=config) + model.to(torch_device) model.eval() loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, start_positions=sequence_labels, end_positions=sequence_labels) result = { @@ -170,6 +172,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_for_sequence_classification(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = DistilBertForSequenceClassification(config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, labels=sequence_labels) result = { @@ -184,6 +187,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_for_token_classification(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = DistilBertForTokenClassification(config=config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, labels=token_labels) @@ -229,12 +233,10 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_distilbert_for_token_classification(*config_and_inputs) - # @pytest.mark.slow + # @slow # def test_model_from_pretrained(self): - # cache_dir = "/tmp/transformers_test/" # for model_name in list(DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - # model = DistilBertModel.from_pretrained(model_name, cache_dir=cache_dir) - # shutil.rmtree(cache_dir) + # model = DistilBertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) # self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_encoder_decoder_test.py b/transformers/tests/modeling_encoder_decoder_test.py index a6c88ed9..64e86df8 100644 --- a/transformers/tests/modeling_encoder_decoder_test.py +++ b/transformers/tests/modeling_encoder_decoder_test.py @@ -15,19 +15,18 @@ import logging import unittest -import pytest from transformers import is_torch_available +from .utils import require_torch, slow if is_torch_available(): from transformers import BertModel, BertForMaskedLM, Model2Model from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") +@require_torch class EncoderDecoderModelTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_model2model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_gpt2_test.py b/transformers/tests/modeling_gpt2_test.py index 4263e51b..ad2ec1fd 100644 --- a/transformers/tests/modeling_gpt2_test.py +++ b/transformers/tests/modeling_gpt2_test.py @@ -17,21 +17,19 @@ from __future__ import division from __future__ import print_function import unittest -import pytest -import shutil from transformers import is_torch_available if is_torch_available(): from transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2DoubleHeadsModel) -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device +@require_torch class GPT2ModelTest(CommonTestCases.CommonModelTester): all_model_classes = (GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel) if is_torch_available() else () @@ -111,7 +109,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = GPT2Config( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, @@ -136,6 +134,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): def create_and_check_gpt2_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = GPT2Model(config=config) + model.to(torch_device) model.eval() model(input_ids, token_type_ids=token_type_ids, head_mask=head_mask) @@ -153,6 +152,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): def create_and_check_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = GPT2LMHeadModel(config) + model.to(torch_device) model.eval() loss, lm_logits, _ = model(input_ids, token_type_ids=token_type_ids, labels=input_ids) @@ -171,6 +171,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): def create_and_check_double_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, mc_token_ids, *args): model = GPT2DoubleHeadsModel(config) + model.to(torch_device) model.eval() @@ -235,12 +236,10 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_double_lm_head_model(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = GPT2Model.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = GPT2Model.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_openai_test.py b/transformers/tests/modeling_openai_test.py index 33218288..1880febc 100644 --- a/transformers/tests/modeling_openai_test.py +++ b/transformers/tests/modeling_openai_test.py @@ -17,21 +17,19 @@ from __future__ import division from __future__ import print_function import unittest -import pytest -import shutil from transformers import is_torch_available if is_torch_available(): from transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device +@require_torch class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): all_model_classes = (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) if is_torch_available() else () @@ -99,7 +97,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = OpenAIGPTConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, @@ -124,6 +122,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): def create_and_check_openai_gpt_model(self, config, input_ids, head_mask, token_type_ids, *args): model = OpenAIGPTModel(config=config) + model.to(torch_device) model.eval() model(input_ids, token_type_ids=token_type_ids, head_mask=head_mask) @@ -139,6 +138,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): def create_and_check_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args): model = OpenAIGPTLMHeadModel(config) + model.to(torch_device) model.eval() loss, lm_logits = model(input_ids, token_type_ids=token_type_ids, labels=input_ids) @@ -157,6 +157,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): def create_and_check_double_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args): model = OpenAIGPTDoubleHeadsModel(config) + model.to(torch_device) model.eval() loss, lm_logits, mc_logits = model(input_ids, token_type_ids=token_type_ids, lm_labels=input_ids) @@ -203,12 +204,10 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_double_lm_head_model(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = OpenAIGPTModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = OpenAIGPTModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_roberta_test.py b/transformers/tests/modeling_roberta_test.py index 0620ddf6..732e589c 100644 --- a/transformers/tests/modeling_roberta_test.py +++ b/transformers/tests/modeling_roberta_test.py @@ -17,8 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest from transformers import is_torch_available @@ -26,14 +24,15 @@ if is_torch_available(): import torch from transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForTokenClassification) + from transformers.modeling_roberta import RobertaEmbeddings from transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device +@require_torch class RobertaModelTest(CommonTestCases.CommonModelTester): all_model_classes = (RobertaForMaskedLM, RobertaModel) if is_torch_available() else () @@ -107,7 +106,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = RobertaConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, @@ -129,6 +128,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): def create_and_check_roberta_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = RobertaModel(config=config) + model.to(torch_device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) @@ -146,6 +146,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): def create_and_check_roberta_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = RobertaForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) result = { @@ -161,6 +162,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = RobertaForTokenClassification(config=config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) @@ -195,22 +197,71 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_roberta_for_masked_lm(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = RobertaModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = RobertaModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) + def test_create_position_ids_respects_padding_index(self): + """ Ensure that the default position ids only assign a sequential . This is a regression + test for https://github.com/huggingface/transformers/issues/1761 + + The position ids should be masked with the embedding object's padding index. Therefore, the + first available non-padding position index is RobertaEmbeddings.padding_idx + 1 + """ + config = self.model_tester.prepare_config_and_inputs()[0] + model = RobertaEmbeddings(config=config) + + input_ids = torch.as_tensor([[12, 31, 13, model.padding_idx]]) + expected_positions = torch.as_tensor([[ + 0 + model.padding_idx + 1, + 1 + model.padding_idx + 1, + 2 + model.padding_idx + 1, + model.padding_idx + ]]) + + position_ids = model.create_position_ids_from_input_ids(input_ids) + self.assertEqual( + position_ids.shape, + expected_positions.shape + ) + self.assertTrue(torch.all(torch.eq(position_ids, expected_positions))) + + def test_create_position_ids_from_inputs_embeds(self): + """ Ensure that the default position ids only assign a sequential . This is a regression + test for https://github.com/huggingface/transformers/issues/1761 + + The position ids should be masked with the embedding object's padding index. Therefore, the + first available non-padding position index is RobertaEmbeddings.padding_idx + 1 + """ + config = self.model_tester.prepare_config_and_inputs()[0] + embeddings = RobertaEmbeddings(config=config) + + inputs_embeds = torch.Tensor(2, 4, 30) + expected_single_positions = [ + 0 + embeddings.padding_idx + 1, + 1 + embeddings.padding_idx + 1, + 2 + embeddings.padding_idx + 1, + 3 + embeddings.padding_idx + 1, + ] + expected_positions = torch.as_tensor([expected_single_positions, expected_single_positions]) + position_ids = embeddings.create_position_ids_from_inputs_embeds(inputs_embeds) + self.assertEqual( + position_ids.shape, + expected_positions.shape + ) + self.assertTrue( + torch.all(torch.eq(position_ids, expected_positions)) + ) class RobertaModelIntegrationTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_inference_masked_lm(self): model = RobertaForMaskedLM.from_pretrained('roberta-base') - + input_ids = torch.tensor([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] expected_shape = torch.Size((1, 11, 50265)) @@ -228,10 +279,10 @@ class RobertaModelIntegrationTest(unittest.TestCase): torch.allclose(output[:, :3, :3], expected_slice, atol=1e-3) ) - @pytest.mark.slow + @slow def test_inference_no_head(self): model = RobertaModel.from_pretrained('roberta-base') - + input_ids = torch.tensor([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] # compare the actual values for a slice. @@ -244,10 +295,10 @@ class RobertaModelIntegrationTest(unittest.TestCase): torch.allclose(output[:, :3, :3], expected_slice, atol=1e-3) ) - @pytest.mark.slow + @slow def test_inference_classification_head(self): model = RobertaForSequenceClassification.from_pretrained('roberta-large-mnli') - + input_ids = torch.tensor([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] expected_shape = torch.Size((1, 3)) diff --git a/transformers/tests/modeling_t5_test.py b/transformers/tests/modeling_t5_test.py new file mode 100644 index 00000000..9fd9a4b3 --- /dev/null +++ b/transformers/tests/modeling_t5_test.py @@ -0,0 +1,182 @@ +# coding=utf-8 +# Copyright 2018 Google T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +from transformers import is_torch_available + +from .modeling_common_test import (CommonTestCases, ids_tensor, floats_tensor) +from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device + +if is_torch_available(): + from transformers import (T5Config, T5Model, T5WithLMHeadModel) + from transformers.modeling_t5 import T5_PRETRAINED_MODEL_ARCHIVE_MAP + + +@require_torch +class T5ModelTest(CommonTestCases.CommonModelTester): + + all_model_classes = (T5Model, T5WithLMHeadModel) if is_torch_available() else () + test_pruning = False + test_torchscript = False + test_resize_embeddings = False + is_encoder_decoder = True + + class T5ModelTester(object): + + def __init__(self, + parent, + batch_size=13, + encoder_seq_length=7, + decoder_seq_length=9, + is_training=True, + use_attention_mask=True, + use_labels=True, + vocab_size=99, + n_positions=14, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + d_ff=37, + relative_attention_num_buckets=8, + dropout_rate=0.1, + initializer_factor=0.002, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.encoder_seq_length = encoder_seq_length + self.decoder_seq_length = decoder_seq_length + self.is_training = is_training + self.use_attention_mask = use_attention_mask + self.use_labels = use_labels + self.vocab_size = vocab_size + self.n_positions = n_positions + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.d_ff = d_ff + self.relative_attention_num_buckets = relative_attention_num_buckets + self.dropout_rate = dropout_rate + self.initializer_factor = initializer_factor + self.scope = scope + + def prepare_config_and_inputs(self): + encoder_input_ids = ids_tensor([self.batch_size, self.encoder_seq_length], self.vocab_size) + decoder_input_ids = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size) + + encoder_attention_mask = None + decoder_attention_mask = None + if self.use_attention_mask: + encoder_attention_mask = ids_tensor([self.batch_size, self.encoder_seq_length], vocab_size=2) + decoder_attention_mask = ids_tensor([self.batch_size, self.decoder_seq_length], vocab_size=2) + + decoder_lm_labels = None + if self.use_labels: + decoder_lm_labels = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size) + + config = T5Config( + vocab_size=self.vocab_size, + n_positions=self.n_positions, + d_model=self.hidden_size, + d_ff=self.d_ff, + d_kv=self.hidden_size // self.num_attention_heads, + num_layers=self.num_hidden_layers, + num_heads=self.num_attention_heads, + relative_attention_num_buckets=self.relative_attention_num_buckets, + dropout_rate=self.dropout_rate, + initializer_factor=self.initializer_factor) + + return (config, encoder_input_ids, decoder_input_ids, encoder_attention_mask, decoder_attention_mask, decoder_lm_labels) + + def check_loss_output(self, result): + self.parent.assertListEqual( + list(result["loss"].size()), + []) + + def create_and_check_t5_model(self, config, encoder_input_ids, decoder_input_ids, encoder_attention_mask, decoder_attention_mask, decoder_lm_labels): + model = T5Model(config=config) + model.eval() + decoder_output, encoder_output = model(encoder_input_ids=encoder_input_ids, + decoder_input_ids=decoder_input_ids, + encoder_attention_mask=encoder_attention_mask, + decoder_attention_mask=decoder_attention_mask) + decoder_output, encoder_output = model(encoder_input_ids=encoder_input_ids, + decoder_input_ids=decoder_input_ids) + + result = { + "encoder_output": encoder_output, + "decoder_output": decoder_output, + } + self.parent.assertListEqual( + list(result["encoder_output"].size()), + [self.batch_size, self.encoder_seq_length, self.hidden_size]) + self.parent.assertListEqual( + list(result["decoder_output"].size()), + [self.batch_size, self.decoder_seq_length, self.hidden_size]) + + + def create_and_check_t5_with_lm_head(self, config, encoder_input_ids, decoder_input_ids, encoder_attention_mask, decoder_attention_mask, decoder_lm_labels): + model = T5WithLMHeadModel(config=config) + model.eval() + outputs = model(encoder_input_ids=encoder_input_ids, decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, decoder_lm_labels=decoder_lm_labels) + loss, prediction_scores = outputs[0], outputs[1] + result = { + "loss": loss, + "prediction_scores": prediction_scores, + } + self.parent.assertListEqual( + list(result["prediction_scores"].size()), + [self.batch_size, self.decoder_seq_length, self.vocab_size]) + self.check_loss_output(result) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, encoder_input_ids, decoder_input_ids, encoder_attention_mask, + decoder_attention_mask, decoder_lm_labels) = config_and_inputs + inputs_dict = {'encoder_input_ids': encoder_input_ids, + 'decoder_input_ids': decoder_input_ids, + 'decoder_attention_mask': decoder_attention_mask, + 'encoder_attention_mask': encoder_attention_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = T5ModelTest.T5ModelTester(self) + self.config_tester = ConfigTester(self, config_class=T5Config, d_model=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_t5_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_t5_model(*config_and_inputs) + + def test_with_lm_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_t5_with_lm_head(*config_and_inputs) + + @slow + def test_model_from_pretrained(self): + for model_name in list(T5_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = T5Model.from_pretrained(model_name, cache_dir=CACHE_DIR) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py new file mode 100644 index 00000000..374417cf --- /dev/null +++ b/transformers/tests/modeling_tf_albert_test.py @@ -0,0 +1,225 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import sys + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow + +from transformers import AlbertConfig, is_tf_available + +if is_tf_available(): + import tensorflow as tf + from transformers.modeling_tf_albert import (TFAlbertModel, TFAlbertForMaskedLM, + TFAlbertForSequenceClassification, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + + +@require_tf +class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = ( + TFAlbertModel, + TFAlbertForMaskedLM, + TFAlbertForSequenceClassification + ) if is_tf_available() else () + + class TFAlbertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + embedding_size=16, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.embedding_size = embedding_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor( + [self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor( + [self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor( + [self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor( + [self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor( + [self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = AlbertConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFAlbertModel(config=config) + # inputs = {'input_ids': input_ids, + # 'attention_mask': input_mask, + # 'token_type_ids': token_type_ids} + # sequence_output, pooled_output = model(**inputs) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output, pooled_output = model(inputs) + + inputs = [input_ids, input_mask] + sequence_output, pooled_output = model(inputs) + + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output.numpy(), + "pooled_output": pooled_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].shape), [ + self.batch_size, self.hidden_size]) + + def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFAlbertForMaskedLM(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores, = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = TFAlbertForSequenceClassification(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.num_labels]) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, + 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFAlbertModelTest.TFAlbertModelTester(self) + self.config_tester = ConfigTester( + self, config_class=AlbertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_albert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_masked_lm( + *config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_sequence_classification( + *config_and_inputs) + + @slow + def test_model_from_pretrained(self): + for model_name in list(TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = TFAlbertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) + self.assertIsNotNone(model) + + +if __name__ == "__main__": + unittest.main() diff --git a/transformers/tests/modeling_tf_auto_test.py b/transformers/tests/modeling_tf_auto_test.py index 2cda3abc..2ad39ddc 100644 --- a/transformers/tests/modeling_tf_auto_test.py +++ b/transformers/tests/modeling_tf_auto_test.py @@ -18,11 +18,12 @@ from __future__ import print_function import unittest import shutil -import pytest import logging from transformers import is_tf_available +from .utils import require_tf, slow, SMALL_MODEL_IDENTIFIER + if is_tf_available(): from transformers import (AutoConfig, BertConfig, TFAutoModel, TFBertModel, @@ -33,11 +34,11 @@ if is_tf_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFAutoModelTest(unittest.TestCase): + @slow def test_model_from_pretrained(self): import h5py self.assertTrue(h5py.version.hdf5_version.startswith("1.10")) @@ -45,50 +46,58 @@ class TFAutoModelTest(unittest.TestCase): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - config = AutoConfig.from_pretrained(model_name, force_download=True) + config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModel.from_pretrained(model_name, force_download=True) + model = TFAutoModel.from_pretrained(model_name) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertModel) + @slow def test_lmhead_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - config = AutoConfig.from_pretrained(model_name, force_download=True) + config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModelWithLMHead.from_pretrained(model_name, force_download=True) + model = TFAutoModelWithLMHead.from_pretrained(model_name) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForMaskedLM) + @slow def test_sequence_classification_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - config = AutoConfig.from_pretrained(model_name, force_download=True) + config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModelForSequenceClassification.from_pretrained(model_name, force_download=True) + model = TFAutoModelForSequenceClassification.from_pretrained(model_name) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForSequenceClassification) + @slow def test_question_answering_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - config = AutoConfig.from_pretrained(model_name, force_download=True) + config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModelForQuestionAnswering.from_pretrained(model_name, force_download=True) + model = TFAutoModelForQuestionAnswering.from_pretrained(model_name) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForQuestionAnswering) + def test_from_pretrained_identifier(self): + logging.basicConfig(level=logging.INFO) + model = TFAutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER) + self.assertIsInstance(model, TFBertForMaskedLM) + if __name__ == "__main__": unittest.main() diff --git a/transformers/tests/modeling_tf_bert_test.py b/transformers/tests/modeling_tf_bert_test.py index bcee9743..abf20b15 100644 --- a/transformers/tests/modeling_tf_bert_test.py +++ b/transformers/tests/modeling_tf_bert_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow from transformers import BertConfig, is_tf_available @@ -36,10 +35,9 @@ if is_tf_available(): TFBertForTokenClassification, TFBertForQuestionAnswering, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFBertModel, TFBertForMaskedLM, TFBertForNextSentencePrediction, @@ -115,7 +113,7 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = BertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, @@ -309,13 +307,11 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - model = TFBertModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFBertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 2bb7cc9c..5a5873e8 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -25,18 +25,17 @@ import unittest import uuid import tempfile -import pytest import sys from transformers import is_tf_available, is_torch_available +from .utils import require_tf, slow + if is_tf_available(): import tensorflow as tf import numpy as np from transformers import TFPreTrainedModel # from transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require TensorFlow") if sys.version_info[0] == 2: import cPickle as pickle @@ -62,6 +61,7 @@ def _config_zero_init(config): class TFCommonTestCases: + @require_tf class TFCommonModelTester(unittest.TestCase): model_tester = None @@ -69,6 +69,7 @@ class TFCommonTestCases: test_torchscript = True test_pruning = True test_resize_embeddings = True + is_encoder_decoder = False def test_initialization(self): pass @@ -129,8 +130,12 @@ class TFCommonTestCases: for name, key in inputs_dict.items()) with torch.no_grad(): pto = pt_model(**pt_inputs_dict) - tfo = tf_model(inputs_dict) - max_diff = np.amax(np.abs(tfo[0].numpy() - pto[0].numpy())) + tfo = tf_model(inputs_dict, training=False) + tf_hidden_states = tfo[0].numpy() + pt_hidden_states = pto[0].numpy() + tf_hidden_states[np.isnan(tf_hidden_states)] = 0 + pt_hidden_states[np.isnan(pt_hidden_states)] = 0 + max_diff = np.amax(np.abs(tf_hidden_states - pt_hidden_states)) self.assertLessEqual(max_diff, 2e-2) # Check we can load pt model in tf and vice-versa with checkpoint => model functions @@ -150,13 +155,21 @@ class TFCommonTestCases: with torch.no_grad(): pto = pt_model(**pt_inputs_dict) tfo = tf_model(inputs_dict) - max_diff = np.amax(np.abs(tfo[0].numpy() - pto[0].numpy())) + tfo = tfo[0].numpy() + pto = pto[0].numpy() + tfo[np.isnan(tfo)] = 0 + pto[np.isnan(pto)] = 0 + max_diff = np.amax(np.abs(tfo - pto)) self.assertLessEqual(max_diff, 2e-2) def test_compile_tf_model(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - input_ids = tf.keras.Input(batch_shape=(2, 2000), name='input_ids', dtype='int32') + if self.is_encoder_decoder: + input_ids = {'decoder_input_ids': tf.keras.Input(batch_shape=(2, 2000), name='decoder_input_ids', dtype='int32'), + 'encoder_input_ids': tf.keras.Input(batch_shape=(2, 2000), name='encoder_input_ids', dtype='int32')} + else: + input_ids = tf.keras.Input(batch_shape=(2, 2000), name='input_ids', dtype='int32') optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0) loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') @@ -164,7 +177,7 @@ class TFCommonTestCases: for model_class in self.all_model_classes: # Prepare our model model = model_class(config) - + # Let's load it from the disk to be sure we can use pretrained weights with TemporaryDirectory() as tmpdirname: outputs = model(inputs_dict) # build the model @@ -189,7 +202,7 @@ class TFCommonTestCases: outputs_dict = model(inputs_dict) inputs_keywords = copy.deepcopy(inputs_dict) - input_ids = inputs_keywords.pop('input_ids') + input_ids = inputs_keywords.pop('input_ids' if not self.is_encoder_decoder else 'decoder_input_ids', None) outputs_keywords = model(input_ids, **inputs_keywords) output_dict = outputs_dict[0].numpy() @@ -200,6 +213,11 @@ class TFCommonTestCases: def test_attention_outputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + decoder_seq_length = self.model_tester.decoder_seq_length if hasattr(self.model_tester, 'decoder_seq_length') else self.model_tester.seq_length + encoder_seq_length = self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length + decoder_key_length = self.model_tester.key_length if hasattr(self.model_tester, 'key_length') else decoder_seq_length + encoder_key_length = self.model_tester.key_length if hasattr(self.model_tester, 'key_length') else encoder_seq_length + for model_class in self.all_model_classes: config.output_attentions = True config.output_hidden_states = False @@ -212,16 +230,28 @@ class TFCommonTestCases: self.assertListEqual( list(attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + encoder_seq_length, + encoder_key_length]) out_len = len(outputs) + if self.is_encoder_decoder: + self.assertEqual(out_len % 2, 0) + decoder_attentions = outputs[(out_len // 2)-1] + self.assertEqual(model.config.output_attentions, True) + self.assertEqual(model.config.output_hidden_states, False) + self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers) + self.assertListEqual( + list(decoder_attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, + decoder_seq_length, + decoder_key_length]) + # Check attention is always last and order is fine config.output_attentions = True config.output_hidden_states = True model = model_class(config) outputs = model(inputs_dict) - self.assertEqual(out_len+1, len(outputs)) + self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1), len(outputs)) self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, True) @@ -230,82 +260,8 @@ class TFCommonTestCases: self.assertListEqual( list(attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) - - def test_headmasking(self): - pass - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - # config.output_attentions = True - # config.output_hidden_states = True - # configs_no_init = _config_zero_init(config) # To be sure we have no Nan - # for model_class in self.all_model_classes: - # model = model_class(config=configs_no_init) - # model.eval() - - # # Prepare head_mask - # # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior) - # head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads) - # head_mask[0, 0] = 0 - # head_mask[-1, :-1] = 0 - # head_mask.requires_grad_(requires_grad=True) - # inputs = inputs_dict.copy() - # inputs['head_mask'] = head_mask - - # outputs = model(**inputs) - - # # Test that we can get a gradient back for importance score computation - # output = sum(t.sum() for t in outputs[0]) - # output = output.sum() - # output.backward() - # multihead_outputs = head_mask.grad - - # attentions = outputs[-1] - # hidden_states = outputs[-2] - - # # Remove Nan - - # self.assertIsNotNone(multihead_outputs) - # self.assertEqual(len(multihead_outputs), self.model_tester.num_hidden_layers) - # self.assertAlmostEqual( - # attentions[0][..., 0, :, :].flatten().sum().item(), 0.0) - # self.assertNotEqual( - # attentions[0][..., -1, :, :].flatten().sum().item(), 0.0) - # self.assertNotEqual( - # attentions[1][..., 0, :, :].flatten().sum().item(), 0.0) - # self.assertAlmostEqual( - # attentions[-1][..., -2, :, :].flatten().sum().item(), 0.0) - # self.assertNotEqual( - # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) - - - def test_head_pruning(self): - pass - # if not self.test_pruning: - # return - - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - # for model_class in self.all_model_classes: - # config.output_attentions = True - # config.output_hidden_states = False - # model = model_class(config=config) - # model.eval() - # heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), - # -1: [0]} - # model.prune_heads(heads_to_prune) - # outputs = model(**inputs_dict) - - # attentions = outputs[-1] - - # self.assertEqual( - # attentions[0].shape[-3], 1) - # self.assertEqual( - # attentions[1].shape[-3], self.model_tester.num_attention_heads) - # self.assertEqual( - # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) - + encoder_seq_length, + encoder_key_length]) def test_hidden_states_output(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -323,43 +279,6 @@ class TFCommonTestCases: list(hidden_states[0].shape[-2:]), [self.model_tester.seq_length, self.model_tester.hidden_size]) - - def test_resize_tokens_embeddings(self): - pass - # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - # if not self.test_resize_embeddings: - # return - - # for model_class in self.all_model_classes: - # config = copy.deepcopy(original_config) - # model = model_class(config) - - # model_vocab_size = config.vocab_size - # # Retrieve the embeddings and clone theme - # model_embed = model.resize_token_embeddings(model_vocab_size) - # cloned_embeddings = model_embed.weight.clone() - - # # Check that resizing the token embeddings with a larger vocab size increases the model's vocab size - # model_embed = model.resize_token_embeddings(model_vocab_size + 10) - # self.assertEqual(model.config.vocab_size, model_vocab_size + 10) - # # Check that it actually resizes the embeddings matrix - # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] + 10) - - # # Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size - # model_embed = model.resize_token_embeddings(model_vocab_size - 15) - # self.assertEqual(model.config.vocab_size, model_vocab_size - 15) - # # Check that it actually resizes the embeddings matrix - # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] - 15) - - # # Check that adding and removing tokens has not modified the first part of the embedding matrix. - # models_equal = True - # for p1, p2 in zip(cloned_embeddings, model_embed.weight): - # if p1.data.ne(p2.data).sum() > 0: - # models_equal = False - - # self.assertTrue(models_equal) - - def test_model_common_attributes(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -369,67 +288,59 @@ class TFCommonTestCases: x = model.get_output_embeddings() assert x is None or isinstance(x, tf.keras.layers.Layer) - - def test_tie_model_weights(self): - pass - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - # def check_same_values(layer_1, layer_2): - # equal = True - # for p1, p2 in zip(layer_1.weight, layer_2.weight): - # if p1.data.ne(p2.data).sum() > 0: - # equal = False - # return equal - - # for model_class in self.all_model_classes: - # if not hasattr(model_class, 'tie_weights'): - # continue - - # config.torchscript = True - # model_not_tied = model_class(config) - # params_not_tied = list(model_not_tied.parameters()) - - # config_tied = copy.deepcopy(config) - # config_tied.torchscript = False - # model_tied = model_class(config_tied) - # params_tied = list(model_tied.parameters()) - - # # Check that the embedding layer and decoding layer are the same in size and in value - # self.assertGreater(len(params_not_tied), len(params_tied)) - - # # Check that after resize they remain tied. - # model_tied.resize_token_embeddings(config.vocab_size + 10) - # params_tied_2 = list(model_tied.parameters()) - # self.assertGreater(len(params_not_tied), len(params_tied)) - # self.assertEqual(len(params_tied_2), len(params_tied)) - def test_determinism(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() for model_class in self.all_model_classes: model = model_class(config) first, second = model(inputs_dict, training=False)[0], model(inputs_dict, training=False)[0] - self.assertTrue(tf.math.equal(first, second).numpy().all()) + out_1 = first.numpy() + out_2 = second.numpy() + out_1 = out_1[~np.isnan(out_1)] + out_2 = out_2[~np.isnan(out_2)] + max_diff = np.amax(np.abs(out_1 - out_2)) + self.assertLessEqual(max_diff, 1e-5) + + def _get_embeds(self, wte, input_ids): + # ^^ In our TF models, the input_embeddings can take slightly different forms, + # so we try a few of them. + # We used to fall back to just synthetically creating a dummy tensor of ones: + try: + x = wte(input_ids, mode="embedding") + except: + try: + x = wte([input_ids], mode="embedding") + except: + try: + x = wte([input_ids, None, None, None], mode="embedding") + except: + if hasattr(self.model_tester, "embedding_size"): + x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) + else: + x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) + return x def test_inputs_embeds(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - input_ids = inputs_dict["input_ids"] - del inputs_dict["input_ids"] + if not self.is_encoder_decoder: + input_ids = inputs_dict["input_ids"] + del inputs_dict["input_ids"] + else: + encoder_input_ids = inputs_dict["encoder_input_ids"] + decoder_input_ids = inputs_dict["decoder_input_ids"] + del inputs_dict["encoder_input_ids"] + del inputs_dict["decoder_input_ids"] for model_class in self.all_model_classes: model = model_class(config) wte = model.get_input_embeddings() - try: - x = wte(input_ids, mode="embedding") - except: - try: - x = wte([input_ids], mode="embedding") - except: - x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) - # ^^ In our TF models, the input_embeddings can take slightly different forms, - # so we try two of them and fall back to just synthetically creating a dummy tensor of ones. - inputs_dict["inputs_embeds"] = x + if not self.is_encoder_decoder: + inputs_dict["inputs_embeds"] = self._get_embeds(wte, input_ids) + else: + inputs_dict["encoder_inputs_embeds"] = self._get_embeds(wte, encoder_input_ids) + inputs_dict["decoder_inputs_embeds"] = self._get_embeds(wte, decoder_input_ids) + outputs = model(inputs_dict) @@ -453,29 +364,5 @@ def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None): return output -class TFModelUtilsTest(unittest.TestCase): - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") - def test_model_from_pretrained(self): - pass - # logging.basicConfig(level=logging.INFO) - # for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - # config = BertConfig.from_pretrained(model_name) - # self.assertIsNotNone(config) - # self.assertIsInstance(config, PretrainedConfig) - - # model = BertModel.from_pretrained(model_name) - # model, loading_info = BertModel.from_pretrained(model_name, output_loading_info=True) - # self.assertIsNotNone(model) - # self.assertIsInstance(model, PreTrainedModel) - # for value in loading_info.values(): - # self.assertEqual(len(value), 0) - - # config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) - # model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) - # self.assertEqual(model.config.output_attentions, True) - # self.assertEqual(model.config.output_hidden_states, True) - # self.assertEqual(model.config, config) - - if __name__ == "__main__": unittest.main() diff --git a/transformers/tests/modeling_tf_ctrl_test.py b/transformers/tests/modeling_tf_ctrl_test.py index a57c8821..93b231e5 100644 --- a/transformers/tests/modeling_tf_ctrl_test.py +++ b/transformers/tests/modeling_tf_ctrl_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow from transformers import CTRLConfig, is_tf_available @@ -30,10 +29,9 @@ if is_tf_available(): import tensorflow as tf from transformers.modeling_tf_ctrl import (TFCTRLModel, TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFCTRLModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFCTRLModel, TFCTRLLMHeadModel) if is_tf_available() else () @@ -113,7 +111,7 @@ class TFCTRLModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = CTRLConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, @@ -188,12 +186,10 @@ class TFCTRLModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_ctrl_lm_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFCTRLModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFCTRLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_distilbert_test.py b/transformers/tests/modeling_tf_distilbert_test.py index e6d37959..f28b5c39 100644 --- a/transformers/tests/modeling_tf_distilbert_test.py +++ b/transformers/tests/modeling_tf_distilbert_test.py @@ -17,10 +17,10 @@ from __future__ import division from __future__ import print_function import unittest -import pytest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow from transformers import DistilBertConfig, is_tf_available @@ -30,10 +30,9 @@ if is_tf_available(): TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, TFDistilBertForSequenceClassification) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFDistilBertModel, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, @@ -108,7 +107,7 @@ class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = DistilBertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, dim=self.hidden_size, n_layers=self.num_hidden_layers, n_heads=self.num_attention_heads, @@ -210,12 +209,10 @@ class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_distilbert_for_sequence_classification(*config_and_inputs) - # @pytest.mark.slow + # @slow # def test_model_from_pretrained(self): - # cache_dir = "/tmp/transformers_test/" # for model_name in list(DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - # model = DistilBertModel.from_pretrained(model_name, cache_dir=cache_dir) - # shutil.rmtree(cache_dir) + # model = DistilBertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) # self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_gpt2_test.py b/transformers/tests/modeling_tf_gpt2_test.py index 76e9ee22..90920342 100644 --- a/transformers/tests/modeling_tf_gpt2_test.py +++ b/transformers/tests/modeling_tf_gpt2_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow from transformers import GPT2Config, is_tf_available @@ -31,10 +30,9 @@ if is_tf_available(): from transformers.modeling_tf_gpt2 import (TFGPT2Model, TFGPT2LMHeadModel, TFGPT2DoubleHeadsModel, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel, @@ -116,7 +114,7 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = GPT2Config( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, @@ -219,12 +217,10 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_gpt2_double_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFGPT2Model.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFGPT2Model.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_openai_gpt_test.py b/transformers/tests/modeling_tf_openai_gpt_test.py index d470c886..065bf2ac 100644 --- a/transformers/tests/modeling_tf_openai_gpt_test.py +++ b/transformers/tests/modeling_tf_openai_gpt_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow from transformers import OpenAIGPTConfig, is_tf_available @@ -31,10 +30,9 @@ if is_tf_available(): from transformers.modeling_tf_openai import (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, @@ -115,7 +113,7 @@ class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = OpenAIGPTConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, @@ -218,12 +216,10 @@ class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_openai_gpt_double_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFOpenAIGPTModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFOpenAIGPTModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_roberta_test.py b/transformers/tests/modeling_tf_roberta_test.py index edbfa4e2..93c478ae 100644 --- a/transformers/tests/modeling_tf_roberta_test.py +++ b/transformers/tests/modeling_tf_roberta_test.py @@ -17,11 +17,10 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow from transformers import RobertaConfig, is_tf_available @@ -32,10 +31,9 @@ if is_tf_available(): TFRobertaForSequenceClassification, TFRobertaForTokenClassification, TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFRobertaModel,TFRobertaForMaskedLM, @@ -110,7 +108,7 @@ class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = RobertaConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, @@ -191,22 +189,20 @@ class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_roberta_for_masked_lm(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFRobertaModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFRobertaModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) class TFRobertaModelIntegrationTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_inference_masked_lm(self): model = TFRobertaForMaskedLM.from_pretrained('roberta-base') - + input_ids = tf.constant([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] expected_shape = [1, 11, 50265] @@ -224,10 +220,10 @@ class TFRobertaModelIntegrationTest(unittest.TestCase): numpy.allclose(output[:, :3, :3].numpy(), expected_slice.numpy(), atol=1e-3) ) - @pytest.mark.slow + @slow def test_inference_no_head(self): model = TFRobertaModel.from_pretrained('roberta-base') - + input_ids = tf.constant([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] # compare the actual values for a slice. @@ -240,10 +236,10 @@ class TFRobertaModelIntegrationTest(unittest.TestCase): numpy.allclose(output[:, :3, :3].numpy(), expected_slice.numpy(), atol=1e-3) ) - @pytest.mark.slow + @slow def test_inference_classification_head(self): model = TFRobertaForSequenceClassification.from_pretrained('roberta-large-mnli') - + input_ids = tf.constant([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] expected_shape = [1, 3] diff --git a/transformers/tests/modeling_tf_t5_test.py b/transformers/tests/modeling_tf_t5_test.py new file mode 100644 index 00000000..da9ce6f8 --- /dev/null +++ b/transformers/tests/modeling_tf_t5_test.py @@ -0,0 +1,169 @@ +# coding=utf-8 +# Copyright 2018 Google T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import sys + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow + +from transformers import T5Config, is_tf_available + +if is_tf_available(): + import tensorflow as tf + from transformers.modeling_tf_t5 import (TFT5Model, TFT5WithLMHeadModel, + TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP) + + +@require_tf +class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): + + is_encoder_decoder = True + all_model_classes = (TFT5Model, TFT5WithLMHeadModel) if is_tf_available() else () + + class TFT5ModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_labels=True, + vocab_size=99, + n_positions=14, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + d_ff=37, + relative_attention_num_buckets=8, + dropout_rate=0.1, + initializer_factor=0.002, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_labels = use_labels + self.vocab_size = vocab_size + self.n_positions = n_positions + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.d_ff = d_ff + self.relative_attention_num_buckets = relative_attention_num_buckets + self.dropout_rate = dropout_rate + self.initializer_factor = initializer_factor + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_labels = None + if self.use_labels: + token_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + config = T5Config( + vocab_size=self.vocab_size, + n_positions=self.n_positions, + d_model=self.hidden_size, + d_ff=self.d_ff, + d_kv=self.hidden_size // self.num_attention_heads, + num_layers=self.num_hidden_layers, + num_heads=self.num_attention_heads, + relative_attention_num_buckets=self.relative_attention_num_buckets, + dropout_rate=self.dropout_rate, + initializer_factor=self.initializer_factor) + + return (config, input_ids, input_mask, token_labels) + + def create_and_check_t5_model(self, config, input_ids, input_mask, token_labels): + model = TFT5Model(config=config) + inputs = {'encoder_input_ids': input_ids, + 'decoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + encoder_output, decoder_output = model(inputs) + + encoder_output, decoder_output = model(input_ids, + decoder_attention_mask=input_mask, + encoder_input_ids=input_ids) + + result = { + "encoder_output": encoder_output.numpy(), + "decoder_output": decoder_output.numpy(), + } + self.parent.assertListEqual( + list(result["encoder_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual( + list(result["decoder_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + + + def create_and_check_t5_with_lm_head(self, config, input_ids, input_mask, token_labels): + model = TFT5WithLMHeadModel(config=config) + inputs = {'encoder_input_ids': input_ids, + 'decoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + prediction_scores, decoder_output = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, input_mask, token_labels) = config_and_inputs + inputs_dict = {'encoder_input_ids': input_ids, + 'decoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFT5ModelTest.TFT5ModelTester(self) + self.config_tester = ConfigTester(self, config_class=T5Config, d_model=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_t5_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_t5_model(*config_and_inputs) + + def test_with_lm_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_t5_with_lm_head(*config_and_inputs) + + @slow + def test_model_from_pretrained(self): + for model_name in ['t5-small']: + model = TFT5Model.from_pretrained(model_name, cache_dir=CACHE_DIR) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() diff --git a/transformers/tests/modeling_tf_transfo_xl_test.py b/transformers/tests/modeling_tf_transfo_xl_test.py index 534fe396..8225c092 100644 --- a/transformers/tests/modeling_tf_transfo_xl_test.py +++ b/transformers/tests/modeling_tf_transfo_xl_test.py @@ -18,11 +18,10 @@ from __future__ import print_function import unittest import random -import shutil -import pytest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow from transformers import TransfoXLConfig, is_tf_available @@ -31,10 +30,9 @@ if is_tf_available(): from transformers.modeling_tf_transfo_xl import (TFTransfoXLModel, TFTransfoXLLMHeadModel, TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFTransfoXLModel, TFTransfoXLLMHeadModel) if is_tf_available() else () @@ -68,7 +66,7 @@ class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): self.batch_size = batch_size self.seq_length = seq_length self.mem_len = mem_len - self.key_len = seq_length + mem_len + self.key_length = seq_length + mem_len self.clamp_len = clamp_len self.is_training = is_training self.use_labels = use_labels @@ -93,7 +91,7 @@ class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): lm_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) config = TransfoXLConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, mem_len=self.mem_len, clamp_len=self.clamp_len, cutoffs=self.cutoffs, @@ -204,12 +202,10 @@ class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_transfo_xl_lm_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFTransfoXLModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFTransfoXLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_tf_xlm_test.py b/transformers/tests/modeling_tf_xlm_test.py index 1bd661be..065d355b 100644 --- a/transformers/tests/modeling_tf_xlm_test.py +++ b/transformers/tests/modeling_tf_xlm_test.py @@ -17,8 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest from transformers import is_tf_available @@ -29,13 +27,13 @@ if is_tf_available(): TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple, TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow +@require_tf class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFXLMModel, TFXLMWithLMHeadModel, @@ -126,7 +124,7 @@ class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): is_impossible_labels = ids_tensor([self.batch_size], 2, dtype=tf.float32) config = XLMConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_special=self.n_special, emb_dim=self.hidden_size, n_layers=self.num_hidden_layers, @@ -251,12 +249,10 @@ class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlm_sequence_classif(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = XLMModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFXLMModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_tf_xlnet_test.py b/transformers/tests/modeling_tf_xlnet_test.py index 12a8fbe3..15fd9174 100644 --- a/transformers/tests/modeling_tf_xlnet_test.py +++ b/transformers/tests/modeling_tf_xlnet_test.py @@ -20,8 +20,6 @@ import os import unittest import json import random -import shutil -import pytest from transformers import XLNetConfig, is_tf_available @@ -30,18 +28,21 @@ if is_tf_available(): from transformers.modeling_tf_xlnet import (TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, + TFXLNetForTokenClassification, TFXLNetForQuestionAnsweringSimple, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_tf, slow + +@require_tf class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes=(TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, + TFXLNetForTokenClassification, TFXLNetForQuestionAnsweringSimple) if is_tf_available() else () test_pruning = False @@ -62,7 +63,6 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): num_attention_heads=4, d_inner=128, num_hidden_layers=5, - max_position_embeddings=10, type_sequence_label_size=2, untie_r=True, bi_data=False, @@ -86,7 +86,6 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): self.num_attention_heads = num_attention_heads self.d_inner = d_inner self.num_hidden_layers = num_hidden_layers - self.max_position_embeddings = max_position_embeddings self.bi_data = bi_data self.untie_r = untie_r self.same_length = same_length @@ -120,13 +119,12 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): is_impossible_labels = ids_tensor([self.batch_size], 2, dtype=tf.float32) config = XLNetConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, d_model=self.hidden_size, n_head=self.num_attention_heads, d_inner=self.d_inner, n_layer=self.num_hidden_layers, untie_r=self.untie_r, - max_position_embeddings=self.max_position_embeddings, mem_len=self.mem_len, clamp_len=self.clamp_len, same_length=self.same_length, @@ -258,6 +256,26 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): list(list(mem.shape) for mem in result["mems_1"]), [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + def create_and_check_xlnet_for_token_classification(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + config.num_labels = input_ids_1.shape[1] + model = TFXLNetForTokenClassification(config) + inputs = {'input_ids': input_ids_1, + 'attention_mask': input_mask, + # 'token_type_ids': token_type_ids + } + logits, mems_1 = model(inputs) + result = { + "mems_1": [mem.numpy() for mem in mems_1], + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.seq_length, config.num_labels]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_1"]), + [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, @@ -282,24 +300,26 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): def test_xlnet_lm_head(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_xlnet_lm_head(*config_and_inputs) + self.model_tester.create_and_check_xlnet_lm_head(*config_and_inputs) def test_xlnet_sequence_classif(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_sequence_classif(*config_and_inputs) + def test_xlnet_token_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlnet_for_token_classification(*config_and_inputs) + def test_xlnet_qa(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_qa(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFXLNetModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFXLNetModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_transfo_xl_test.py b/transformers/tests/modeling_transfo_xl_test.py index f7b913da..acbe95fe 100644 --- a/transformers/tests/modeling_transfo_xl_test.py +++ b/transformers/tests/modeling_transfo_xl_test.py @@ -18,8 +18,6 @@ from __future__ import print_function import unittest import random -import shutil -import pytest from transformers import is_torch_available @@ -27,12 +25,13 @@ if is_torch_available(): import torch from transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) from transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device + +@require_torch class TransfoXLModelTest(CommonTestCases.CommonModelTester): all_model_classes = (TransfoXLModel, TransfoXLLMHeadModel) if is_torch_available() else () @@ -66,7 +65,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): self.batch_size = batch_size self.seq_length = seq_length self.mem_len = mem_len - self.key_len = seq_length + mem_len + self.key_length = seq_length + mem_len self.clamp_len = clamp_len self.is_training = is_training self.use_labels = use_labels @@ -91,7 +90,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): lm_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) config = TransfoXLConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, mem_len=self.mem_len, clamp_len=self.clamp_len, cutoffs=self.cutoffs, @@ -111,6 +110,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): def create_transfo_xl_model(self, config, input_ids_1, input_ids_2, lm_labels): model = TransfoXLModel(config) + model.to(torch_device) model.eval() hidden_states_1, mems_1 = model(input_ids_1) @@ -140,6 +140,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): def create_transfo_xl_lm_head(self, config, input_ids_1, input_ids_2, lm_labels): model = TransfoXLLMHeadModel(config) + model.to(torch_device) model.eval() lm_logits_1, mems_1 = model(input_ids_1) @@ -204,12 +205,10 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): output_result = self.model_tester.create_transfo_xl_lm_head(*config_and_inputs) self.model_tester.check_transfo_xl_lm_head_output(output_result) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TransfoXLModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TransfoXLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_xlm_test.py b/transformers/tests/modeling_xlm_test.py index 0133febb..fcc2f469 100644 --- a/transformers/tests/modeling_xlm_test.py +++ b/transformers/tests/modeling_xlm_test.py @@ -17,8 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import pytest from transformers import is_torch_available @@ -26,13 +24,13 @@ if is_torch_available(): from transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, XLMForSequenceClassification, XLMForQuestionAnsweringSimple) from transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device +@require_torch class XLMModelTest(CommonTestCases.CommonModelTester): all_model_classes = (XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, @@ -122,7 +120,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): is_impossible_labels = ids_tensor([self.batch_size], 2).float() config = XLMConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_special=self.n_special, emb_dim=self.hidden_size, n_layers=self.num_hidden_layers, @@ -148,6 +146,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_model(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMModel(config=config) + model.to(torch_device) model.eval() outputs = model(input_ids, lengths=input_lengths, langs=token_type_ids) outputs = model(input_ids, langs=token_type_ids) @@ -163,6 +162,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_lm_head(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMWithLMHeadModel(config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, token_type_ids=token_type_ids, labels=token_labels) @@ -182,6 +182,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_simple_qa(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMForQuestionAnsweringSimple(config) + model.to(torch_device) model.eval() outputs = model(input_ids) @@ -206,6 +207,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_qa(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMForQuestionAnswering(config) + model.to(torch_device) model.eval() outputs = model(input_ids) @@ -260,6 +262,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_sequence_classif(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMForSequenceClassification(config) + model.to(torch_device) model.eval() (logits,) = model(input_ids) @@ -312,12 +315,10 @@ class XLMModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlm_sequence_classif(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = XLMModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = XLMModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_xlnet_test.py b/transformers/tests/modeling_xlnet_test.py index d97ea6a4..6d218d6e 100644 --- a/transformers/tests/modeling_xlnet_test.py +++ b/transformers/tests/modeling_xlnet_test.py @@ -20,25 +20,25 @@ import os import unittest import json import random -import shutil -import pytest from transformers import is_torch_available if is_torch_available(): import torch - from transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) + from transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, + XLNetForTokenClassification, XLNetForQuestionAnswering) from transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import CACHE_DIR, require_torch, slow, torch_device + +@require_torch class XLNetModelTest(CommonTestCases.CommonModelTester): - all_model_classes=(XLNetModel, XLNetLMHeadModel, + all_model_classes=(XLNetModel, XLNetLMHeadModel, XLNetForTokenClassification, XLNetForSequenceClassification, XLNetForQuestionAnswering) if is_torch_available() else () test_pruning = False @@ -59,7 +59,6 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): num_attention_heads=4, d_inner=128, num_hidden_layers=5, - max_position_embeddings=10, type_sequence_label_size=2, untie_r=True, bi_data=False, @@ -83,7 +82,6 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): self.num_attention_heads = num_attention_heads self.d_inner = d_inner self.num_hidden_layers = num_hidden_layers - self.max_position_embeddings = max_position_embeddings self.bi_data = bi_data self.untie_r = untie_r self.same_length = same_length @@ -99,27 +97,28 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): input_mask = ids_tensor([self.batch_size, self.seq_length], 2).float() input_ids_q = ids_tensor([self.batch_size, self.seq_length + 1], self.vocab_size) - perm_mask = torch.zeros(self.batch_size, self.seq_length + 1, self.seq_length + 1, dtype=torch.float) + perm_mask = torch.zeros(self.batch_size, self.seq_length + 1, self.seq_length + 1, dtype=torch.float, device=torch_device) perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token - target_mapping = torch.zeros(self.batch_size, 1, self.seq_length + 1, dtype=torch.float) + target_mapping = torch.zeros(self.batch_size, 1, self.seq_length + 1, dtype=torch.float, device=torch_device) target_mapping[:, 0, -1] = 1.0 # predict last token sequence_labels = None lm_labels = None is_impossible_labels = None + token_labels = None if self.use_labels: lm_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) is_impossible_labels = ids_tensor([self.batch_size], 2).float() + token_labels = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) config = XLNetConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, d_model=self.hidden_size, n_head=self.num_attention_heads, d_inner=self.d_inner, n_layer=self.num_hidden_layers, untie_r=self.untie_r, - max_position_embeddings=self.max_position_embeddings, mem_len=self.mem_len, clamp_len=self.clamp_len, same_length=self.same_length, @@ -129,15 +128,16 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): num_labels=self.type_sequence_label_size) return (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels) + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels) def set_seed(self): random.seed(self.seed) torch.manual_seed(self.seed) def create_and_check_xlnet_base_model(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetModel(config) + model.to(torch_device) model.eval() _, _ = model(input_ids_1, input_mask=input_mask) @@ -152,6 +152,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): config.mem_len = 0 model = XLNetModel(config) + model.to(torch_device) model.eval() no_mems_outputs = model(input_ids_1) self.parent.assertEqual(len(no_mems_outputs), 1) @@ -163,9 +164,23 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): list(list(mem.size()) for mem in result["mems_1"]), [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + def create_and_check_xlnet_base_model_with_att_output(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): + model = XLNetModel(config) + model.to(torch_device) + model.eval() + + _, _, attentions = model(input_ids_1, target_mapping=target_mapping) + + self.parent.assertEqual(len(attentions), config.n_layer) + self.parent.assertIsInstance(attentions[0], tuple) + self.parent.assertEqual(len(attentions[0]), 2) + self.parent.assertTrue(attentions[0][0].shape, attentions[0][0].shape) + def create_and_check_xlnet_lm_head(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetLMHeadModel(config) + model.to(torch_device) model.eval() loss_1, all_logits_1, mems_1 = model(input_ids_1, token_type_ids=segment_ids, labels=lm_labels) @@ -204,8 +219,9 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): [[self.mem_len, self.batch_size, self.hidden_size]] * self.num_hidden_layers) def create_and_check_xlnet_qa(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetForQuestionAnswering(config) + model.to(torch_device) model.eval() outputs = model(input_ids_1) @@ -261,9 +277,43 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): list(list(mem.size()) for mem in result["mems"]), [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + def create_and_check_xlnet_token_classif(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): + model = XLNetForTokenClassification(config) + model.to(torch_device) + model.eval() + + logits, mems_1 = model(input_ids_1) + loss, logits, mems_1 = model(input_ids_1, labels=token_labels) + + result = { + "loss": loss, + "mems_1": mems_1, + "logits": logits, + } + + self.parent.assertListEqual( + list(result["loss"].size()), + []) + self.parent.assertListEqual( + list(result["logits"].size()), + [self.batch_size, self.seq_length, self.type_sequence_label_size]) + self.parent.assertListEqual( + list(list(mem.size()) for mem in result["mems_1"]), + [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, + sequence_labels, is_impossible_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids_1} + return config, inputs_dict + def create_and_check_xlnet_sequence_classif(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetForSequenceClassification(config) + model.to(torch_device) model.eval() logits, mems_1 = model(input_ids_1) @@ -289,7 +339,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.prepare_config_and_inputs() (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, - sequence_labels, is_impossible_labels) = config_and_inputs + sequence_labels, is_impossible_labels, token_labels) = config_and_inputs inputs_dict = {'input_ids': input_ids_1} return config, inputs_dict @@ -306,27 +356,36 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_base_model(*config_and_inputs) + def test_xlnet_base_model_with_att_output(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + config_and_inputs[0].output_attentions = True + self.model_tester.create_and_check_xlnet_base_model_with_att_output(*config_and_inputs) + def test_xlnet_lm_head(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_xlnet_lm_head(*config_and_inputs) + self.model_tester.create_and_check_xlnet_lm_head(*config_and_inputs) def test_xlnet_sequence_classif(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_sequence_classif(*config_and_inputs) + def test_xlnet_token_classif(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlnet_token_classif(*config_and_inputs) + def test_xlnet_qa(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_qa(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = XLNetModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = XLNetModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/optimization_test.py b/transformers/tests/optimization_test.py index ab9afbfc..cc10ad59 100644 --- a/transformers/tests/optimization_test.py +++ b/transformers/tests/optimization_test.py @@ -18,7 +18,6 @@ from __future__ import print_function import unittest import os -import pytest from transformers import is_torch_available @@ -31,10 +30,9 @@ if is_torch_available(): get_cosine_schedule_with_warmup, get_cosine_with_hard_restarts_schedule_with_warmup, get_linear_schedule_with_warmup) -else: - pytestmark = pytest.mark.skip("Require Torch") from .tokenization_tests_commons import TemporaryDirectory +from .utils import require_torch def unwrap_schedule(scheduler, num_steps=10): @@ -58,6 +56,7 @@ def unwrap_and_save_reload_schedule(scheduler, num_steps=10): scheduler.load_state_dict(state_dict) return lrs +@require_torch class OptimizationTest(unittest.TestCase): def assertListAlmostEqual(self, list1, list2, tol): @@ -80,6 +79,7 @@ class OptimizationTest(unittest.TestCase): self.assertListAlmostEqual(w.tolist(), [0.4, 0.2, -0.5], tol=1e-2) +@require_torch class ScheduleInitTest(unittest.TestCase): m = torch.nn.Linear(50, 50) if is_torch_available() else None optimizer = AdamW(m.parameters(), lr=10.) if is_torch_available() else None diff --git a/transformers/tests/optimization_tf_test.py b/transformers/tests/optimization_tf_test.py new file mode 100644 index 00000000..515d12a1 --- /dev/null +++ b/transformers/tests/optimization_tf_test.py @@ -0,0 +1,90 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest + +from transformers import is_tf_available + +from .utils import require_tf + +if is_tf_available(): + import tensorflow as tf + from tensorflow.python.eager import context + from tensorflow.python.framework import ops + from transformers import (create_optimizer, GradientAccumulator) + + +@require_tf +class OptimizationFTest(unittest.TestCase): + def assertListAlmostEqual(self, list1, list2, tol): + self.assertEqual(len(list1), len(list2)) + for a, b in zip(list1, list2): + self.assertAlmostEqual(a, b, delta=tol) + + def testGradientAccumulator(self): + accumulator = GradientAccumulator() + accumulator([tf.constant([1.0, 2.0])]) + accumulator([tf.constant([-2.0, 1.0])]) + accumulator([tf.constant([-1.0, 2.0])]) + with self.assertRaises(ValueError): + accumulator([tf.constant([1.0, 1.0]), tf.constant([2.0, 2.0])]) + self.assertEqual(accumulator.step, 3) + self.assertEqual(len(accumulator.gradients), 1) + self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist(), [-2.0, 5.0], tol=1e-2) + accumulator.reset() + self.assertEqual(accumulator.step, 0) + self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist(), [0.0, 0.0], tol=1e-2) + + def testGradientAccumulatorDistributionStrategy(self): + context._context = None + ops.enable_eager_execution_internal() + physical_devices = tf.config.experimental.list_physical_devices("CPU") + tf.config.experimental.set_virtual_device_configuration( + physical_devices[0], + [tf.config.experimental.VirtualDeviceConfiguration(), + tf.config.experimental.VirtualDeviceConfiguration()]) + + devices = tf.config.experimental.list_logical_devices(device_type="CPU") + strategy = tf.distribute.MirroredStrategy(devices=[device.name for device in devices]) + + with strategy.scope(): + accumulator = GradientAccumulator() + variable = tf.Variable([4.0, 3.0]) + optimizer = create_optimizer(5e-5, 10, 5) + gradient_placeholder = tf.Variable([0.0, 0.0], trainable=False) + + def accumulate_on_replica(gradient): + accumulator([gradient]) + + def apply_on_replica(): + optimizer.apply_gradients(list(zip(accumulator.gradients, [variable])), 1.0) + + @tf.function + def accumulate(grad1, grad2): + with strategy.scope(): + gradient_placeholder.values[0].assign(grad1) + gradient_placeholder.values[1].assign(grad2) + strategy.experimental_run_v2(accumulate_on_replica, args=(gradient_placeholder,)) + + @tf.function + def apply_grad(): + with strategy.scope(): + strategy.experimental_run_v2(apply_on_replica) + + accumulate([1.0, 2.0], [-1.0, 1.0]) + accumulate([3.0, -1.0], [-1.0, -1.0]) + accumulate([-2.0, 2.0], [3.0, -2.0]) + self.assertEqual(accumulator.step, 3) + self.assertListAlmostEqual(accumulator._gradients[0].values[0].value().numpy().tolist(), [2.0, 3.0], tol=1e-2) + self.assertListAlmostEqual(accumulator._gradients[0].values[1].value().numpy().tolist(), [1.0, -2.0], tol=1e-2) + apply_grad() + self.assertListAlmostEqual(variable.value().numpy().tolist(), [4.0, 3.0], tol=1e-2) + accumulator.reset() + self.assertEqual(accumulator.step, 0) + self.assertListAlmostEqual(accumulator._gradients[0].values[0].value().numpy().tolist(), [0.0, 0.0], tol=1e-2) + self.assertListAlmostEqual(accumulator._gradients[0].values[1].value().numpy().tolist(), [0.0, 0.0], tol=1e-2) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/transformers/tests/pipelines_test.py b/transformers/tests/pipelines_test.py new file mode 100644 index 00000000..08a15077 --- /dev/null +++ b/transformers/tests/pipelines_test.py @@ -0,0 +1,210 @@ +import unittest + +from typing import Iterable + +from transformers import pipeline +from transformers.tests.utils import require_tf, require_torch + +QA_FINETUNED_MODELS = { + ('bert-base-uncased', 'bert-large-uncased-whole-word-masking-finetuned-squad', None), + ('bert-base-cased', 'bert-large-cased-whole-word-masking-finetuned-squad', None), + ('bert-base-uncased', 'distilbert-base-uncased-distilled-squad', None) +} + +TF_QA_FINETUNED_MODELS = { + ('bert-base-uncased', 'bert-large-uncased-whole-word-masking-finetuned-squad', None), + ('bert-base-cased', 'bert-large-cased-whole-word-masking-finetuned-squad', None), + ('bert-base-uncased', 'distilbert-base-uncased-distilled-squad', None) +} + +TF_NER_FINETUNED_MODELS = { + ( + 'bert-base-cased', + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-tf_model.h5', + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json' + ) +} + +NER_FINETUNED_MODELS = { + ( + 'bert-base-cased', + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-pytorch_model.bin', + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json' + ) +} + +FEATURE_EXTRACT_FINETUNED_MODELS = { + ('bert-base-cased', 'bert-base-cased', None), + # ('xlnet-base-cased', 'xlnet-base-cased', None), # Disabled for now as it crash for TF2 + ('distilbert-base-uncased', 'distilbert-base-uncased', None) +} + +TF_FEATURE_EXTRACT_FINETUNED_MODELS = { + ('bert-base-cased', 'bert-base-cased', None), + # ('xlnet-base-cased', 'xlnet-base-cased', None), # Disabled for now as it crash for TF2 + ('distilbert-base-uncased', 'distilbert-base-uncased', None) +} + +TF_TEXT_CLASSIF_FINETUNED_MODELS = { + ( + 'bert-base-uncased', + 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-tf_model.h5', + 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json' + ) +} + +TEXT_CLASSIF_FINETUNED_MODELS = { + ( + 'bert-base-uncased', + 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin', + 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json' + ) +} + + +class MonoColumnInputTestCase(unittest.TestCase): + def _test_mono_column_pipeline(self, nlp, valid_inputs: list, invalid_inputs: list, output_keys: Iterable[str]): + self.assertIsNotNone(nlp) + + mono_result = nlp(valid_inputs[0]) + self.assertIsInstance(mono_result, list) + self.assertIsInstance(mono_result[0], (dict, list)) + + if isinstance(mono_result[0], list): + mono_result = mono_result[0] + + for key in output_keys: + self.assertIn(key, mono_result[0]) + + multi_result = nlp(valid_inputs) + self.assertIsInstance(multi_result, list) + self.assertIsInstance(multi_result[0], (dict, list)) + + if isinstance(multi_result[0], list): + multi_result = multi_result[0] + + for result in multi_result: + for key in output_keys: + self.assertIn(key, result) + + self.assertRaises(Exception, nlp, invalid_inputs) + + @require_torch + def test_ner(self): + mandatory_keys = {'entity', 'word', 'score'} + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in NER_FINETUNED_MODELS: + nlp = pipeline(task='ner', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + + @require_tf + def test_tf_ner(self): + mandatory_keys = {'entity', 'word', 'score'} + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in TF_NER_FINETUNED_MODELS: + nlp = pipeline(task='ner', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + + @require_torch + def test_sentiment_analysis(self): + mandatory_keys = {'label'} + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in TEXT_CLASSIF_FINETUNED_MODELS: + nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + + @require_tf + def test_tf_sentiment_analysis(self): + mandatory_keys = {'label'} + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in TF_TEXT_CLASSIF_FINETUNED_MODELS: + nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + + @require_torch + def test_features_extraction(self): + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in FEATURE_EXTRACT_FINETUNED_MODELS: + nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, {}) + + @require_tf + def test_tf_features_extraction(self): + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in TF_FEATURE_EXTRACT_FINETUNED_MODELS: + nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, {}) + + +class MultiColumnInputTestCase(unittest.TestCase): + def _test_multicolumn_pipeline(self, nlp, valid_inputs: list, invalid_inputs: list, output_keys: Iterable[str]): + self.assertIsNotNone(nlp) + + mono_result = nlp(valid_inputs[0]) + self.assertIsInstance(mono_result, dict) + + for key in output_keys: + self.assertIn(key, mono_result) + + multi_result = nlp(valid_inputs) + self.assertIsInstance(multi_result, list) + self.assertIsInstance(multi_result[0], dict) + + for result in multi_result: + for key in output_keys: + self.assertIn(key, result) + + self.assertRaises(Exception, nlp, invalid_inputs[0]) + self.assertRaises(Exception, nlp, invalid_inputs) + + @require_torch + def test_question_answering(self): + mandatory_output_keys = {'score', 'answer', 'start', 'end'} + valid_samples = [ + {'question': 'Where was HuggingFace founded ?', 'context': 'HuggingFace was founded in Paris.'}, + { + 'question': 'In what field is HuggingFace working ?', + 'context': 'HuggingFace is a startup based in New-York founded in Paris which is trying to solve NLP.' + } + ] + invalid_samples = [ + {'question': '', 'context': 'This is a test to try empty question edge case'}, + {'question': None, 'context': 'This is a test to try empty question edge case'}, + {'question': 'What is does with empty context ?', 'context': ''}, + {'question': 'What is does with empty context ?', 'context': None}, + ] + + for tokenizer, model, config in QA_FINETUNED_MODELS: + nlp = pipeline(task='question-answering', model=model, config=config, tokenizer=tokenizer) + self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys) + + @require_tf + def test_tf_question_answering(self): + mandatory_output_keys = {'score', 'answer', 'start', 'end'} + valid_samples = [ + {'question': 'Where was HuggingFace founded ?', 'context': 'HuggingFace was founded in Paris.'}, + { + 'question': 'In what field is HuggingFace working ?', + 'context': 'HuggingFace is a startup based in New-York founded in Paris which is trying to solve NLP.' + } + ] + invalid_samples = [ + {'question': '', 'context': 'This is a test to try empty question edge case'}, + {'question': None, 'context': 'This is a test to try empty question edge case'}, + {'question': 'What is does with empty context ?', 'context': ''}, + {'question': 'What is does with empty context ?', 'context': None}, + ] + + for tokenizer, model, config in TF_QA_FINETUNED_MODELS: + nlp = pipeline(task='question-answering', model=model, config=config, tokenizer=tokenizer) + self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys) + + +if __name__ == '__main__': + unittest.main() diff --git a/transformers/tests/tokenization_albert_test.py b/transformers/tests/tokenization_albert_test.py new file mode 100644 index 00000000..59eb3bce --- /dev/null +++ b/transformers/tests/tokenization_albert_test.py @@ -0,0 +1,78 @@ +# coding=utf-8 +# Copyright 2019 Hugging Face inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import unittest + +from transformers.tokenization_albert import (AlbertTokenizer, SPIECE_UNDERLINE) + +from .tokenization_tests_commons import CommonTestCases + +SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'fixtures/spiece.model') + +class AlbertTokenizationTest(CommonTestCases.CommonTokenizerTester): + + tokenizer_class = AlbertTokenizer + + def setUp(self): + super(AlbertTokenizationTest, self).setUp() + + # We have a SentencePiece fixture for testing + tokenizer = AlbertTokenizer(SAMPLE_VOCAB) + tokenizer.save_pretrained(self.tmpdirname) + + def get_tokenizer(self, **kwargs): + return AlbertTokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def get_input_output_texts(self): + input_text = u"this is a test" + output_text = u"this is a test" + return input_text, output_text + + + def test_full_tokenizer(self): + tokenizer = AlbertTokenizer(SAMPLE_VOCAB, keep_accents=True) + + tokens = tokenizer.tokenize(u'This is a test') + self.assertListEqual(tokens, [u'▁this', u'▁is', u'▁a', u'▁test']) + + self.assertListEqual( + tokenizer.convert_tokens_to_ids(tokens), [48, 25, 21, 1289]) + + tokens = tokenizer.tokenize(u"I was born in 92000, and this is falsé.") + self.assertListEqual(tokens, [u'▁i', u'▁was', u'▁born', u'▁in', u'▁9', u'2000', u',', u'▁and', u'▁this', u'▁is', u'▁fal', u's', u'é', u'.']) + ids = tokenizer.convert_tokens_to_ids(tokens) + self.assertListEqual(ids, [31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]) + + back_tokens = tokenizer.convert_ids_to_tokens(ids) + self.assertListEqual(back_tokens, ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '', '.']) + + def test_sequence_builders(self): + tokenizer = AlbertTokenizer(SAMPLE_VOCAB) + + text = tokenizer.encode("sequence builders") + text_2 = tokenizer.encode("multi-sequence build") + + encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_2 + [tokenizer.sep_token_id] + + +if __name__ == '__main__': + unittest.main() diff --git a/transformers/tests/tokenization_auto_test.py b/transformers/tests/tokenization_auto_test.py index 79370811..0a894cac 100644 --- a/transformers/tests/tokenization_auto_test.py +++ b/transformers/tests/tokenization_auto_test.py @@ -18,15 +18,16 @@ from __future__ import print_function import unittest import shutil -import pytest import logging from transformers import AutoTokenizer, BertTokenizer, AutoTokenizer, GPT2Tokenizer from transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP +from .utils import slow, SMALL_MODEL_IDENTIFIER + class AutoTokenizerTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_tokenizer_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]: @@ -41,6 +42,11 @@ class AutoTokenizerTest(unittest.TestCase): self.assertIsInstance(tokenizer, GPT2Tokenizer) self.assertGreater(len(tokenizer), 0) + def test_tokenizer_from_pretrained_identifier(self): + logging.basicConfig(level=logging.INFO) + tokenizer = AutoTokenizer.from_pretrained(SMALL_MODEL_IDENTIFIER) + self.assertIsInstance(tokenizer, BertTokenizer) + self.assertEqual(len(tokenizer), 12) if __name__ == "__main__": unittest.main() diff --git a/transformers/tests/tokenization_bert_japanese_test.py b/transformers/tests/tokenization_bert_japanese_test.py new file mode 100644 index 00000000..545193c7 --- /dev/null +++ b/transformers/tests/tokenization_bert_japanese_test.py @@ -0,0 +1,191 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import unittest +from io import open + +from transformers.tokenization_bert import WordpieceTokenizer +from transformers.tokenization_bert_japanese import (BertJapaneseTokenizer, + MecabTokenizer, CharacterTokenizer, + VOCAB_FILES_NAMES) + +from .tokenization_tests_commons import CommonTestCases +from .utils import slow, custom_tokenizers + + +@custom_tokenizers +class BertJapaneseTokenizationTest(CommonTestCases.CommonTokenizerTester): + + tokenizer_class = BertJapaneseTokenizer + + def setUp(self): + super(BertJapaneseTokenizationTest, self).setUp() + + vocab_tokens = [u"[UNK]", u"[CLS]", u"[SEP]", + u"こんにちは", u"こん", u"にちは", u"ばんは", u"##こん", u"##にちは", u"##ばんは", + u"世界", u"##世界", u"、", u"##、", u"。", u"##。"] + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) + + def get_tokenizer(self, **kwargs): + return BertJapaneseTokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def get_input_output_texts(self): + input_text = u"こんにちは、世界。 \nこんばんは、世界。" + output_text = u"こんにちは 、 世界 。 こんばんは 、 世界 。" + return input_text, output_text + + def test_full_tokenizer(self): + tokenizer = self.tokenizer_class(self.vocab_file) + + tokens = tokenizer.tokenize(u"こんにちは、世界。\nこんばんは、世界。") + self.assertListEqual(tokens, + [u"こんにちは", u"、", u"世界", u"。", + u"こん", u"##ばんは", u"、", u"世界", "。"]) + self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), + [3, 12, 10, 14, 4, 9, 12, 10, 14]) + + def test_mecab_tokenizer(self): + tokenizer = MecabTokenizer() + + self.assertListEqual( + tokenizer.tokenize(u" \tｱｯﾌﾟﾙストアでiPhone８ が \n 発売された　。 "), + [u"アップルストア", u"で", u"iPhone", u"8", u"が", + u"発売", u"さ", u"れ", u"た", u"。"]) + + def test_mecab_tokenizer_lower(self): + tokenizer = MecabTokenizer(do_lower_case=True) + + self.assertListEqual( + tokenizer.tokenize(u" \tｱｯﾌﾟﾙストアでiPhone８ が \n 発売された　。 "), + [u"アップルストア", u"で", u"iphone", u"8", u"が", + u"発売", u"さ", u"れ", u"た", u"。"]) + + def test_mecab_tokenizer_no_normalize(self): + tokenizer = MecabTokenizer(normalize_text=False) + + self.assertListEqual( + tokenizer.tokenize(u" \tｱｯﾌﾟﾙストアでiPhone８ が \n 発売された　。 "), + [u"ｱｯﾌﾟﾙストア", u"で", u"iPhone", u"８", u"が", + u"発売", u"さ", u"れ", u"た", u"　", u"。"]) + + def test_wordpiece_tokenizer(self): + vocab_tokens = [u"[UNK]", u"[CLS]", u"[SEP]", + u"こんにちは", u"こん", u"にちは" u"ばんは", u"##こん", u"##にちは", u"##ばんは"] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + tokenizer = WordpieceTokenizer(vocab=vocab, unk_token=u"[UNK]") + + self.assertListEqual(tokenizer.tokenize(u""), []) + + self.assertListEqual(tokenizer.tokenize(u"こんにちは"), + [u"こんにちは"]) + + self.assertListEqual(tokenizer.tokenize(u"こんばんは"), + [u"こん", u"##ばんは"]) + + self.assertListEqual(tokenizer.tokenize(u"こんばんは こんばんにちは こんにちは"), + [u"こん", u"##ばんは", u"[UNK]", u"こんにちは"]) + + @slow + def test_sequence_builders(self): + tokenizer = self.tokenizer_class.from_pretrained("bert-base-japanese") + + text = tokenizer.encode(u"ありがとう。", add_special_tokens=False) + text_2 = tokenizer.encode(u"どういたしまして。", add_special_tokens=False) + + encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + # 2 is for "[CLS]", 3 is for "[SEP]" + assert encoded_sentence == [2] + text + [3] + assert encoded_pair == [2] + text + [3] + text_2 + [3] + + +class BertJapaneseCharacterTokenizationTest(CommonTestCases.CommonTokenizerTester): + + tokenizer_class = BertJapaneseTokenizer + + def setUp(self): + super(BertJapaneseCharacterTokenizationTest, self).setUp() + + vocab_tokens = [u"[UNK]", u"[CLS]", u"[SEP]", + u"こ", u"ん", u"に", u"ち", u"は", u"ば", u"世", u"界", u"、", u"。"] + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) + + def get_tokenizer(self, **kwargs): + return BertJapaneseTokenizer.from_pretrained(self.tmpdirname, + subword_tokenizer_type="character", + **kwargs) + + def get_input_output_texts(self): + input_text = u"こんにちは、世界。 \nこんばんは、世界。" + output_text = u"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。" + return input_text, output_text + + def test_full_tokenizer(self): + tokenizer = self.tokenizer_class(self.vocab_file, + subword_tokenizer_type="character") + + tokens = tokenizer.tokenize(u"こんにちは、世界。 \nこんばんは、世界。") + self.assertListEqual(tokens, + [u"こ", u"ん", u"に", u"ち", u"は", u"、", u"世", u"界", u"。", + u"こ", u"ん", u"ば", u"ん", u"は", u"、", u"世", u"界", u"。"]) + self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), + [3, 4, 5, 6, 7, 11, 9, 10, 12, + 3, 4, 8, 4, 7, 11, 9, 10, 12]) + + def test_character_tokenizer(self): + vocab_tokens = [u"[UNK]", u"[CLS]", u"[SEP]", + u"こ", u"ん", u"に", u"ち", u"は", u"ば", u"世", u"界"u"、", u"。"] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + tokenizer = CharacterTokenizer(vocab=vocab, unk_token=u"[UNK]") + + self.assertListEqual(tokenizer.tokenize(u""), []) + + self.assertListEqual(tokenizer.tokenize(u"こんにちは"), + [u"こ", u"ん", u"に", u"ち", u"は"]) + + self.assertListEqual(tokenizer.tokenize(u"こんにちほ"), + [u"こ", u"ん", u"に", u"ち", u"[UNK]"]) + + @slow + def test_sequence_builders(self): + tokenizer = self.tokenizer_class.from_pretrained("bert-base-japanese-char") + + text = tokenizer.encode(u"ありがとう。", add_special_tokens=False) + text_2 = tokenizer.encode(u"どういたしまして。", add_special_tokens=False) + + encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + # 2 is for "[CLS]", 3 is for "[SEP]" + assert encoded_sentence == [2] + text + [3] + assert encoded_pair == [2] + text + [3] + text_2 + [3] + + + diff --git a/transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py index 73ea38e2..c503ea5e 100644 --- a/transformers/tests/tokenization_bert_test.py +++ b/transformers/tests/tokenization_bert_test.py @@ -16,7 +16,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from io import open from transformers.tokenization_bert import (BasicTokenizer, @@ -26,6 +25,7 @@ from transformers.tokenization_bert import (BasicTokenizer, _is_whitespace, VOCAB_FILES_NAMES) from .tokenization_tests_commons import CommonTestCases +from .utils import slow class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): @@ -126,7 +126,7 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertFalse(_is_punctuation(u"A")) self.assertFalse(_is_punctuation(u" ")) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = self.tokenizer_class.from_pretrained("bert-base-uncased") @@ -139,5 +139,6 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): assert encoded_sentence == [101] + text + [102] assert encoded_pair == [101] + text + [102] + text_2 + [102] + if __name__ == '__main__': unittest.main() diff --git a/transformers/tests/tokenization_distilbert_test.py b/transformers/tests/tokenization_distilbert_test.py index 77a48765..e815eca6 100644 --- a/transformers/tests/tokenization_distilbert_test.py +++ b/transformers/tests/tokenization_distilbert_test.py @@ -16,13 +16,13 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from io import open from transformers.tokenization_distilbert import (DistilBertTokenizer) from .tokenization_tests_commons import CommonTestCases from .tokenization_bert_test import BertTokenizationTest +from .utils import slow class DistilBertTokenizationTest(BertTokenizationTest): @@ -31,7 +31,7 @@ class DistilBertTokenizationTest(BertTokenizationTest): def get_tokenizer(self, **kwargs): return DistilBertTokenizer.from_pretrained(self.tmpdirname, **kwargs) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased") diff --git a/transformers/tests/tokenization_gpt2_test.py b/transformers/tests/tokenization_gpt2_test.py index a77cc75e..5eae767b 100644 --- a/transformers/tests/tokenization_gpt2_test.py +++ b/transformers/tests/tokenization_gpt2_test.py @@ -67,6 +67,5 @@ class GPT2TokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertListEqual( tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens) - if __name__ == '__main__': unittest.main() diff --git a/transformers/tests/tokenization_roberta_test.py b/transformers/tests/tokenization_roberta_test.py index a27bf7d6..8ad0b595 100644 --- a/transformers/tests/tokenization_roberta_test.py +++ b/transformers/tests/tokenization_roberta_test.py @@ -17,11 +17,11 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import json import unittest -import pytest from io import open from transformers.tokenization_roberta import RobertaTokenizer, VOCAB_FILES_NAMES from .tokenization_tests_commons import CommonTestCases +from .utils import slow class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester): @@ -79,7 +79,7 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester): [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2] ) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = RobertaTokenizer.from_pretrained("roberta-base") diff --git a/transformers/tests/tokenization_t5_test.py b/transformers/tests/tokenization_t5_test.py new file mode 100644 index 00000000..0b4f960e --- /dev/null +++ b/transformers/tests/tokenization_t5_test.py @@ -0,0 +1,77 @@ +# coding=utf-8 +# Copyright 2018 Google T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import unittest + +from transformers.tokenization_t5 import (T5Tokenizer) +from transformers.tokenization_xlnet import SPIECE_UNDERLINE + +from .tokenization_tests_commons import CommonTestCases + +SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'fixtures/test_sentencepiece.model') + +class T5TokenizationTest(CommonTestCases.CommonTokenizerTester): + + tokenizer_class = T5Tokenizer + + def setUp(self): + super(T5TokenizationTest, self).setUp() + + # We have a SentencePiece fixture for testing + tokenizer = T5Tokenizer(SAMPLE_VOCAB) + tokenizer.save_pretrained(self.tmpdirname) + + def get_tokenizer(self, **kwargs): + return T5Tokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def get_input_output_texts(self): + input_text = u"This is a test" + output_text = u"This is a test" + return input_text, output_text + + def test_full_tokenizer(self): + tokenizer = T5Tokenizer(SAMPLE_VOCAB) + + tokens = tokenizer.tokenize(u'This is a test') + self.assertListEqual(tokens, [u'▁This', u'▁is', u'▁a', u'▁t', u'est']) + + self.assertListEqual( + tokenizer.convert_tokens_to_ids(tokens), [285, 46, 10, 170, 382]) + + tokens = tokenizer.tokenize(u"I was born in 92000, and this is falsé.") + self.assertListEqual(tokens, [SPIECE_UNDERLINE + u'I', SPIECE_UNDERLINE + u'was', SPIECE_UNDERLINE + u'b', + u'or', u'n', SPIECE_UNDERLINE + u'in', SPIECE_UNDERLINE + u'', + u'9', u'2', u'0', u'0', u'0', u',', SPIECE_UNDERLINE + u'and', SPIECE_UNDERLINE + u'this', + SPIECE_UNDERLINE + u'is', SPIECE_UNDERLINE + u'f', u'al', u's', u'é', u'.']) + ids = tokenizer.convert_tokens_to_ids(tokens) + self.assertListEqual( + ids, [8, 21, 84, 55, 24, 19, 7, 0, + 602, 347, 347, 347, 3, 12, 66, + 46, 72, 80, 6, 0, 4]) + + back_tokens = tokenizer.convert_ids_to_tokens(ids) + self.assertListEqual(back_tokens, [SPIECE_UNDERLINE + u'I', SPIECE_UNDERLINE + u'was', SPIECE_UNDERLINE + u'b', + u'or', u'n', SPIECE_UNDERLINE + u'in', + SPIECE_UNDERLINE + u'', u'', u'2', u'0', u'0', u'0', u',', + SPIECE_UNDERLINE + u'and', SPIECE_UNDERLINE + u'this', + SPIECE_UNDERLINE + u'is', SPIECE_UNDERLINE + u'f', u'al', u's', + u'', u'.']) + + +if __name__ == '__main__': + unittest.main() diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index fdaf8cc1..c417d033 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -102,14 +102,55 @@ class CommonTestCases: with TemporaryDirectory() as tmpdirname: filename = os.path.join(tmpdirname, u"tokenizer.bin") - pickle.dump(tokenizer, open(filename, "wb")) + with open(filename, "wb") as handle: + pickle.dump(tokenizer, handle) - tokenizer_new = pickle.load(open(filename, "rb")) + with open(filename, "rb") as handle: + tokenizer_new = pickle.load(handle) subwords_loaded = tokenizer_new.tokenize(text) self.assertListEqual(subwords, subwords_loaded) + def test_added_tokens_do_lower_case(self): + tokenizer = self.get_tokenizer(do_lower_case=True) + + special_token = tokenizer.all_special_tokens[0] + + text = special_token + " aaaaa bbbbbb low cccccccccdddddddd l " + special_token + text2 = special_token + " AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l " + special_token + + toks0 = tokenizer.tokenize(text) # toks before adding new_toks + + new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd", 'AAAAA BBBBBB', 'CCCCCCCCCDDDDDDDD'] + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 2) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) + self.assertNotEqual(len(toks), len(toks0)) # toks0 should be longer + self.assertListEqual(toks, toks2) + + # Check that none of the special tokens are lowercased + sequence_with_special_tokens = "A " + " yEs ".join(tokenizer.all_special_tokens) + " B" + tokenized_sequence = tokenizer.tokenize(sequence_with_special_tokens) + + for special_token in tokenizer.all_special_tokens: + self.assertTrue(special_token in tokenized_sequence) + + tokenizer = self.get_tokenizer(do_lower_case=False) + + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 4) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) # Length should still be the same + self.assertNotEqual(len(toks), len(toks0)) + self.assertNotEqual(toks[1], toks2[1]) # But at least the first non-special tokens should differ def test_add_tokens_tokenizer(self): tokenizer = self.get_tokenizer() @@ -198,6 +239,15 @@ class CommonTestCases: self.assertNotEqual(len(tokens_2), 0) self.assertIsInstance(text_2, (str, unicode)) + def test_encode_decode_with_spaces(self): + tokenizer = self.get_tokenizer() + + new_toks = ['[ABC]', '[DEF]', 'GHI IHG'] + tokenizer.add_tokens(new_toks) + input = "[ABC] [DEF] [ABC] GHI IHG [DEF]" + encoded = tokenizer.encode(input, add_special_tokens=False) + decoded = tokenizer.decode(encoded) + self.assertEqual(decoded, input) def test_pretrained_model_lists(self): weights_list = list(self.tokenizer_class.max_model_input_sizes.keys()) @@ -243,7 +293,11 @@ class CommonTestCases: sequence = tokenizer.encode(seq_0, add_special_tokens=False) num_added_tokens = tokenizer.num_added_tokens() total_length = len(sequence) + num_added_tokens - information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride) + information = tokenizer.encode_plus(seq_0, + max_length=total_length - 2, + add_special_tokens=True, + stride=stride, + return_overflowing_tokens=True) truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] @@ -270,10 +324,12 @@ class CommonTestCases: ) information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, - stride=stride, truncation_strategy='only_second') + stride=stride, truncation_strategy='only_second', + return_overflowing_tokens=True) information_first_truncated = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, stride=stride, - truncation_strategy='only_first') + truncation_strategy='only_first', + return_overflowing_tokens=True) truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] @@ -305,7 +361,7 @@ class CommonTestCases: # Testing single inputs encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True, return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) @@ -317,7 +373,8 @@ class CommonTestCases: # Testing inputs pairs encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False) + tokenizer.encode(sequence_1, add_special_tokens=False) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, sequence_1, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, sequence_1, add_special_tokens=True, + return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) @@ -329,9 +386,98 @@ class CommonTestCases: # Testing with already existing special tokens if tokenizer.cls_token_id == tokenizer.unk_token_id and tokenizer.cls_token_id == tokenizer.unk_token_id: tokenizer.add_special_tokens({'cls_token': '', 'sep_token': ''}) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, + add_special_tokens=True, + return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask_orig = encoded_sequence_dict["special_tokens_mask"] special_tokens_mask = tokenizer.get_special_tokens_mask(encoded_sequence_w_special, already_has_special_tokens=True) self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) self.assertEqual(special_tokens_mask_orig, special_tokens_mask) + + def test_padding_to_max_length(self): + tokenizer = self.get_tokenizer() + + sequence = "Sequence" + padding_size = 10 + padding_idx = tokenizer.pad_token_id + + # RIGHT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True + tokenizer.padding_side = "right" + encoded_sequence = tokenizer.encode(sequence) + sequence_length = len(encoded_sequence) + padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True) + padded_sequence_length = len(padded_sequence) + assert sequence_length + padding_size == padded_sequence_length + assert encoded_sequence + [padding_idx] * padding_size == padded_sequence + + # LEFT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True + tokenizer.padding_side = "left" + encoded_sequence = tokenizer.encode(sequence) + sequence_length = len(encoded_sequence) + padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True) + padded_sequence_length = len(padded_sequence) + assert sequence_length + padding_size == padded_sequence_length + assert [padding_idx] * padding_size + encoded_sequence == padded_sequence + + # RIGHT & LEFT PADDING - Check that nothing is done when a maximum length is not specified + encoded_sequence = tokenizer.encode(sequence) + sequence_length = len(encoded_sequence) + + tokenizer.padding_side = "right" + padded_sequence_right = tokenizer.encode(sequence, pad_to_max_length=True) + padded_sequence_right_length = len(padded_sequence_right) + + tokenizer.padding_side = "left" + padded_sequence_left = tokenizer.encode(sequence, pad_to_max_length=True) + padded_sequence_left_length = len(padded_sequence_left) + + assert sequence_length == padded_sequence_right_length + assert encoded_sequence == padded_sequence_right + assert sequence_length == padded_sequence_left_length + assert encoded_sequence == padded_sequence_left + + def test_encode_plus_with_padding(self): + tokenizer = self.get_tokenizer() + + sequence = "Sequence" + padding_size = 10 + padding_idx = tokenizer.pad_token_id + token_type_padding_idx = tokenizer.pad_token_type_id + + encoded_sequence = tokenizer.encode_plus(sequence, return_special_tokens_mask=True) + input_ids = encoded_sequence['input_ids'] + token_type_ids = encoded_sequence['token_type_ids'] + attention_mask = encoded_sequence['attention_mask'] + special_tokens_mask = encoded_sequence['special_tokens_mask'] + sequence_length = len(input_ids) + + # Test right padding + tokenizer.padding_side = "right" + padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True, return_special_tokens_mask=True) + padded_input_ids = padded_sequence['input_ids'] + padded_token_type_ids = padded_sequence['token_type_ids'] + padded_attention_mask = padded_sequence['attention_mask'] + padded_special_tokens_mask = padded_sequence['special_tokens_mask'] + padded_sequence_length = len(padded_input_ids) + + assert sequence_length + padding_size == padded_sequence_length + assert input_ids + [padding_idx] * padding_size == padded_input_ids + assert token_type_ids + [token_type_padding_idx] * padding_size == padded_token_type_ids + assert attention_mask + [0] * padding_size == padded_attention_mask + assert special_tokens_mask + [1] * padding_size == padded_special_tokens_mask + + # Test left padding + tokenizer.padding_side = "left" + padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True, return_special_tokens_mask=True) + padded_input_ids = padded_sequence['input_ids'] + padded_token_type_ids = padded_sequence['token_type_ids'] + padded_attention_mask = padded_sequence['attention_mask'] + padded_special_tokens_mask = padded_sequence['special_tokens_mask'] + padded_sequence_length = len(padded_input_ids) + + assert sequence_length + padding_size == padded_sequence_length + assert [padding_idx] * padding_size + input_ids == padded_input_ids + assert [token_type_padding_idx] * padding_size + token_type_ids == padded_token_type_ids + assert [0] * padding_size + attention_mask == padded_attention_mask + assert [1] * padding_size + special_tokens_mask == padded_special_tokens_mask \ No newline at end of file diff --git a/transformers/tests/tokenization_transfo_xl_test.py b/transformers/tests/tokenization_transfo_xl_test.py index 4e99484b..5495ebd3 100644 --- a/transformers/tests/tokenization_transfo_xl_test.py +++ b/transformers/tests/tokenization_transfo_xl_test.py @@ -16,7 +16,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from io import open from transformers import is_torch_available @@ -24,11 +23,12 @@ from transformers import is_torch_available if is_torch_available(): import torch from transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES -else: - pytestmark = pytest.mark.skip("Require Torch") # TODO: untangle Transfo-XL tokenizer from torch.load and torch.save from .tokenization_tests_commons import CommonTestCases +from .utils import require_torch + +@require_torch class TransfoXLTokenizationTest(CommonTestCases.CommonTokenizerTester): tokenizer_class = TransfoXLTokenizer if is_torch_available() else None diff --git a/transformers/tests/tokenization_utils_test.py b/transformers/tests/tokenization_utils_test.py index 8630191c..ff3f80ff 100644 --- a/transformers/tests/tokenization_utils_test.py +++ b/transformers/tests/tokenization_utils_test.py @@ -18,13 +18,14 @@ from __future__ import print_function import unittest import six -import pytest from transformers import PreTrainedTokenizer from transformers.tokenization_gpt2 import GPT2Tokenizer +from .utils import slow + class TokenizerUtilsTest(unittest.TestCase): - @pytest.mark.slow + def check_tokenizer_from_pretrained(self, tokenizer_class): s3_models = list(tokenizer_class.max_model_input_sizes.keys()) for model_name in s3_models[:1]: @@ -41,6 +42,7 @@ class TokenizerUtilsTest(unittest.TestCase): special_tok_id = tokenizer.convert_tokens_to_ids(special_tok) self.assertIsInstance(special_tok_id, int) + @slow def test_pretrained_tokenizers(self): self.check_tokenizer_from_pretrained(GPT2Tokenizer) diff --git a/transformers/tests/tokenization_xlm_test.py b/transformers/tests/tokenization_xlm_test.py index 3ff6564e..7582a466 100644 --- a/transformers/tests/tokenization_xlm_test.py +++ b/transformers/tests/tokenization_xlm_test.py @@ -17,11 +17,11 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest import json -import pytest from transformers.tokenization_xlm import XLMTokenizer, VOCAB_FILES_NAMES from .tokenization_tests_commons import CommonTestCases +from .utils import slow class XLMTokenizationTest(CommonTestCases.CommonTokenizerTester): @@ -67,7 +67,7 @@ class XLMTokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertListEqual( tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = XLMTokenizer.from_pretrained("xlm-mlm-en-2048") diff --git a/transformers/tests/tokenization_xlnet_test.py b/transformers/tests/tokenization_xlnet_test.py index 2e14ffeb..b68495a7 100644 --- a/transformers/tests/tokenization_xlnet_test.py +++ b/transformers/tests/tokenization_xlnet_test.py @@ -16,11 +16,11 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from transformers.tokenization_xlnet import (XLNetTokenizer, SPIECE_UNDERLINE) from .tokenization_tests_commons import CommonTestCases +from .utils import slow SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fixtures/test_sentencepiece.model') @@ -90,7 +90,7 @@ class XLNetTokenizationTest(CommonTestCases.CommonTokenizerTester): u'9', u'2', u'0', u'0', u'0', u',', SPIECE_UNDERLINE + u'and', SPIECE_UNDERLINE + u'this', SPIECE_UNDERLINE + u'is', SPIECE_UNDERLINE + u'f', u'al', u'se', u'.']) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased") diff --git a/transformers/tests/utils.py b/transformers/tests/utils.py new file mode 100644 index 00000000..ba0e19f4 --- /dev/null +++ b/transformers/tests/utils.py @@ -0,0 +1,88 @@ +import os +import unittest +import tempfile + +from distutils.util import strtobool + +from transformers.file_utils import _tf_available, _torch_available + + +CACHE_DIR = os.path.join(tempfile.gettempdir(), "transformers_test") + +SMALL_MODEL_IDENTIFIER = "julien-c/bert-xsmall-dummy" + + +def parse_flag_from_env(key, default=False): + try: + value = os.environ[key] + except KeyError: + # KEY isn't set, default to `default`. + _value = default + else: + # KEY is set, convert it to True or False. + try: + _value = strtobool(value) + except ValueError: + # More values are supported, but let's keep the message simple. + raise ValueError("If set, {} must be yes or no.".format(key)) + return _value + +_run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False) +_run_custom_tokenizers = parse_flag_from_env("RUN_CUSTOM_TOKENIZERS", default=False) + + +def slow(test_case): + """ + Decorator marking a test as slow. + + Slow tests are skipped by default. Set the RUN_SLOW environment variable + to a truthy value to run them. + + """ + if not _run_slow_tests: + test_case = unittest.skip("test is slow")(test_case) + return test_case + + +def custom_tokenizers(test_case): + """ + Decorator marking a test for a custom tokenizer. + + Custom tokenizers require additional dependencies, and are skipped + by default. Set the RUN_CUSTOM_TOKENIZERS environment variable + to a truthy value to run them. + """ + if not _run_custom_tokenizers: + test_case = unittest.skip("test of custom tokenizers")(test_case) + return test_case + + +def require_torch(test_case): + """ + Decorator marking a test that requires PyTorch. + + These tests are skipped when PyTorch isn't installed. + + """ + if not _torch_available: + test_case = unittest.skip("test requires PyTorch")(test_case) + return test_case + + +def require_tf(test_case): + """ + Decorator marking a test that requires TensorFlow. + + These tests are skipped when TensorFlow isn't installed. + + """ + if not _tf_available: + test_case = unittest.skip("test requires TensorFlow")(test_case) + return test_case + + +if _torch_available: + # Set the USE_CUDA environment variable to select a GPU. + torch_device = "cuda" if parse_flag_from_env("USE_CUDA") else "cpu" +else: + torch_device = None diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py new file mode 100644 index 00000000..6b92d072 --- /dev/null +++ b/transformers/tokenization_albert.py @@ -0,0 +1,252 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Tokenization classes for ALBERT model.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +from .tokenization_utils import PreTrainedTokenizer +import logging +import unicodedata +import six +import os +from shutil import copyfile + +logger = logging.getLogger(__name__) +VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-spiece.model", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-spiece.model", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-spiece.model", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-spiece.model", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'albert-base-v1': 512, + 'albert-large-v1': 512, + 'albert-xlarge-v1': 512, + 'albert-xxlarge-v1': 512, + 'albert-base-v2': 512, + 'albert-large-v2': 512, + 'albert-xlarge-v2': 512, + 'albert-xxlarge-v2': 512, +} + +SPIECE_UNDERLINE = u'▁' + +class AlbertTokenizer(PreTrainedTokenizer): + """ + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + """ + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, + do_lower_case=True, remove_space=True, keep_accents=False, + bos_token="[CLS]", eos_token="[SEP]", unk_token="", sep_token="[SEP]", + pad_token="", cls_token="[CLS]", mask_token="[MASK]", **kwargs): + super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, + unk_token=unk_token, sep_token=sep_token, + pad_token=pad_token, cls_token=cls_token, + mask_token=mask_token, **kwargs) + + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + + self.do_lower_case = do_lower_case + self.remove_space = remove_space + self.keep_accents = keep_accents + self.vocab_file = vocab_file + + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(vocab_file) + + @property + def vocab_size(self): + return len(self.sp_model) + + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(self.vocab_file) + + def preprocess_text(self, inputs): + if self.remove_space: + outputs = ' '.join(inputs.strip().split()) + else: + outputs = inputs + outputs = outputs.replace("``", '"').replace("''", '"') + + if six.PY2 and isinstance(outputs, str): + outputs = outputs.decode('utf-8') + + if not self.keep_accents: + outputs = unicodedata.normalize('NFKD', outputs) + outputs = ''.join([c for c in outputs if not unicodedata.combining(c)]) + if self.do_lower_case: + outputs = outputs.lower() + + return outputs + + def _tokenize(self, text, return_unicode=True, sample=False): + """ Tokenize a string. + return_unicode is used only for py2 + """ + text = self.preprocess_text(text) + # note(zhiliny): in some systems, sentencepiece only accepts str for py2 + if six.PY2 and isinstance(text, unicode): + text = text.encode('utf-8') + + if not sample: + pieces = self.sp_model.EncodeAsPieces(text) + else: + pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1) + new_pieces = [] + for piece in pieces: + if len(piece) > 1 and piece[-1] == str(',') and piece[-2].isdigit(): + cur_pieces = self.sp_model.EncodeAsPieces( + piece[:-1].replace(SPIECE_UNDERLINE, '')) + if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: + if len(cur_pieces[0]) == 1: + cur_pieces = cur_pieces[1:] + else: + cur_pieces[0] = cur_pieces[0][1:] + cur_pieces.append(piece[-1]) + new_pieces.extend(cur_pieces) + else: + new_pieces.append(piece) + + # note(zhiliny): convert back to unicode for py2 + if six.PY2 and return_unicode: + ret_pieces = [] + for piece in new_pieces: + if isinstance(piece, str): + piece = piece.decode('utf-8') + ret_pieces.append(piece) + new_pieces = ret_pieces + + return new_pieces + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + return self.sp_model.PieceToId(token) + + def _convert_id_to_token(self, index, return_unicode=True): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + token = self.sp_model.IdToPiece(index) + if six.PY2 and return_unicode and isinstance(token, str): + token = token.decode('utf-8') + return token + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (strings for sub-words) in a single string.""" + out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() + return out_string + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + An ALBERT sequence has the following format: + single sequence: [CLS] X [SEP] + pair of sequences: [CLS] A [SEP] B [SEP] + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is not None: + return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + An ALBERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index ec056de1..5377bd48 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging from .tokenization_bert import BertTokenizer +from .tokenization_bert_japanese import BertJapaneseTokenizer from .tokenization_openai import OpenAIGPTTokenizer from .tokenization_gpt2 import GPT2Tokenizer from .tokenization_ctrl import CTRLTokenizer @@ -27,6 +28,10 @@ from .tokenization_xlnet import XLNetTokenizer from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_camembert import CamembertTokenizer +from .tokenization_albert import AlbertTokenizer +from .tokenization_t5 import T5Tokenizer +from .tokenization_xlm_roberta import XLMRobertaTokenizer logger = logging.getLogger(__name__) @@ -41,15 +46,19 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Tokenizer (T5 model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) + - contains `albert`: AlbertTokenizer (ALBERT model) + - contains `camembert`: CamembertTokenizer (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaTokenizer (XLM-RoBERTa model) - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert`: BertTokenizer (Bert model) - contains `openai-gpt`: OpenAIGPTTokenizer (OpenAI GPT model) - contains `gpt2`: GPT2Tokenizer (OpenAI GPT-2 model) - - contains `ctrl`: CTRLTokenizer (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLTokenizer (Transformer-XL model) - contains `xlnet`: XLNetTokenizer (XLNet model) - contains `xlm`: XLMTokenizer (XLM model) + - contains `ctrl`: CTRLTokenizer (Salesforce CTRL model) This class cannot be instantiated using `__init__()` (throw an error). """ @@ -64,20 +73,26 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Tokenizer (T5 model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) - - contains `roberta`: RobertaTokenizer (XLM model) + - contains `albert`: AlbertTokenizer (ALBERT model) + - contains `camembert`: CamembertTokenizer (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaTokenizer (XLM-RoBERTa model) + - contains `roberta`: RobertaTokenizer (RoBERTa model) + - contains `bert-base-japanese`: BertJapaneseTokenizer (Bert model) - contains `bert`: BertTokenizer (Bert model) - contains `openai-gpt`: OpenAIGPTTokenizer (OpenAI GPT model) - contains `gpt2`: GPT2Tokenizer (OpenAI GPT-2 model) - - contains `ctrl`: CTRLTokenizer (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLTokenizer (Transformer-XL model) - contains `xlnet`: XLNetTokenizer (XLNet model) - contains `xlm`: XLMTokenizer (XLM model) + - contains `ctrl`: CTRLTokenizer (Salesforce CTRL model) Params: pretrained_model_name_or_path: either: - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a predefined tokenizer that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``. @@ -87,6 +102,9 @@ class AutoTokenizer(object): force_download: (`optional`) boolean, default False: Force to (re-)download the vocabulary files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -97,14 +115,30 @@ class AutoTokenizer(object): Examples:: - tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') # Download vocabulary from S3 and cache. - tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/') # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')` + # Download vocabulary from S3 and cache. + tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') + + # Download vocabulary from S3 (user-uploaded) and cache. + tokenizer = AutoTokenizer.from_pretrained('dbmdz/bert-base-german-cased') + + # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`) + tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/') """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return T5Tokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return DistilBertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'bert-base-japanese' in pretrained_model_name_or_path: + return BertJapaneseTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'bert' in pretrained_model_name_or_path: return BertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'openai-gpt' in pretrained_model_name_or_path: @@ -121,4 +155,4 @@ class AutoTokenizer(object): return CTRLTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm-roberta', 'xlm', 'roberta', 'distilbert,' 'camembert', 'ctrl', 'albert'".format(pretrained_model_name_or_path)) diff --git a/transformers/tokenization_bert.py b/transformers/tokenization_bert.py index ded5072e..edc26d88 100644 --- a/transformers/tokenization_bert.py +++ b/transformers/tokenization_bert.py @@ -46,6 +46,8 @@ PRETRAINED_VOCAB_FILES_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-vocab.txt", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-vocab.txt", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-vocab.txt", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/vocab.txt", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1/vocab.txt", } } @@ -65,6 +67,8 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'bert-base-cased-finetuned-mrpc': 512, 'bert-base-german-dbmdz-cased': 512, 'bert-base-german-dbmdz-uncased': 512, + 'bert-base-finnish-cased-v1': 512, + 'bert-base-finnish-uncased-v1': 512, } PRETRAINED_INIT_CONFIGURATION = { @@ -83,6 +87,8 @@ PRETRAINED_INIT_CONFIGURATION = { 'bert-base-cased-finetuned-mrpc': {'do_lower_case': False}, 'bert-base-german-dbmdz-cased': {'do_lower_case': False}, 'bert-base-german-dbmdz-uncased': {'do_lower_case': True}, + 'bert-base-finnish-cased-v1': {'do_lower_case': False}, + 'bert-base-finnish-uncased-v1': {'do_lower_case': True}, } @@ -113,12 +119,12 @@ class BertTokenizer(PreTrainedTokenizer): Args: vocab_file: Path to a one-wordpiece-per-line vocabulary file - do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False + do_lower_case: Whether to lower case the input. Only has an effect when do_basic_tokenize=True do_basic_tokenize: Whether to do basic tokenization before wordpiece. max_len: An artificial maximum length to truncate tokenized sequences to; Effective maximum length is always the minimum of this value (if specified) and the underlying BERT model's sequence length. never_split: List of tokens which will never be split during tokenization. Only has an effect when - do_wordpiece_only=False + do_basic_tokenize=True """ vocab_files_names = VOCAB_FILES_NAMES diff --git a/transformers/tokenization_bert_japanese.py b/transformers/tokenization_bert_japanese.py new file mode 100644 index 00000000..0ff45cbf --- /dev/null +++ b/transformers/tokenization_bert_japanese.py @@ -0,0 +1,253 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import logging +import os +import six +import unicodedata +from io import open + +from .tokenization_bert import BertTokenizer, BasicTokenizer, WordpieceTokenizer, load_vocab +from .tokenization_utils import PreTrainedTokenizer + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-vocab.txt", + 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-vocab.txt", + 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-vocab.txt", + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-vocab.txt" + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'bert-base-japanese': 512, + 'bert-base-japanese-whole-word-masking': 512, + 'bert-base-japanese-char': 512, + 'bert-base-japanese-char-whole-word-masking': 512 +} + +PRETRAINED_INIT_CONFIGURATION = { + 'bert-base-japanese': { + 'do_lower_case': False, + 'word_tokenizer_type': 'mecab', + 'subword_tokenizer_type': 'wordpiece' + }, + 'bert-base-japanese-whole-word-masking':{ + 'do_lower_case': False, + 'word_tokenizer_type': 'mecab', + 'subword_tokenizer_type': 'wordpiece' + }, + 'bert-base-japanese-char': { + 'do_lower_case': False, + 'word_tokenizer_type': 'mecab', + 'subword_tokenizer_type': 'character' + }, + 'bert-base-japanese-char-whole-word-masking': { + 'do_lower_case': False, + 'word_tokenizer_type': 'mecab', + 'subword_tokenizer_type': 'character' + } +} + + +class BertJapaneseTokenizer(BertTokenizer): + """BERT tokenizer for Japanese text""" + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, do_lower_case=False, + do_word_tokenize=True, do_subword_tokenize=True, + word_tokenizer_type='basic', subword_tokenizer_type='wordpiece', + never_split=None, unk_token='[UNK]', sep_token='[SEP]', + pad_token='[PAD]', cls_token='[CLS]', mask_token='[MASK]', **kwargs): + """Constructs a MecabBertTokenizer. + + Args: + **vocab_file**: Path to a one-wordpiece-per-line vocabulary file. + **do_lower_case**: (`optional`) boolean (default True) + Whether to lower case the input. + Only has an effect when do_basic_tokenize=True. + **do_word_tokenize**: (`optional`) boolean (default True) + Whether to do word tokenization. + **do_subword_tokenize**: (`optional`) boolean (default True) + Whether to do subword tokenization. + **word_tokenizer_type**: (`optional`) string (default "basic") + Type of word tokenizer. + **subword_tokenizer_type**: (`optional`) string (default "wordpiece") + Type of subword tokenizer. + """ + super(BertTokenizer, self).__init__(unk_token=unk_token, sep_token=sep_token, + pad_token=pad_token, cls_token=cls_token, + mask_token=mask_token, **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " + "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file)) + self.vocab = load_vocab(vocab_file) + self.ids_to_tokens = collections.OrderedDict( + [(ids, tok) for tok, ids in self.vocab.items()]) + + self.do_word_tokenize = do_word_tokenize + if do_word_tokenize: + if word_tokenizer_type == 'basic': + self.word_tokenizer = BasicTokenizer(do_lower_case=do_lower_case, + never_split=never_split, + tokenize_chinese_chars=False) + elif word_tokenizer_type == 'mecab': + self.word_tokenizer = MecabTokenizer(do_lower_case=do_lower_case, + never_split=never_split) + else: + raise ValueError( + "Invalid word_tokenizer_type '{}' is specified.".format(word_tokenizer_type)) + + self.do_subword_tokenize = do_subword_tokenize + if do_subword_tokenize: + if subword_tokenizer_type == 'wordpiece': + self.subword_tokenizer = WordpieceTokenizer(vocab=self.vocab, + unk_token=self.unk_token) + elif subword_tokenizer_type == 'character': + self.subword_tokenizer = CharacterTokenizer(vocab=self.vocab, + unk_token=self.unk_token) + else: + raise ValueError( + "Invalid subword_tokenizer_type '{}' is specified.".format(subword_tokenizer_type)) + + + def _tokenize(self, text): + if self.do_word_tokenize: + tokens = self.word_tokenizer.tokenize(text, + never_split=self.all_special_tokens) + else: + tokens = [text] + + if self.do_subword_tokenize: + split_tokens = [sub_token for token in tokens + for sub_token in self.subword_tokenizer.tokenize(token)] + else: + split_tokens = tokens + + return split_tokens + + +class MecabTokenizer(object): + """Runs basic tokenization with MeCab morphological parser.""" + + def __init__(self, do_lower_case=False, never_split=None, normalize_text=True): + """Constructs a MecabTokenizer. + + Args: + **do_lower_case**: (`optional`) boolean (default True) + Whether to lower case the input. + **never_split**: (`optional`) list of str + Kept for backward compatibility purposes. + Now implemented directly at the base class level (see :func:`PreTrainedTokenizer.tokenize`) + List of token not to split. + **normalize_text**: (`optional`) boolean (default True) + Whether to apply unicode normalization to text before tokenization. + """ + self.do_lower_case = do_lower_case + self.never_split = never_split if never_split is not None else [] + self.normalize_text = normalize_text + + import MeCab + self.mecab = MeCab.Tagger() + + def tokenize(self, text, never_split=None, **kwargs): + """Tokenizes a piece of text.""" + if self.normalize_text: + text = unicodedata.normalize('NFKC', text) + + never_split = self.never_split + (never_split if never_split is not None else []) + tokens = [] + + if six.PY2: + mecab_output = self.mecab.parse(text.encode('utf-8')).decode('utf-8') + else: + mecab_output = self.mecab.parse(text) + + cursor = 0 + for line in mecab_output.split('\n'): + if line == 'EOS': + break + + token, _ = line.split('\t') + token_start = text.index(token, cursor) + token_end = token_start + len(token) + if self.do_lower_case and token not in never_split: + token = token.lower() + + tokens.append(token) + cursor = token_end + + return tokens + + +class CharacterTokenizer(object): + """Runs Character tokenziation.""" + + def __init__(self, vocab, unk_token, normalize_text=True): + """Constructs a CharacterTokenizer. + + Args: + **vocab**: + Vocabulary object. + **unk_token**: str + A special symbol for out-of-vocabulary token. + **normalize_text**: (`optional`) boolean (default True) + Whether to apply unicode normalization to text before tokenization. + """ + self.vocab = vocab + self.unk_token = unk_token + self.normalize_text = normalize_text + + def tokenize(self, text): + """Tokenizes a piece of text into characters. + + For example: + input = "apple" + output = ["a", "p", "p", "l", "e"] + Args: + text: A single token or whitespace separated tokens. + This should have already been passed through `BasicTokenizer`. + Returns: + A list of characters. + """ + if self.normalize_text: + text = unicodedata.normalize('NFKC', text) + + output_tokens = [] + for i, char in enumerate(text): + if char not in self.vocab: + output_tokens.append(self.unk_token) + continue + + output_tokens.append(char) + + return output_tokens diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py new file mode 100644 index 00000000..4c4615eb --- /dev/null +++ b/transformers/tokenization_camembert.py @@ -0,0 +1,166 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +""" Tokenization classes for Camembert model.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +import os +from shutil import copyfile + +import sentencepiece as spm +from transformers.tokenization_utils import PreTrainedTokenizer +from .tokenization_xlnet import SPIECE_UNDERLINE + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'camembert-base': None, +} + +class CamembertTokenizer(PreTrainedTokenizer): + """ + Adapted from RobertaTokenizer and XLNetTokenizer + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + """ + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="", + cls_token="", unk_token="", pad_token='', mask_token='', + additional_special_tokens=['NOTUSED', 'NOTUSED'], **kwargs): + super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, + sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, + mask_token=mask_token, additional_special_tokens=additional_special_tokens, + **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(str(vocab_file)) + self.vocab_file = vocab_file + # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual + # sentencepiece vocabulary (this is the case for and + self.fairseq_tokens_to_ids = {'NOTUSED': 0, '': 1, 'NOTUSED': 2, '': 3} + self.fairseq_offset = len(self.fairseq_tokens_to_ids) + self.fairseq_tokens_to_ids[''] = len(self.sp_model) + len(self.fairseq_tokens_to_ids) + self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()} + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A RoBERTa sequence has the following format: + single sequence: X + pair of sequences: A B + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A RoBERTa sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1] + + @property + def vocab_size(self): + return len(self.fairseq_tokens_to_ids) + len(self.sp_model) + + def _tokenize(self, text): + return self.sp_model.EncodeAsPieces(text) + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + if token in self.fairseq_tokens_to_ids: + return self.fairseq_tokens_to_ids[token] + elif self.sp_model.PieceToId(token) == 0: + # Convert sentence piece unk token to fairseq unk token index + return self.unk_token_id + return self.fairseq_offset + self.sp_model.PieceToId(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + if index in self.fairseq_ids_to_tokens: + return self.fairseq_ids_to_tokens[index] + return self.sp_model.IdToPiece(index - self.fairseq_offset) + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (strings for sub-words) in a single string.""" + out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() + return out_string + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) diff --git a/transformers/tokenization_ctrl.py b/transformers/tokenization_ctrl.py index 3d67fa2c..219f17c4 100644 --- a/transformers/tokenization_ctrl.py +++ b/transformers/tokenization_ctrl.py @@ -133,9 +133,11 @@ class CTRLTokenizer(PreTrainedTokenizer): self.max_len_single_sentence = self.max_len # no default special tokens - you can update this value if you add special tokens self.max_len_sentences_pair = self.max_len # no default special tokens - you can update this value if you add special tokens - self.encoder = json.load(open(vocab_file, encoding="utf-8")) + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) self.decoder = {v:k for k,v in self.encoder.items()} - merges = open(merges_file, encoding='utf-8').read().split('\n')[1:-1] + with open(merges_file, encoding='utf-8') as merges_handle: + merges = merges_handle.read().split('\n')[1:-1] merges = [tuple(merge.split()) for merge in merges] self.bpe_ranks = dict(zip(merges, range(len(merges)))) self.cache = {} @@ -192,9 +194,9 @@ class CTRLTokenizer(PreTrainedTokenizer): """ split_tokens = [] - text = text.split(' ') + words = re.findall(r'\S+\n?', text) - for token in text: + for token in words: split_tokens.extend([t for t in self.bpe(token).split(' ')]) return split_tokens diff --git a/transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py index dfa02926..2f245d71 100644 --- a/transformers/tokenization_distilbert.py +++ b/transformers/tokenization_distilbert.py @@ -33,12 +33,16 @@ PRETRAINED_VOCAB_FILES_MAP = { { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt", 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", + 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-vocab.txt", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt", } } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'distilbert-base-uncased': 512, 'distilbert-base-uncased-distilled-squad': 512, + 'distilbert-base-german-cased': 512, + 'distilbert-base-multilingual-cased': 512, } @@ -49,12 +53,12 @@ class DistilBertTokenizer(BertTokenizer): Args: vocab_file: Path to a one-wordpiece-per-line vocabulary file - do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False + do_lower_case: Whether to lower case the input. Only has an effect when do_basic_tokenize=True do_basic_tokenize: Whether to do basic tokenization before wordpiece. max_len: An artificial maximum length to truncate tokenized sequences to; Effective maximum length is always the minimum of this value (if specified) and the underlying BERT model's sequence length. never_split: List of tokens which will never be split during tokenization. Only has an effect when - do_wordpiece_only=False + do_basic_tokenize=True """ vocab_files_names = VOCAB_FILES_NAMES diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index 4bec5159..68c61018 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -72,7 +72,7 @@ def bytes_to_unicode(): """ Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control characters the bpe code barfs on. - + The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. @@ -107,10 +107,10 @@ class GPT2Tokenizer(PreTrainedTokenizer): """ GPT-2 BPE tokenizer. Peculiarities: - Byte-level Byte-Pair-Encoding - - Requires a space to start the input string => the encoding methods should be called with the + - Requires a space to start the input string => the encoding and tokenize methods should be called with the ``add_prefix_space`` flag set to ``True``. - Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve - the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"` + Otherwise, this tokenizer's ``encode``, ``decode``, and ``tokenize`` methods will not conserve + the spaces at the beginning of a string: `tokenizer.decode(tokenizer.encode(" Hello")) = "Hello"` """ vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP @@ -122,13 +122,15 @@ class GPT2Tokenizer(PreTrainedTokenizer): self.max_len_single_sentence = self.max_len # no default special tokens - you can update this value if you add special tokens self.max_len_sentences_pair = self.max_len # no default special tokens - you can update this value if you add special tokens - self.encoder = json.load(open(vocab_file, encoding="utf-8")) + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) self.decoder = {v: k for k, v in self.encoder.items()} self.errors = errors # how to handle errors in decoding self.byte_encoder = bytes_to_unicode() self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} - bpe_data = open(merges_file, encoding='utf-8').read().split('\n')[1:-1] - bpe_merges = [tuple(merge.split()) for merge in bpe_data] + with open(merges_file, encoding='utf-8') as merges_handle: + bpe_merges = merges_handle.read().split('\n')[1:-1] + bpe_merges = [tuple(merge.split()) for merge in bpe_merges] self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) self.cache = {} @@ -184,7 +186,7 @@ class GPT2Tokenizer(PreTrainedTokenizer): """ Tokenize a string. Args: - add_prefix_space (boolean, default False): - Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers. + Begin the sentence with at least one space to get invariance to word order in GPT-2 (and RoBERTa) tokenizers. """ if add_prefix_space: text = ' ' + text @@ -234,4 +236,4 @@ class GPT2Tokenizer(PreTrainedTokenizer): writer.write(' '.join(bpe_tokens) + u'\n') index += 1 - return vocab_file, merge_file \ No newline at end of file + return vocab_file, merge_file diff --git a/transformers/tokenization_openai.py b/transformers/tokenization_openai.py index 0efbdb37..a4c64b70 100644 --- a/transformers/tokenization_openai.py +++ b/transformers/tokenization_openai.py @@ -101,9 +101,11 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer): self.nlp = BasicTokenizer(do_lower_case=True) self.fix_text = None - self.encoder = json.load(open(vocab_file, encoding="utf-8")) + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) self.decoder = {v:k for k,v in self.encoder.items()} - merges = open(merges_file, encoding='utf-8').read().split('\n')[1:-1] + with open(merges_file, encoding='utf-8') as merges_handle: + merges = merges_handle.read().split('\n')[1:-1] merges = [tuple(merge.split()) for merge in merges] self.bpe_ranks = dict(zip(merges, range(len(merges)))) self.cache = {} diff --git a/transformers/tokenization_t5.py b/transformers/tokenization_t5.py new file mode 100644 index 00000000..9fd37b67 --- /dev/null +++ b/transformers/tokenization_t5.py @@ -0,0 +1,176 @@ +# coding=utf-8 +# Copyright 2018 T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Tokenization class for model T5.""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import logging +import os +import re +import six +from shutil import copyfile + +from .tokenization_utils import PreTrainedTokenizer + +logger = logging.getLogger(__name__) + +SPIECE_UNDERLINE = u'▁' + +#################################################### +# Mapping from the keyword arguments names of Tokenizer `__init__` +# to file names for serializing Tokenizer instances +#################################################### +VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} + +#################################################### +# Mapping from the keyword arguments names of Tokenizer `__init__` +# to pretrained vocabulary URL for all the model shortcut names. +#################################################### +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-3b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-11b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + } +} + +#################################################### +# Mapping from model shortcut names to max length of inputs +#################################################### +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 't5-small': 512, + 't5-base': 512, + 't5-large': 512, + 't5-3b': 512, + 't5-11b': 512, +} + +class T5Tokenizer(PreTrainedTokenizer): + """ + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + - `extra_ids` add a number of extra ids added to the end of the vocabulary for use as sentinels. + These tokens are accessible as `` where `{%d}` is a number between 0 and extra_ids-1. + Extra tokens are indexed from the end of the vocabulary up to beginnning ( is the last token in the vocabulary) + (like in T5 preprocessing + see: https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117) + """ + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, eos_token="", unk_token="", + pad_token="", extra_ids=100, additional_special_tokens=None, **kwargs): + # Add extra_ids to the special token list + if extra_ids > 0: + if additional_special_tokens is None: + additional_special_tokens = [] + additional_special_tokens.extend([u"".format(i) for i in range(extra_ids)]) + + super(T5Tokenizer, self).__init__(eos_token=eos_token, unk_token=unk_token, + pad_token=pad_token, additional_special_tokens=additional_special_tokens, + **kwargs) + + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use T5Tokenizer:" + "https://github.com/google/sentencepiece" + "pip install sentencepiece") + + self.vocab_file = vocab_file + self._extra_ids = extra_ids + + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(vocab_file) + + @property + def vocab_size(self): + return self.sp_model.get_piece_size() + self._extra_ids + + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(self.vocab_file) + + def _tokenize(self, text, return_unicode=True, sample=False): + """ Take as input a string and return a list of strings (tokens) for words/sub-words + """ + if not sample: + pieces = self.sp_model.EncodeAsPieces(text) + else: + pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1) + + # convert back to unicode for py2 + if six.PY2 and return_unicode: + ret_pieces = [] + for piece in pieces: + if isinstance(piece, str): + piece = piece.decode('utf-8') + ret_pieces.append(piece) + pieces = ret_pieces + + return pieces + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + if token.startswith(u"', token) + num = int(l.group(1)) + return self.vocab_size - num - 1 + return self.sp_model.piece_to_id(token) + + def _convert_id_to_token(self, index, return_unicode=True): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + if index < self.sp_model.get_piece_size(): + token = self.sp_model.IdToPiece(index) + else: + token = u"".format(self.vocab_size - 1 - index) + if six.PY2 and return_unicode and isinstance(token, str): + token = token.decode('utf-8') + return token + + def convert_tokens_to_string(self, tokens): + """ Converts a sequence of tokens (string) in a single string. """ + out_string = self.sp_model.decode_pieces(tokens) + return out_string + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 4fa26a26..eda89f22 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -22,9 +22,10 @@ import json import six import copy import itertools +import re from io import open -from .file_utils import cached_path, is_tf_available, is_torch_available +from .file_utils import cached_path, is_remote_url, hf_bucket_url, is_tf_available, is_torch_available if is_tf_available(): import tensorflow as tf @@ -77,6 +78,8 @@ class PreTrainedTokenizer(object): "pad_token", "cls_token", "mask_token", "additional_special_tokens"] + padding_side = "right" + @property def bos_token(self): """ Beginning of sentence token (string). Log an error if used while not having been set. """ @@ -190,6 +193,11 @@ class PreTrainedTokenizer(object): """ Id of the padding token in the vocabulary. Log an error if used while not having been set. """ return self.convert_tokens_to_ids(self.pad_token) + @property + def pad_token_type_id(self): + """ Id of the padding token type in the vocabulary.""" + return self._pad_token_type_id + @property def cls_token_id(self): """ Id of the classification token in the vocabulary. E.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model. Log an error if used while not having been set. """ @@ -213,10 +221,14 @@ class PreTrainedTokenizer(object): self._pad_token = None self._cls_token = None self._mask_token = None + self._pad_token_type_id = 0 self._additional_special_tokens = [] self.max_len = max_len if max_len is not None else int(1e12) + # Padding side is right by default and over-riden in subclasses. If specified in the kwargs, it is changed. + self.padding_side = kwargs.pop('padding_side', self.padding_side) + # Added tokens self.added_tokens_encoder = {} self.added_tokens_decoder = {} @@ -243,6 +255,7 @@ class PreTrainedTokenizer(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a predefined tokenizer that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``. @@ -252,6 +265,9 @@ class PreTrainedTokenizer(object): force_download: (`optional`) boolean, default False: Force to (re-)download the vocabulary files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -267,6 +283,9 @@ class PreTrainedTokenizer(object): # Download vocabulary from S3 and cache. tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + # Download vocabulary from S3 (user-uploaded) and cache. + tokenizer = BertTokenizer.from_pretrained('dbmdz/bert-base-german-cased') + # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`) tokenizer = BertTokenizer.from_pretrained('./test/saved_model/') @@ -287,6 +306,7 @@ class PreTrainedTokenizer(object): def _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): cache_dir = kwargs.pop('cache_dir', None) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) s3_models = list(cls.max_model_input_sizes.keys()) @@ -311,12 +331,15 @@ class PreTrainedTokenizer(object): if os.path.isdir(pretrained_model_name_or_path): # If a directory is provided we look for the standard filenames full_file_name = os.path.join(pretrained_model_name_or_path, file_name) - else: + if not os.path.exists(full_file_name): + logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) + full_file_name = None + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): # If a path to a file is provided we use it (will only work for non-BPE tokenizer using a single vocabulary file) full_file_name = pretrained_model_name_or_path - if not os.path.exists(full_file_name): - logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) - full_file_name = None + else: + full_file_name = hf_bucket_url(pretrained_model_name_or_path, postfix=file_name) + vocab_files[file_id] = full_file_name # Look for the additional tokens files @@ -343,7 +366,7 @@ class PreTrainedTokenizer(object): "We assumed '{}' was a path or url to a directory containing vocabulary files " "named {} but couldn't find such vocabulary files at this path or url.".format( pretrained_model_name_or_path, ', '.join(s3_models), - pretrained_model_name_or_path, + pretrained_model_name_or_path, list(cls.vocab_files_names.values()))) # Get files from url, cache, or disk depending on the case @@ -353,7 +376,7 @@ class PreTrainedTokenizer(object): if file_path is None: resolved_vocab_files[file_id] = None else: - resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in s3_models: msg = "Couldn't reach server at '{}' to download vocabulary files." @@ -378,7 +401,8 @@ class PreTrainedTokenizer(object): # Did we saved some inputs and kwargs to reload ? tokenizer_config_file = resolved_vocab_files.pop('tokenizer_config_file', None) if tokenizer_config_file is not None: - init_kwargs = json.load(open(tokenizer_config_file, encoding="utf-8")) + with open(tokenizer_config_file, encoding="utf-8") as tokenizer_config_handle: + init_kwargs = json.load(tokenizer_config_handle) saved_init_inputs = init_kwargs.pop('init_inputs', ()) if not init_inputs: init_inputs = saved_init_inputs @@ -403,13 +427,18 @@ class PreTrainedTokenizer(object): if args_name not in init_kwargs: init_kwargs[args_name] = file_path if special_tokens_map_file is not None: - special_tokens_map = json.load(open(special_tokens_map_file, encoding="utf-8")) + with open(special_tokens_map_file, encoding="utf-8") as special_tokens_map_handle: + special_tokens_map = json.load(special_tokens_map_handle) for key, value in special_tokens_map.items(): if key not in init_kwargs: init_kwargs[key] = value # Instantiate tokenizer. - tokenizer = cls(*init_inputs, **init_kwargs) + try: + tokenizer = cls(*init_inputs, **init_kwargs) + except OSError: + OSError("Unable to load vocabulary from file. " + "Please check that the provided vocabulary is accessible and not corrupted.") # Save inputs and kwargs for saving and re-loading with ``save_pretrained`` tokenizer.init_inputs = init_inputs @@ -417,7 +446,8 @@ class PreTrainedTokenizer(object): # Add supplementary tokens. if added_tokens_file is not None: - added_tok_encoder = json.load(open(added_tokens_file, encoding="utf-8")) + with open(added_tokens_file, encoding="utf-8") as added_tokens_handle: + added_tok_encoder = json.load(added_tokens_handle) added_tok_decoder = {v:k for k, v in added_tok_encoder.items()} tokenizer.added_tokens_encoder.update(added_tok_encoder) tokenizer.added_tokens_decoder.update(added_tok_decoder) @@ -513,6 +543,8 @@ class PreTrainedTokenizer(object): to_add_tokens = [] for token in new_tokens: assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode)) + if self.init_kwargs.get('do_lower_case', False) and token not in self.all_special_tokens: + token = token.lower() if token != self.unk_token and \ self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \ token not in to_add_tokens: @@ -605,7 +637,25 @@ class PreTrainedTokenizer(object): vocabularies (BPE/SentencePieces/WordPieces). Take care of added tokens. + + text: The sequence to be encoded. + **kwargs: passed to the child `self.tokenize()` method """ + all_special_tokens = self.all_special_tokens + + def lowercase_text(t): + # convert non-special tokens to lowercase + escaped_special_toks = [re.escape(s_tok) for s_tok in all_special_tokens] + pattern = r'(' + r'|'.join(escaped_special_toks) + r')|' + \ + r'(.+?)' + return re.sub( + pattern, + lambda m: m.groups()[0] or m.groups()[1].lower(), + t) + + if self.init_kwargs.get('do_lower_case', False): + text = lowercase_text(text) + def split_on_token(tok, text): result = [] split_text = text.split(tok) @@ -625,7 +675,7 @@ class PreTrainedTokenizer(object): return result def split_on_tokens(tok_list, text): - if not text: + if not text.strip(): return [] if not tok_list: return self._tokenize(text, **kwargs) @@ -636,17 +686,17 @@ class PreTrainedTokenizer(object): tokenized_text = [] for sub_text in text_list: if sub_text not in self.added_tokens_encoder \ - and sub_text not in self.all_special_tokens: + and sub_text not in all_special_tokens: tokenized_text += split_on_token(tok, sub_text) else: tokenized_text += [sub_text] text_list = tokenized_text return list(itertools.chain.from_iterable((self._tokenize(token, **kwargs) if token not \ - in self.added_tokens_encoder and token not in self.all_special_tokens \ + in self.added_tokens_encoder and token not in all_special_tokens \ else [token] for token in tokenized_text))) - added_tokens = list(self.added_tokens_encoder.keys()) + self.all_special_tokens + added_tokens = list(self.added_tokens_encoder.keys()) + all_special_tokens tokenized_text = split_on_tokens(added_tokens, text) return tokenized_text @@ -692,6 +742,7 @@ class PreTrainedTokenizer(object): max_length=None, stride=0, truncation_strategy='longest_first', + pad_to_max_length=False, return_tensors=None, **kwargs): """ @@ -718,6 +769,12 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + pad_to_max_length: if set to True, the returned sequences will be padded according to the model's padding side and + padding index, up to their max length. If no max length is specified, the padding is done up to the model's max length. + The tokenizer padding sides are handled by the following strings: + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + Defaults to False: no padding. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. **kwargs: passed to the `self.tokenize()` method @@ -728,6 +785,7 @@ class PreTrainedTokenizer(object): add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, + pad_to_max_length=pad_to_max_length, return_tensors=return_tensors, **kwargs) @@ -740,7 +798,12 @@ class PreTrainedTokenizer(object): max_length=None, stride=0, truncation_strategy='longest_first', + pad_to_max_length=False, return_tensors=None, + return_token_type_ids=True, + return_attention_mask=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False, **kwargs): """ Returns a dictionary containing the encoded sequence or sequence pair and additional informations: @@ -765,9 +828,40 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + pad_to_max_length: if set to True, the returned sequences will be padded according to the model's padding side and + padding index, up to their max length. If no max length is specified, the padding is done up to the model's max length. + The tokenizer padding sides are handled by the following strings: + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + Defaults to False: no padding. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_attention_mask: (optional) Set to False to avoir returning attention mask (default True) + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). **kwargs: passed to the `self.tokenize()` method + + Return: + A Dictionary of shape:: + + { + input_ids: list[int], + token_type_ids: list[int] if return_token_type_ids is True (default) + attention_mask: list[int] if return_attention_mask is True (default) + overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True + num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True + } + + With the fields: + ``input_ids``: list of token ids to be fed to a model + ``token_type_ids``: list of token type ids to be fed to a model + ``attention_mask``: list of indices specifying which tokens should be attended to by the model + ``overflowing_tokens``: list of overflowing tokens if a max length is specified. + ``num_truncated_tokens``: number of overflowing tokens a ``max_length`` is specified + ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added + tokens and 1 specifying sequence tokens. """ def get_input_ids(text): @@ -786,13 +880,110 @@ class PreTrainedTokenizer(object): return self.prepare_for_model(first_ids, pair_ids=second_ids, max_length=max_length, + pad_to_max_length=pad_to_max_length, add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, - return_tensors=return_tensors) + return_tensors=return_tensors, + return_attention_mask=return_attention_mask, + return_token_type_ids=return_token_type_ids, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask) + + def batch_encode_plus(self, + batch_text_or_text_pairs=None, + add_special_tokens=False, + max_length=None, + stride=0, + truncation_strategy='longest_first', + return_tensors=None, + return_input_lengths=False, + return_attention_masks=False, + **kwargs): + """ + Returns a dictionary containing the encoded sequence or sequence pair and additional information: + the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. + + Args: + batch_text_or_text_pairs: Batch of sequences or pair of sequences to be encoded. + This can be a list of string/string-sequences/int-sequences or a list of pair of + string/string-sequences/int-sequence (see details in encode_plus) + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. + max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. + If there are overflowing tokens, those will be added to the returned dictionary` + stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens + from the main sequence returned. The value of this argument defines the number of additional tokens. + truncation_strategy: string selected in the following options: + - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_length + starting from the longest one at each token (when there is a pair of input sequences) + - 'only_first': Only truncate the first sequence + - 'only_second': Only truncate the second sequence + - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant + or PyTorch torch.Tensor instead of a list of python integers. + **kwargs: passed to the `self.tokenize()` method + """ + batch_outputs = {} + for ids_or_pair_ids in batch_text_or_text_pairs: + if isinstance(ids_or_pair_ids, (list, tuple)): + assert len(ids_or_pair_ids) == 2 + ids, pair_ids = ids_or_pair_ids + else: + ids, pair_ids = ids_or_pair_ids, None + outputs = self.encode_plus(ids, pair_ids, add_special_tokens=add_special_tokens, max_length=max_length, + stride=stride, truncation_strategy=truncation_strategy, return_tensors=None) + + # Append the non-padded length to the output + if return_input_lengths: + outputs['input_len'] = len(outputs['input_ids']) + + for key, value in outputs.items(): + if key not in batch_outputs: + batch_outputs[key] = [] + batch_outputs[key].append(value) + + # Compute longest sequence size + max_seq_len = max(map(len, batch_outputs['input_ids'])) + + if return_attention_masks: + # Allow the model to not give any special attention to padded input + batch_outputs['attention_mask'] = [[0] * len(v) for v in batch_outputs['input_ids']] + + if return_tensors is not None: + + # Do the tensor conversion in batch + for key, value in batch_outputs.items(): + + padded_value = value + if key != 'input_len': + # Padding handle + padded_value = [v + [self.pad_token_id if key == 'input_ids' else 1] * (max_seq_len - len(v)) for v in padded_value] + + if return_tensors == 'tf' and is_tf_available(): + batch_outputs[key] = tf.constant(padded_value) + elif return_tensors == 'pt' and is_torch_available(): + batch_outputs[key] = torch.tensor(padded_value) + elif return_tensors is not None: + logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) + + # encoder_attention_mask requires 1 for real token, 0 for padding, just invert value + if return_attention_masks: + if is_tf_available(): + batch_outputs['attention_mask'] = tf.abs(batch_outputs['attention_mask'] - 1) + else: + batch_outputs['attention_mask'] = torch.abs(batch_outputs['attention_mask'] - 1) + + return batch_outputs def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=True, stride=0, - truncation_strategy='longest_first', return_tensors=None): + truncation_strategy='longest_first', + pad_to_max_length=False, + return_tensors=None, + return_token_type_ids=True, + return_attention_mask=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False): """ Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It adds special tokens, truncates @@ -815,23 +1006,36 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + pad_to_max_length: if set to True, the returned sequences will be padded according to the model's padding side and + padding index, up to their max length. If no max length is specified, the padding is done up to the model's max length. + The tokenizer padding sides are handled by the following strings: + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + Defaults to False: no padding. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_attention_mask: (optional) Set to False to avoid returning attention mask (default True) + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). Return: A Dictionary of shape:: { input_ids: list[int], - overflowing_tokens: list[int] if a ``max_length`` is specified, else None - special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` + token_type_ids: list[int] if return_token_type_ids is True (default) + overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True + num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True } With the fields: - ``input_ids``: list of tokens to be fed to a model + ``input_ids``: list of token ids to be fed to a model + ``token_type_ids``: list of token type ids to be fed to a model ``overflowing_tokens``: list of overflowing tokens if a max length is specified. - + ``num_truncated_tokens``: number of overflowing tokens a ``max_length`` is specified ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added tokens and 1 specifying sequence tokens. """ @@ -840,45 +1044,99 @@ class PreTrainedTokenizer(object): len_pair_ids = len(pair_ids) if pair else 0 encoded_inputs = {} + + # Handle max sequence length total_len = len_ids + len_pair_ids + (self.num_added_tokens(pair=pair) if add_special_tokens else 0) if max_length and total_len > max_length: ids, pair_ids, overflowing_tokens = self.truncate_sequences(ids, pair_ids=pair_ids, num_tokens_to_remove=total_len-max_length, truncation_strategy=truncation_strategy, stride=stride) - encoded_inputs["overflowing_tokens"] = overflowing_tokens - encoded_inputs["num_truncated_tokens"] = total_len - max_length + if return_overflowing_tokens: + encoded_inputs["overflowing_tokens"] = overflowing_tokens + encoded_inputs["num_truncated_tokens"] = total_len - max_length + # Handle special_tokens if add_special_tokens: sequence = self.build_inputs_with_special_tokens(ids, pair_ids) token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids) - encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) else: sequence = ids + pair_ids if pair else ids token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) - if return_tensors == 'tf' and is_tf_available(): - sequence = tf.constant([sequence]) - token_type_ids = tf.constant([token_type_ids]) - elif return_tensors == 'pt' and is_torch_available(): - sequence = torch.tensor([sequence]) - token_type_ids = torch.tensor([token_type_ids]) - elif return_tensors is not None: - logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) encoded_inputs["input_ids"] = sequence - encoded_inputs["token_type_ids"] = token_type_ids + if return_token_type_ids: + encoded_inputs["token_type_ids"] = token_type_ids if max_length and len(encoded_inputs["input_ids"]) > max_length: encoded_inputs["input_ids"] = encoded_inputs["input_ids"][:max_length] - encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] - encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] + if return_token_type_ids: + encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] if max_length is None and len(encoded_inputs["input_ids"]) > self.max_len: logger.warning("Token indices sequence length is longer than the specified maximum sequence length " "for this model ({} > {}). Running this sequence through the model will result in " "indexing errors".format(len(ids), self.max_len)) + needs_to_be_padded = pad_to_max_length and ( + max_length and len(encoded_inputs["input_ids"]) < max_length + or + max_length is None and len(encoded_inputs["input_ids"]) < self.max_len and self.max_len <= 10000 + ) + + if pad_to_max_length and max_length is None and self.max_len > 10000: + logger.warning("Sequence can't be padded as no maximum length is specified and the model maximum length is too high.") + + if needs_to_be_padded: + difference = (max_length if max_length is not None else self.max_len) - len(encoded_inputs["input_ids"]) + + if self.padding_side == 'right': + if return_attention_mask: + encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + [0] * difference + if return_token_type_ids: + encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"] + [self.pad_token_type_id] * difference + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference + encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [self.pad_token_id] * difference + elif self.padding_side == 'left': + if return_attention_mask: + encoded_inputs["attention_mask"] = [0] * difference + [1] * len(encoded_inputs["input_ids"]) + if return_token_type_ids: + encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs["token_type_ids"] + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"] + encoded_inputs["input_ids"] = [self.pad_token_id] * difference + encoded_inputs["input_ids"] + + else: + raise ValueError("Invalid padding strategy:" + str(self.padding_side)) + + elif return_attention_mask: + encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + + # Prepare inputs as tensors if asked + if return_tensors == 'tf' and is_tf_available(): + encoded_inputs["input_ids"] = tf.constant([encoded_inputs["input_ids"]]) + encoded_inputs["token_type_ids"] = tf.constant([encoded_inputs["token_type_ids"]]) + + if "attention_mask" in encoded_inputs: + encoded_inputs["attention_mask"] = tf.constant([encoded_inputs["attention_mask"]]) + + elif return_tensors == 'pt' and is_torch_available(): + encoded_inputs["input_ids"] = torch.tensor([encoded_inputs["input_ids"]]) + encoded_inputs["token_type_ids"] = torch.tensor([encoded_inputs["token_type_ids"]]) + + if "attention_mask" in encoded_inputs: + encoded_inputs["attention_mask"] = torch.tensor([encoded_inputs["attention_mask"]]) + elif return_tensors is not None: + logger.warning( + "Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format( + return_tensors)) + return encoded_inputs def truncate_sequences(self, ids, pair_ids=None, num_tokens_to_remove=0, truncation_strategy='longest_first', stride=0): @@ -922,7 +1180,6 @@ class PreTrainedTokenizer(object): return (ids, pair_ids, overflowing_tokens) def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): - logger.warning("This tokenizer does not make use of special tokens.") if token_ids_1 is None: return len(token_ids_0) * [0] return [0] * len(token_ids_0) + [1] * len(token_ids_1) @@ -935,7 +1192,6 @@ class PreTrainedTokenizer(object): single sequence: X pair of sequences: A B """ - logger.warning("This tokenizer does not make use of special tokens. Input is returned with no modification.") if token_ids_1 is None: return token_ids_0 return token_ids_0 + token_ids_1 @@ -971,6 +1227,7 @@ class PreTrainedTokenizer(object): return self._convert_id_to_token(ids) tokens = [] for index in ids: + index = int(index) if skip_special_tokens and index in self.all_special_ids: continue if index in self.added_tokens_decoder: @@ -1014,12 +1271,12 @@ class PreTrainedTokenizer(object): if current_sub_text: sub_texts.append(self.convert_tokens_to_string(current_sub_text)) current_sub_text = [] - sub_texts.append(" " + token) + sub_texts.append(token) else: current_sub_text.append(token) if current_sub_text: sub_texts.append(self.convert_tokens_to_string(current_sub_text)) - text = ''.join(sub_texts) + text = ' '.join(sub_texts) if clean_up_tokenization_spaces: clean_text = self.clean_up_tokenization(text) diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py index 01f8721d..8def80be 100644 --- a/transformers/tokenization_xlm.py +++ b/transformers/tokenization_xlm.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tokenization classes for OpenAI GPT.""" +"""Tokenization classes for XLM.""" from __future__ import (absolute_import, division, print_function, unicode_literals) @@ -524,7 +524,7 @@ class XLMTokenizer(PreTrainedTokenizer): - argument ``special_tokens`` and function ``set_special_tokens``, can be used to add additional symbols \ (ex: "__classify__") to a vocabulary - + - `lang2id` attribute maps the languages supported by the model with their ids if provided (automatically set for pretrained vocabularies) - `id2lang` attributes does reverse mapping if provided (automatically set for pretrained vocabularies) @@ -549,6 +549,10 @@ class XLMTokenizer(PreTrainedTokenizer): additional_special_tokens=additional_special_tokens, **kwargs) + + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + # cache of sm.MosesPunctNormalizer instance self.cache_moses_punct_normalizer = dict() # cache of sm.MosesTokenizer instance @@ -564,9 +568,11 @@ class XLMTokenizer(PreTrainedTokenizer): self.ja_word_tokenizer = None self.zh_word_tokenizer = None - self.encoder = json.load(open(vocab_file, encoding="utf-8")) + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) self.decoder = {v:k for k,v in self.encoder.items()} - merges = open(merges_file, encoding='utf-8').read().split('\n')[:-1] + with open(merges_file, encoding='utf-8') as merges_handle: + merges = merges_handle.read().split('\n')[:-1] merges = [tuple(merge.split()[:2]) for merge in merges] self.bpe_ranks = dict(zip(merges, range(len(merges)))) self.cache = {} @@ -758,9 +764,9 @@ class XLMTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: + A XLM sequence has the following format: single sequence: X - pair of sequences: A B + pair of sequences: A B """ if token_ids_1 is None: return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] diff --git a/transformers/tokenization_xlm_roberta.py b/transformers/tokenization_xlm_roberta.py new file mode 100644 index 00000000..adbc8cd6 --- /dev/null +++ b/transformers/tokenization_xlm_roberta.py @@ -0,0 +1,182 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +""" Tokenization classes for XLM-RoBERTa model.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +import os +from shutil import copyfile + +import sentencepiece as spm +from transformers.tokenization_utils import PreTrainedTokenizer +from .tokenization_xlnet import SPIECE_UNDERLINE + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'xlm-roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-base-sentencepiece.bpe.model", + 'xlm-roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-sentencepiece.bpe.model", + 'xlm-roberta-large-finetuned-conll02-dutch': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-dutch-sentencepiece.bpe.model", + 'xlm-roberta-large-finetuned-conll02-spanish': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-spanish-sentencepiece.bpe.model", + 'xlm-roberta-large-finetuned-conll03-english': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-english-sentencepiece.bpe.model", + 'xlm-roberta-large-finetuned-conll03-german': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-german-sentencepiece.bpe.model", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'xlm-roberta-base': 512, + 'xlm-roberta-large': 512, + 'xlm-roberta-large-finetuned-conll02-dutch': 512, + 'xlm-roberta-large-finetuned-conll02-spanish': 512, + 'xlm-roberta-large-finetuned-conll03-english': 512, + 'xlm-roberta-large-finetuned-conll03-german': 512, +} + +class XLMRobertaTokenizer(PreTrainedTokenizer): + """ + Adapted from RobertaTokenizer and XLNetTokenizer + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + """ + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="", + cls_token="", unk_token="", pad_token='', mask_token='', + **kwargs): + super(XLMRobertaTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, + sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, + mask_token=mask_token, + **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(str(vocab_file)) + self.vocab_file = vocab_file + + # Original fairseq vocab and spm vocab must be "aligned": + # Vocab | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 + # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ---- + # fairseq | '' | '' | '' | '' | ',' | '.' | '▁' | 's' | '▁de' | '-' + # spm | '' | '' | '' | ',' | '.' | '▁' | 's' | '▁de' | '-' | '▁a' + + # Mimic fairseq token-to-id alignment for the first 4 token + self.fairseq_tokens_to_ids = {"": 0, "": 1, "": 2, "": 3} + + # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab + self.fairseq_offset = 1 + + self.fairseq_tokens_to_ids[''] = len(self.sp_model) + len(self.fairseq_tokens_to_ids) + self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()} + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A RoBERTa sequence has the following format: + single sequence: X + pair of sequences: A B + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A RoBERTa sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1] + + @property + def vocab_size(self): + return len(self.sp_model) + len(self.fairseq_tokens_to_ids) + + def _tokenize(self, text): + return self.sp_model.EncodeAsPieces(text) + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + if token in self.fairseq_tokens_to_ids: + return self.fairseq_tokens_to_ids[token] + return self.sp_model.PieceToId(token) + self.fairseq_offset + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + if index in self.fairseq_ids_to_tokens: + return self.fairseq_ids_to_tokens[index] + return self.sp_model.IdToPiece(index - self.fairseq_offset) + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (strings for sub-words) in a single string.""" + out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() + return out_string + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py index a4f1a6e3..a8369df6 100644 --- a/transformers/tokenization_xlnet.py +++ b/transformers/tokenization_xlnet.py @@ -60,6 +60,7 @@ class XLNetTokenizer(PreTrainedTokenizer): vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + padding_side = "left" def __init__(self, vocab_file, do_lower_case=False, remove_space=True, keep_accents=False, @@ -74,6 +75,7 @@ class XLNetTokenizer(PreTrainedTokenizer): self.max_len_single_sentence = self.max_len - 2 # take into account special tokens self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + self._pad_token_type_id = 3 try: import sentencepiece as spm @@ -141,7 +143,7 @@ class XLNetTokenizer(PreTrainedTokenizer): pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1) new_pieces = [] for piece in pieces: - if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit(): + if len(piece) > 1 and piece[-1] == str(',') and piece[-2].isdigit(): cur_pieces = self.sp_model.EncodeAsPieces( piece[:-1].replace(SPIECE_UNDERLINE, '')) if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: @@ -185,9 +187,9 @@ class XLNetTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: - single sequence: X - pair of sequences: A B + An XLNet sequence has the following format: + single sequence: X + pair of sequences: A B """ sep = [self.sep_token_id] cls = [self.cls_token_id] @@ -224,10 +226,10 @@ class XLNetTokenizer(PreTrainedTokenizer): def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. - A BERT sequence pair mask has the following format: + An XLNet sequence pair mask has the following format: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 | first sequence | second sequence | CLS segment ID - + if token_ids_1 is None, only returns the first portion of the mask (0's). """ sep = [self.sep_token_id] diff --git a/try.py b/try.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/download_glue_data.py b/utils/download_glue_data.py new file mode 100644 index 00000000..de8cfa9e --- /dev/null +++ b/utils/download_glue_data.py @@ -0,0 +1,142 @@ +''' Script for downloading all GLUE data. +Original source: https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e + +Note: for legal reasons, we are unable to host MRPC. +You can either use the version hosted by the SentEval team, which is already tokenized, +or you can download the original data from (https://download.microsoft.com/download/D/4/6/D46FF87A-F6B9-4252-AA8B-3604ED519838/MSRParaphraseCorpus.msi) and extract the data from it manually. +For Windows users, you can run the .msi file. For Mac and Linux users, consider an external library such as 'cabextract' (see below for an example). +You should then rename and place specific files in a folder (see below for an example). + +mkdir MRPC +cabextract MSRParaphraseCorpus.msi -d MRPC +cat MRPC/_2DEC3DBE877E4DB192D17C0256E90F1D | tr -d $'\r' > MRPC/msr_paraphrase_train.txt +cat MRPC/_D7B391F9EAFF4B1B8BCE8F21B20B1B61 | tr -d $'\r' > MRPC/msr_paraphrase_test.txt +rm MRPC/_* +rm MSRParaphraseCorpus.msi + +1/30/19: It looks like SentEval is no longer hosting their extracted and tokenized MRPC data, so you'll need to download the data from the original source for now. +2/11/19: It looks like SentEval actually *is* hosting the extracted data. Hooray! +''' + +import os +import sys +import shutil +import argparse +import tempfile +import urllib.request +import zipfile + +TASKS = ["CoLA", "SST", "MRPC", "QQP", "STS", "MNLI", "SNLI", "QNLI", "RTE", "WNLI", "diagnostic"] +TASK2PATH = {"CoLA":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FCoLA.zip?alt=media&token=46d5e637-3411-4188-bc44-5809b5bfb5f4', + "SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8', + "MRPC":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc', + "QQP":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQQP.zip?alt=media&token=700c6acf-160d-4d89-81d1-de4191d02cb5', + "STS":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSTS-B.zip?alt=media&token=bddb94a7-8706-4e0d-a694-1109e12273b5', + "MNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FMNLI.zip?alt=media&token=50329ea1-e339-40e2-809c-10c40afff3ce', + "SNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSNLI.zip?alt=media&token=4afcfbb2-ff0c-4b2d-a09a-dbf07926f4df', + "QNLI": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQNLIv2.zip?alt=media&token=6fdcf570-0fc5-4631-8456-9505272d1601', + "RTE":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FRTE.zip?alt=media&token=5efa7e85-a0bb-4f19-8ea2-9e1840f077fb', + "WNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FWNLI.zip?alt=media&token=068ad0a0-ded7-4bd7-99a5-5e00222e0faf', + "diagnostic":'https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D'} + +MRPC_TRAIN = 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt' +MRPC_TEST = 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt' + +def download_and_extract(task, data_dir): + print("Downloading and extracting %s..." % task) + data_file = "%s.zip" % task + urllib.request.urlretrieve(TASK2PATH[task], data_file) + with zipfile.ZipFile(data_file) as zip_ref: + zip_ref.extractall(data_dir) + os.remove(data_file) + print("\tCompleted!") + +def format_mrpc(data_dir, path_to_data): + print("Processing MRPC...") + mrpc_dir = os.path.join(data_dir, "MRPC") + if not os.path.isdir(mrpc_dir): + os.mkdir(mrpc_dir) + if path_to_data: + mrpc_train_file = os.path.join(path_to_data, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(path_to_data, "msr_paraphrase_test.txt") + else: + print("Local MRPC data not specified, downloading data from %s" % MRPC_TRAIN) + mrpc_train_file = os.path.join(mrpc_dir, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(mrpc_dir, "msr_paraphrase_test.txt") + urllib.request.urlretrieve(MRPC_TRAIN, mrpc_train_file) + urllib.request.urlretrieve(MRPC_TEST, mrpc_test_file) + assert os.path.isfile(mrpc_train_file), "Train data not found at %s" % mrpc_train_file + assert os.path.isfile(mrpc_test_file), "Test data not found at %s" % mrpc_test_file + urllib.request.urlretrieve(TASK2PATH["MRPC"], os.path.join(mrpc_dir, "dev_ids.tsv")) + + dev_ids = [] + with open(os.path.join(mrpc_dir, "dev_ids.tsv"), encoding="utf8") as ids_fh: + for row in ids_fh: + dev_ids.append(row.strip().split('\t')) + + with open(mrpc_train_file, encoding="utf8") as data_fh, \ + open(os.path.join(mrpc_dir, "train.tsv"), 'w', encoding="utf8") as train_fh, \ + open(os.path.join(mrpc_dir, "dev.tsv"), 'w', encoding="utf8") as dev_fh: + header = data_fh.readline() + train_fh.write(header) + dev_fh.write(header) + for row in data_fh: + label, id1, id2, s1, s2 = row.strip().split('\t') + if [id1, id2] in dev_ids: + dev_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + else: + train_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + + with open(mrpc_test_file, encoding="utf8") as data_fh, \ + open(os.path.join(mrpc_dir, "test.tsv"), 'w', encoding="utf8") as test_fh: + header = data_fh.readline() + test_fh.write("index\t#1 ID\t#2 ID\t#1 String\t#2 String\n") + for idx, row in enumerate(data_fh): + label, id1, id2, s1, s2 = row.strip().split('\t') + test_fh.write("%d\t%s\t%s\t%s\t%s\n" % (idx, id1, id2, s1, s2)) + print("\tCompleted!") + +def download_diagnostic(data_dir): + print("Downloading and extracting diagnostic...") + if not os.path.isdir(os.path.join(data_dir, "diagnostic")): + os.mkdir(os.path.join(data_dir, "diagnostic")) + data_file = os.path.join(data_dir, "diagnostic", "diagnostic.tsv") + urllib.request.urlretrieve(TASK2PATH["diagnostic"], data_file) + print("\tCompleted!") + return + +def get_tasks(task_names): + task_names = task_names.split(',') + if "all" in task_names: + tasks = TASKS + else: + tasks = [] + for task_name in task_names: + assert task_name in TASKS, "Task %s not found!" % task_name + tasks.append(task_name) + return tasks + +def main(arguments): + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', help='directory to save data to', type=str, default='glue_data') + parser.add_argument('--tasks', help='tasks to download data for as a comma separated string', + type=str, default='all') + parser.add_argument('--path_to_mrpc', help='path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt', + type=str, default='') + args = parser.parse_args(arguments) + + if not os.path.isdir(args.data_dir): + os.mkdir(args.data_dir) + tasks = get_tasks(args.tasks) + + for task in tasks: + if task == 'MRPC': + format_mrpc(args.data_dir, args.path_to_mrpc) + elif task == 'diagnostic': + download_diagnostic(args.data_dir) + else: + download_and_extract(task, args.data_dir) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/utils/link_tester.py b/utils/link_tester.py new file mode 100644 index 00000000..fe3990d2 --- /dev/null +++ b/utils/link_tester.py @@ -0,0 +1,79 @@ +""" Link tester. + +This little utility reads all the python files in the repository, +scans for links pointing to S3 and tests the links one by one. Raises an error +at the end of the scan if at least one link was reported broken. +""" +import os +import re +import sys + +import requests + + +REGEXP_FIND_S3_LINKS = r"""([\"'])(https:\/\/s3)(.*)?\1""" + + +def list_python_files_in_repository(): + """ List all python files in the repository. + + This function assumes that the script is executed in the root folder. + """ + source_code_files = [] + for path, subdirs, files in os.walk("."): + if "templates" in path: + continue + for name in files: + if ".py" in name and ".pyc" not in name: + path_to_files = os.path.join(path, name) + source_code_files.append(path_to_files) + + return source_code_files + + +def find_all_links(file_paths): + links = [] + for path in file_paths: + links += scan_code_for_links(path) + + return links + + +def scan_code_for_links(source): + """ Scans the file to find links using a regular expression. + Returns a list of links. + """ + with open(source, 'r') as content: + content = content.read() + raw_links = re.findall(REGEXP_FIND_S3_LINKS, content) + links = [prefix + suffix for _, prefix, suffix in raw_links] + + return links + + +def check_all_links(links): + """ Check that the provided links are valid. + + Links are considered valid if a HEAD request to the server + returns a 200 status code. + """ + broken_links = [] + for link in links: + head = requests.head(link) + if head.status_code != 200: + broken_links.append(link) + + return broken_links + + +if __name__ == "__main__": + file_paths = list_python_files_in_repository() + links = find_all_links(file_paths) + broken_links = check_all_links(links) + print("Looking for broken links to pre-trained models/configs/tokenizers...") + if broken_links: + print("The following links did not respond:") + for link in broken_links: + print("- {}".format(link)) + sys.exit(1) + print("All links are ok.") diff --git a/valohai.yaml b/valohai.yaml new file mode 100644 index 00000000..2573551b --- /dev/null +++ b/valohai.yaml @@ -0,0 +1,94 @@ +--- + +- step: + name: Execute python examples/run_glue.py + image: pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 + command: + - python /valohai/repository/utils/download_glue_data.py --data_dir=/glue_data + - pip install -e . + - pip install -r examples/requirements.txt + - python examples/run_glue.py --do_train --data_dir=/glue_data/{parameter-value:task_name} {parameters} + parameters: + - name: model_type + pass-as: --model_type={v} + type: string + default: bert + - name: model_name_or_path + pass-as: --model_name_or_path={v} + type: string + default: bert-base-uncased + - name: task_name + pass-as: --task_name={v} + type: string + default: MRPC + - name: max_seq_length + pass-as: --max_seq_length={v} + description: The maximum total input sequence length after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded. + type: integer + default: 128 + - name: per_gpu_train_batch_size + pass-as: --per_gpu_train_batch_size={v} + description: Batch size per GPU/CPU for training. + type: integer + default: 8 + - name: per_gpu_eval_batch_size + pass-as: --per_gpu_eval_batch_size={v} + description: Batch size per GPU/CPU for evaluation. + type: integer + default: 8 + - name: gradient_accumulation_steps + pass-as: --gradient_accumulation_steps={v} + description: Number of updates steps to accumulate before performing a backward/update pass. + type: integer + default: 1 + - name: learning_rate + pass-as: --learning_rate={v} + description: The initial learning rate for Adam. + type: float + default: 0.00005 + - name: adam_epsilon + pass-as: --adam_epsilon={v} + description: Epsilon for Adam optimizer. + type: float + default: 0.00000001 + - name: max_grad_norm + pass-as: --max_grad_norm={v} + description: Max gradient norm. + type: float + default: 1.0 + - name: num_train_epochs + pass-as: --num_train_epochs={v} + description: Total number of training epochs to perform. + type: integer + default: 3 + - name: max_steps + pass-as: --max_steps={v} + description: If > 0, set total number of training steps to perform. Override num_train_epochs. + type: integer + default: -1 + - name: warmup_steps + pass-as: --warmup_steps={v} + description: Linear warmup over warmup_steps. + type: integer + default: -1 + - name: logging_steps + pass-as: --logging_steps={v} + description: Log every X updates steps. + type: integer + default: 25 + - name: save_steps + pass-as: --save_steps={v} + description: Save checkpoint every X updates steps. + type: integer + default: -1 + - name: output_dir + pass-as: --output_dir={v} + type: string + default: /valohai/outputs + - name: evaluate_during_training + description: Run evaluation during training at each logging step. + type: flag + default: true + - name: do_lower_case + description: Set this flag if you are using an uncased model. + type: flag