From 128cfdee9bdd13f2f0bbef977907ff014ae398ad Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 19:28:16 +0100 Subject: [PATCH] tokenization add XLM-RoBERTa base model --- transformers/tokenization_xlm_roberta.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transformers/tokenization_xlm_roberta.py b/transformers/tokenization_xlm_roberta.py index d8484e7f..93b1c397 100644 --- a/transformers/tokenization_xlm_roberta.py +++ b/transformers/tokenization_xlm_roberta.py @@ -30,11 +30,13 @@ VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { + 'xlm-roberta-base': "https://schweter.eu/cloud/transformers/xlm-roberta-base-sentencepiece.bpe.model", 'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-sentencepiece.bpe.model", } } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'xlm-roberta-base': None, 'xlm-roberta-large': None, }