Sacremoses in TransfoXL

This commit is contained in:
LysandreJik
2020-11-16 16:34:58 -05:00
parent 791b6928eb
commit e411222a47
+6 -2
View File
@@ -27,8 +27,6 @@ from typing import List, Optional, Tuple
import numpy as np
import sacremoses as sm
from .file_utils import cached_path, is_torch_available, torch_only_method
from .tokenization_utils import PreTrainedTokenizer
from .utils import logging
@@ -40,6 +38,12 @@ if is_torch_available():
logger = logging.get_logger(__name__)
try:
import sacremoses as sm
except ModuleNotFoundError:
logger.error("Sacremoses not found. Will not be able to instantiate an XLM tokenizer.")
VOCAB_FILES_NAMES = {
"pretrained_vocab_file": "vocab.pkl",
"pretrained_vocab_file_torch": "vocab.bin",