Sacremoses in TransfoXL

src/transformers/tokenization_fsmt.py

@@ -21,8 +21,6 @@ import re
 import unicodedata
 from typing import Dict, List, Optional, Tuple
 
-import sacremoses as sm
-
 from .file_utils import add_start_docstrings
 from .tokenization_utils import BatchEncoding, PreTrainedTokenizer
 from .tokenization_utils_base import PREPARE_SEQ2SEQ_BATCH_DOCSTRING
@@ -31,6 +29,11 @@ from .utils import logging
 
 logger = logging.get_logger(__name__)
 
+try:
+    import sacremoses as sm
+except ModuleNotFoundError:
+    logger.error("Sacremoses not found. Will not be able to instantiate an XLM tokenizer.")
+
 VOCAB_FILES_NAMES = {
     "src_vocab_file": "vocab-src.json",
     "tgt_vocab_file": "vocab-tgt.json",

src/transformers/tokenization_transfo_xl.py

@@ -27,8 +27,6 @@ from typing import List, Optional, Tuple
 
 import numpy as np
 
-import sacremoses as sm
-
 from .file_utils import cached_path, is_torch_available, torch_only_method
 from .tokenization_utils import PreTrainedTokenizer
 from .utils import logging
@@ -40,6 +38,12 @@ if is_torch_available():
 
 logger = logging.get_logger(__name__)
 
+try:
+    import sacremoses as sm
+except ModuleNotFoundError:
+    logger.error("Sacremoses not found. Will not be able to instantiate an XLM tokenizer.")
+
+
 VOCAB_FILES_NAMES = {
     "pretrained_vocab_file": "vocab.pkl",
     "pretrained_vocab_file_torch": "vocab.bin",

src/transformers/tokenization_xlm.py

@@ -22,7 +22,6 @@ import sys
 import unicodedata
 from typing import List, Optional, Tuple
 
-import sacremoses as sm
 
 from .tokenization_utils import PreTrainedTokenizer
 from .utils import logging
@@ -30,6 +29,11 @@ from .utils import logging
 
 logger = logging.get_logger(__name__)
 
+try:
+    import sacremoses as sm
+except ModuleNotFoundError:
+    logger.error("Sacremoses not found. Will not be able to instantiate an XLM tokenizer.")
+
 VOCAB_FILES_NAMES = {
     "vocab_file": "vocab.json",
     "merges_file": "merges.txt",