c-my/transformers
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pass config to AutoTokenizer.from_pretrained for Rag tokenizers

Browse Source
This commit is contained in:
Quentin Lhoest committed 2020-09-15 13:00:00 +02:00
1 parent 2faaa4ad3c
commit 00a1fc9ae4
1 file changed
+4 -4
Download Patch File Download Diff File Expand all files Collapse all files
src/transformers/tokenization_rag.py
+4 -4
View File
@@ -78,18 +78,18 @@ class RagTokenizer:
if os.path.isfile(save_directory):
logger.error("Provided path ({}) should be a directory, not a file".format(save_directory))
return
os.makedirs(save_directory)
os.makedirs(save_directory, exist_ok=True)
question_encoder_tokenizer_path = os.path.join(save_directory, "question_encoder_tokenizer")
generator_tokenizer_path = os.path.join(save_directory, "generator_tokenizer")
self.question_encoder_tokenizer.save_pretrained(question_encoder_tokenizer_path)
self.generator_tokenizer.save_pretrained(generator_tokenizer_path)
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path):
def from_pretrained(cls, pretrained_model_name_or_path, config):
question_encoder_tokenizer_path = os.path.join(pretrained_model_name_or_path, "question_encoder_tokenizer")
generator_tokenizer_path = os.path.join(pretrained_model_name_or_path, "generator_tokenizer")
question_encoder_tokenizer = AutoTokenizer.from_pretrained(question_encoder_tokenizer_path)
generator_tokenizer = AutoTokenizer.from_pretrained(generator_tokenizer_path)
question_encoder_tokenizer = AutoTokenizer.from_pretrained(question_encoder_tokenizer_path, config=config.question_encoder)
generator_tokenizer = AutoTokenizer.from_pretrained(generator_tokenizer_path, config=config.generator)
return cls(question_encoder_tokenizer=question_encoder_tokenizer, generator_tokenizer=generator_tokenizer)
def __call__(self, *args, **kwargs):