fix(run_language_modeling): use arg overwrite_cache (#4407)
This commit is contained in:
1 parent
d39bf0ac2d
commit
d9ece8233d
1 file changed
+4
-1
@@ -120,7 +120,9 @@ def get_dataset(args: DataTrainingArguments, tokenizer: PreTrainedTokenizer, eva
|
|||||||
if args.line_by_line:
|
if args.line_by_line:
|
||||||
return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
|
return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
|
||||||
else:
|
else:
|
||||||
return TextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
|
return TextDataset(
|
||||||
|
tokenizer=tokenizer, file_path=file_path, block_size=args.block_size, overwrite_cache=args.overwrite_cache
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
@@ -216,6 +218,7 @@ def main():
|
|||||||
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
|
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
|
||||||
|
|
||||||
# Get datasets
|
# Get datasets
|
||||||
|
|
||||||
train_dataset = get_dataset(data_args, tokenizer=tokenizer) if training_args.do_train else None
|
train_dataset = get_dataset(data_args, tokenizer=tokenizer) if training_args.do_train else None
|
||||||
eval_dataset = get_dataset(data_args, tokenizer=tokenizer, evaluate=True) if training_args.do_eval else None
|
eval_dataset = get_dataset(data_args, tokenizer=tokenizer, evaluate=True) if training_args.do_eval else None
|
||||||
data_collator = DataCollatorForLanguageModeling(
|
data_collator = DataCollatorForLanguageModeling(
|
||||||
|
|||||||
Reference in new issue
Block a user