Fixed up the notes on a possible future low-memory path
This commit is contained in:
1 parent
a8a577ba93
commit
8a861048dd
2 files changed
+3
-3
No files matched your search
@@ -54,7 +54,7 @@ def convert_example_to_features(example, tokenizer, max_seq_length):
|
||||
|
||||
class PregeneratedDataset(Dataset):
|
||||
def __init__(self, training_path, epoch, tokenizer, num_data_epochs):
|
||||
# TODO Add an option to memmap and shuffle the training data if needed (see note in pregenerate_training_data)
|
||||
# TODO Add an option to memmap the training data if needed (see note in pregenerate_training_data)
|
||||
self.vocab = tokenizer.vocab
|
||||
self.tokenizer = tokenizer
|
||||
self.epoch = epoch
|
||||
|
||||
Reference in new issue
Block a user