Fixed up the notes on a possible future low-memory path

This commit is contained in:
Matthew Carrigan committed 2019-03-21 14:08:39 +00:00
1 parent a8a577ba93
commit 8a861048dd
2 files changed
+3 -3

No files matched your search

@@ -54,7 +54,7 @@ def convert_example_to_features(example, tokenizer, max_seq_length):
class PregeneratedDataset(Dataset):
def __init__(self, training_path, epoch, tokenizer, num_data_epochs):
# TODO Add an option to memmap and shuffle the training data if needed (see note in pregenerate_training_data)
# TODO Add an option to memmap the training data if needed (see note in pregenerate_training_data)
self.vocab = tokenizer.vocab
self.tokenizer = tokenizer
self.epoch = epoch