simpler batch_load_train_bert [pr] (#8582)

don't think that buffer is really beneficial. 5% faster data_time and 1ms faster per step.
https://wandb.ai/chenyuxyz/MLPerf-BERT/runs/69c9lx8y/overview
This commit is contained in:
chenyu
2025-01-12 20:25:05 -05:00
committed by GitHub
parent 05e5de6a91
commit 994944920b
+1 -7
View File
@@ -223,14 +223,8 @@ def batch_load_train_bert(BS:int):
assert cycle_length > 0, "cycle_length must be greater than 0"
dataset = InterleavedDataset(train_files, cycle_length)
buffer = [dataset.get() for _ in range(1000)]
while True:
batch = []
for _ in range(BS):
index = random.randint(0, 999)
batch.append(buffer[index])
buffer[index] = dataset.get()
yield process_batch_bert(batch)
yield process_batch_bert([dataset.get() for _ in range(BS)])
# Reference: https://github.com/mlcommons/training/blob/1c8a098ae3e70962a4f7422c0b0bd35ae639e357/language_model/tensorflow/bert/run_pretraining.py, Line 416
def batch_load_val_bert(BS:int):