diff --git a/examples/mlperf/model_train.py b/examples/mlperf/model_train.py index 309485b8df..09ca95cd31 100644 --- a/examples/mlperf/model_train.py +++ b/examples/mlperf/model_train.py @@ -77,7 +77,7 @@ def train_resnet(): target, achieved = getenv("TARGET", 0.759), False eval_start_epoch = getenv("EVAL_START_EPOCH", 0) - eval_epochs = getenv("EVAL_EPOCHS", 1) + eval_freq = getenv("EVAL_FREQ", 1) steps_in_train_epoch = config["steps_in_train_epoch"] = (len(get_train_files()) // BS) steps_in_val_epoch = config["steps_in_val_epoch"] = (round_up(len(get_val_files()), EVAL_BS) // EVAL_BS) @@ -237,7 +237,8 @@ def train_resnet(): MLLOGGER.event(key=mllog_constants.EPOCH_STOP, value=e+1, metadata=dict(epoch_num=e+1)) # ** eval loop ** - if (e + 1 - eval_start_epoch) % eval_epochs == 0 and steps_in_val_epoch > 0: + # always eval for epoch >= 33 to stop the clock as soon as eval target hits, it can converge in epoch in [33, 37] + if steps_in_val_epoch > 0 and ((e + 1 - eval_start_epoch) % eval_freq == 0 or e + 1 >= 33): if MLLOGGER and RUNMLPERF: MLLOGGER.start(key=mllog_constants.EVAL_START, value=e+1, metadata=dict(epoch_num=e+1)) if getenv("RESET_STEP", 1): train_step.reset() # free the train step memory :( diff --git a/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/dev_run.sh b/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/dev_run.sh index b34f9c446e..6ff6846e2e 100755 --- a/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/dev_run.sh +++ b/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/dev_run.sh @@ -6,6 +6,8 @@ export SPLIT_REDUCEOP=1 LAZYCACHE=0 RESET_STEP=0 export TRAIN_BEAM=3 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=128 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=50 +export EVAL_START_EPOCH=3 EVAL_FREQ=4 + export WANDB=1 PARALLEL=0 python3 examples/mlperf/model_train.py diff --git a/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/run_and_time.sh b/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/run_and_time.sh index 99b99112bd..2c60088e96 100755 --- a/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/run_and_time.sh +++ b/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/run_and_time.sh @@ -20,4 +20,4 @@ LOGFILE="resnet_green_${DATETIME}_${SEED}.log" BENCHMARK=10 INITMLPERF=1 python3 examples/mlperf/model_train.py | tee $LOGFILE # run -WANDB=1 PARALLEL=0 RUNMLPERF=1 python3 examples/mlperf/model_train.py | tee -a $LOGFILE \ No newline at end of file +WANDB=1 PARALLEL=0 RUNMLPERF=1 EVAL_START_EPOCH=3 EVAL_FREQ=4 python3 examples/mlperf/model_train.py | tee -a $LOGFILE \ No newline at end of file diff --git a/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/dev_run.sh b/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/dev_run.sh index ddd7152b34..c4c2ec5907 100755 --- a/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/dev_run.sh +++ b/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/dev_run.sh @@ -6,6 +6,8 @@ export SPLIT_REDUCEOP=1 LAZYCACHE=0 RESET_STEP=0 export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=128 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=25 +export EVAL_START_EPOCH=3 EVAL_FREQ=4 + export WANDB=1 PARALLEL=0 python3 examples/mlperf/model_train.py diff --git a/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh b/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh index be97e0549c..990f60496c 100755 --- a/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh +++ b/examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh @@ -20,4 +20,4 @@ LOGFILE="resnet_red_${DATETIME}_${SEED}.log" BENCHMARK=10 INITMLPERF=1 python3 examples/mlperf/model_train.py | tee $LOGFILE # run -WANDB=1 PARALLEL=0 RUNMLPERF=1 python3 examples/mlperf/model_train.py | tee -a $LOGFILE \ No newline at end of file +WANDB=1 PARALLEL=0 RUNMLPERF=1 EVAL_START_EPOCH=3 EVAL_FREQ=4 python3 examples/mlperf/model_train.py | tee -a $LOGFILE \ No newline at end of file