diff --git a/.gitignore b/.gitignore index a1b09eff5d..5ea6977d92 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ notebooks *.txt build !examples/tinychat/assets/cdn.jsdelivr.net/npm/purecss@3.0.0/build/ +!examples/mlperf/training_submission_*/**/*.txt /dist *.egg-info /env diff --git a/examples/mlperf/model_train.py b/examples/mlperf/model_train.py index b91a5faadd..ec8ef9d3f4 100644 --- a/examples/mlperf/model_train.py +++ b/examples/mlperf/model_train.py @@ -358,11 +358,43 @@ def train_retinanet(): config, target_metric = {}, 0.34 + config["SEED"] = SEED = getenv("SEED", random.SystemRandom().randint(0, 2**32 - 1)) + Tensor.manual_seed(SEED) + NUM_CLASSES = len(MLPERF_CLASSES) BASEDIR = getenv("BASEDIR", BASEDIR) BENCHMARK = getenv("BENCHMARK") - # INITMLPERF = getenv("INITMLPERF") + INITMLPERF = getenv("INITMLPERF") RUNMLPERF = getenv("RUNMLPERF") + + if getenv("LOGMLPERF"): + from mlperf_logging import mllog + import mlperf_logging.mllog.constants as mllog_constants + + mllog.config(filename=f"result_retinanet_{SEED}.log") + mllog.config(root_dir=Path(__file__).parents[3].as_posix()) + MLLOGGER = mllog.get_mllogger() + MLLOGGER.logger.propagate = False + + if INITMLPERF: + assert BENCHMARK, "BENCHMARK must be set for INITMLPERF" + MLLOGGER.event(key=mllog_constants.SUBMISSION_ORG, value="tinycorp") + MLLOGGER.event(key=mllog_constants.SUBMISSION_PLATFORM, value=getenv("SUBMISSION_PLATFORM", "tinybox")) + MLLOGGER.event(key=mllog_constants.SUBMISSION_DIVISION, value=mllog_constants.CLOSED) + MLLOGGER.event(key=mllog_constants.SUBMISSION_STATUS, value=mllog_constants.ONPREM) + + MLLOGGER.event(key=mllog_constants.SUBMISSION_BENCHMARK, value=mllog_constants.RETINANET) + + diskcache_clear() + MLLOGGER.event(key=mllog_constants.CACHE_CLEAR, value=True) + MLLOGGER.start(key=mllog_constants.INIT_START) + + if RUNMLPERF: + MLLOGGER.start(key=mllog_constants.RUN_START) + MLLOGGER.event(key=mllog_constants.SEED, value=SEED) + else: + MLLOGGER = None + config["gpus"] = GPUS = [f"{Device.DEFAULT}:{i}" for i in range(getenv("GPUS", 6))] for x in GPUS: Device[x] @@ -415,24 +447,21 @@ def train_retinanet(): return out.to(GPUS[0]).realize() # ** hyperparameters ** - config["seed"] = SEED = getenv("SEED", random.SystemRandom().randint(0, 2**32 - 1)) - config["bs"] = BS = getenv("BS", 16 * len(GPUS) if dtypes.default_float == dtypes.float16 else 12 * len(GPUS)) - config["eval_bs"] = EVAL_BS = getenv("EVAL_BS", BS) - config["epochs"] = EPOCHS = getenv("EPOCHS", 4) - config["train_beam"] = TRAIN_BEAM = getenv("TRAIN_BEAM", BEAM.value) - config["eval_beam"] = EVAL_BEAM = getenv("EVAL_BEAM", BEAM.value) - config["lr"] = lr = getenv("LR", 9.5e-5 * (BS / 96)) - config["loss_scaler"] = loss_scaler = getenv("LOSS_SCALER", 2**11 if dtypes.default_float == dtypes.float16 else 1.0) - config["default_float"] = dtypes.default_float.name - config["eval_freq"] = eval_freq = getenv("EVAL_FREQ", 1) + config["BS"] = BS = getenv("BS", 16 * len(GPUS) if dtypes.default_float == dtypes.float16 else 12 * len(GPUS)) + config["EVAL_BS"] = EVAL_BS = getenv("EVAL_BS", BS) + config["EPOCHS"] = EPOCHS = getenv("EPOCHS", 4) + config["TRAIN_BEAM"] = TRAIN_BEAM = getenv("TRAIN_BEAM", BEAM.value) + config["EVAL_BEAM"] = EVAL_BEAM = getenv("EVAL_BEAM", BEAM.value) + config["LR"] = lr = getenv("LR", 9.5e-5 * (BS / 96)) + config["LOSS_SCALER"] = loss_scaler = getenv("LOSS_SCALER", 2**11 if dtypes.default_float == dtypes.float16 else 1.0) + config["DEFAULT_FLOAT"] = dtypes.default_float.name + config["EVAL_FREQ"] = eval_freq = getenv("EVAL_FREQ", 1) # ** initialize wandb ** if (WANDB:=getenv("WANDB")): import wandb wandb.init(config=config, project="MLPerf-RetinaNet") - if SEED: Tensor.manual_seed(SEED) - # ** model initializers ** resnet.BatchNorm = FrozenBatchNorm2dRetinaNet resnet.Linear = Linear @@ -465,8 +494,24 @@ def train_retinanet(): optim = Adam(params, lr=lr) # ** dataset ** - config["steps_in_train_epoch"] = steps_in_train_epoch = round_up(get_dataset_count((base_dir_path:=Path(BASEDIR)), False), BS) // BS - config["steps_in_val_epoch"] = steps_in_val_epoch = (round_up(get_dataset_count(base_dir_path, True), EVAL_BS) // EVAL_BS) + config["STEPS_IN_TRAIN_EPOCH"] = steps_in_train_epoch = round_up(get_dataset_count((base_dir_path:=Path(BASEDIR)), False), BS) // BS + config["STEPS_IN_VAL_EPOCH"] = steps_in_val_epoch = (round_up(get_dataset_count(base_dir_path, True), EVAL_BS) // EVAL_BS) + + # log mlperf hparams + if MLLOGGER: + if RUNMLPERF: + MLLOGGER.event(key=mllog_constants.GLOBAL_BATCH_SIZE, value=config["BS"]) + MLLOGGER.event(key=mllog_constants.TRAIN_SAMPLES, value=config["STEPS_IN_TRAIN_EPOCH"]) + MLLOGGER.event(key=mllog_constants.EVAL_SAMPLES, value=config["STEPS_IN_VAL_EPOCH"]) + MLLOGGER.event(key=mllog_constants.EPOCH_COUNT, value=config["EPOCHS"]) + MLLOGGER.event(key=mllog_constants.FIRST_EPOCH_NUM, value=start_epoch) + + MLLOGGER.event(key=mllog_constants.OPT_NAME, value=mllog_constants.ADAM) + MLLOGGER.event(key=mllog_constants.OPT_BASE_LR, value=config["LR"]) + MLLOGGER.event(key=mllog_constants.OPT_WEIGHT_DECAY, value=0) + MLLOGGER.event(key=mllog_constants.OPT_LR_WARMUP_EPOCHS, value=0) + MLLOGGER.event(key=mllog_constants.OPT_LR_WARMUP_FACTOR, value=0) + MLLOGGER.event(key=mllog_constants.GRADIENT_ACCUMULATION_STEPS, value=1) if RUNMLPERF: train_dataset = COCO(download_dataset(BASEDIR, "train")) @@ -477,13 +522,16 @@ def train_retinanet(): for e in range(start_epoch, EPOCHS): # ** training loop ** + if MLLOGGER and RUNMLPERF: + MLLOGGER.start(key=mllog_constants.EPOCH_START, value=e + 1, metadata={"epoch_num": e + 1}) + BEAM.value = TRAIN_BEAM if not RUNMLPERF: i, proc = 0, _fake_data_get(BS) else: train_dataloader = batch_load_retinanet(train_dataset, False, base_dir_path, batch_size=BS, seed=SEED) - it = iter(tqdm(train_dataloader, total=steps_in_train_epoch, desc=f"epoch {e}", disable=BENCHMARK)) + it = iter(tqdm(train_dataloader, total=steps_in_train_epoch, desc=f"epoch {e + 1}", disable=BENCHMARK)) i, proc = 0, _data_get(it) prev_cookies = [] @@ -545,8 +593,14 @@ def train_retinanet(): if (TRAIN_BEAM or EVAL_BEAM) and e == start_epoch: break return + if MLLOGGER and RUNMLPERF: + MLLOGGER.event(key=mllog_constants.EPOCH_STOP, value=e + 1, metadata={"epoch_num": e + 1}) + # ** eval loop ** if (e + 1) % eval_freq == 0: + if MLLOGGER and RUNMLPERF: + MLLOGGER.start(key=mllog_constants.EVAL_START, value=e + 1, metadata={"epoch_num": e + 1}) + BEAM.value = EVAL_BEAM if getenv("RESET_STEP", 1): _train_step.reset() @@ -594,12 +648,15 @@ def train_retinanet(): proc, next_proc = next_proc, None i += 1 - if i == BENCHMARK: - return - et = time.time() eval_times.append(et - st) + if i == BENCHMARK: + # assume INITMLPERF has BENCHMARK set + if MLLOGGER and INITMLPERF: + MLLOGGER.event(key=mllog_constants.INIT_STOP) + return + if getenv("RESET_STEP", 1): _eval_step.reset() total_fw_time = sum(eval_times) / len(eval_times) @@ -617,8 +674,16 @@ def train_retinanet(): if WANDB: wandb.log({"eval/forward_time": total_fw_time, "eval/metric": val_metric, "epoch": e + 1}) + if MLLOGGER: + MLLOGGER.event(key=mllog_constants.EVAL_ACCURACY, value=val_metric, metadata={"epoch_num": e + 1}, clear_line=True) + MLLOGGER.end(key=mllog_constants.EVAL_STOP, value=e + 1, metadata={"epoch_num": e + 1}) + if val_metric >= target_metric: print(colored(f"target metric reached: {val_metric:.2f}/{target_metric:.2f}", color="green")) + + if MLLOGGER: + MLLOGGER.end(key=mllog_constants.RUN_STOP, metadata={"status": mllog_constants.SUCCESS}) + break def train_unet3d(): diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/README.md b/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/README.md new file mode 100644 index 0000000000..b0ff8f3259 --- /dev/null +++ b/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/README.md @@ -0,0 +1,38 @@ +# 1. Problem + +This problem uses RetinaNet for SSD. + +## Requirements + +Install tinygrad and mlperf-logging from master. +``` +git clone https://github.com/tinygrad/tinygrad.git +python3 -m pip install -e ".[mlperf]" +``` + +Also install the following dependencies: +``` +pip install tqdm numpy pycocotools boto3 pandas torch torchvision +``` + +### tinybox_green +Install the p2p driver per [README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md) +This is the default on production tinybox green. + +# 2. Directions + +## Steps to download data + +Run the following: +``` +BASEDIR=/raid/datasets/openimages python3 extra/datasets/openimages.py +``` + +## Running + +### tinybox_green + +#### Steps to run benchmark +``` +examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/run_and_time.sh +``` diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/tinybox_green/dev_beam.sh b/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/dev_beam.sh similarity index 100% rename from examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/tinybox_green/dev_beam.sh rename to examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/dev_beam.sh diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/tinybox_green/dev_run.sh b/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/dev_run.sh similarity index 100% rename from examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/tinybox_green/dev_run.sh rename to examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/dev_run.sh diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/run_and_time.sh b/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/run_and_time.sh new file mode 100755 index 0000000000..2c6d0d14fe --- /dev/null +++ b/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/run_and_time.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +export PYTHONPATH="." NV=1 +export MODEL="retinanet" +export SUBMISSION_PLATFORM="tinybox_green" +export DEFAULT_FLOAT="HALF" GPUS=6 BS=96 EVAL_BS=96 + +export TRAIN_BEAM=2 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0 +export IGNORE_JIT_FIRST_BEAM=1 +export BASEDIR="/raid/datasets/openimages" + +# pip install -e ".[mlperf]" +export LOGMLPERF=1 + +export SEED=$RANDOM +DATETIME=$(date "+%m%d%H%M") +LOGFILE="retinanet_green_${DATETIME}_${SEED}.log" + +# init +BENCHMARK=10 INITMLPERF=1 python3 examples/mlperf/model_train.py | tee $LOGFILE + +# run +PARALLEL=0 RUNMLPERF=1 python3 examples/mlperf/model_train.py | tee -a $LOGFILE diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/tinybox_red/dev_beam.sh b/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_red/dev_beam.sh similarity index 100% rename from examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/tinybox_red/dev_beam.sh rename to examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_red/dev_beam.sh diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/tinybox_red/dev_run.sh b/examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_red/dev_run.sh similarity index 100% rename from examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/tinybox_red/dev_run.sh rename to examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_red/dev_run.sh diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_0.txt b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_0.txt new file mode 100644 index 0000000000..9099264eff --- /dev/null +++ b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_0.txt @@ -0,0 +1,46 @@ +:::MLLOG {"namespace": "", "time_ms": 1745596628137, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}} +:::MLLOG {"namespace": "", "time_ms": 1745596628178, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}} +:::MLLOG {"namespace": "", "time_ms": 1745596628178, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 384}} +:::MLLOG {"namespace": "", "time_ms": 1745596628178, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}} +:::MLLOG {"namespace": "", "time_ms": 1745596628178, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 387}} +:::MLLOG {"namespace": "", "time_ms": 1745596629954, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 390}} +:::MLLOG {"namespace": "", "time_ms": 1745596629955, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 391}} +:::MLLOG {"namespace": "", "time_ms": 1745598065772, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 658}} +:::MLLOG {"namespace": "", "time_ms": 1745598081470, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 394}} +:::MLLOG {"namespace": "", "time_ms": 1745598081512, "event_type": "POINT_IN_TIME", "key": "seed", "value": 23282, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 395}} +:::MLLOG {"namespace": "", "time_ms": 1745598088273, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}} +:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}} +:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}} +:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 507}} +:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}} +:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}} +:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}} +:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}} +:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}} +:::MLLOG {"namespace": "", "time_ms": 1745598088275, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 514}} +:::MLLOG {"namespace": "", "time_ms": 1745598088275, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 515}} +:::MLLOG {"namespace": "", "time_ms": 1745598144406, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 527, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745605078062, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 598, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745605078063, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 603, "epoch_num": 1}} + +:::MLLOG {"namespace": "", "time_ms": 1745610378469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.2608930553164607, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 679, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745610378469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 680, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745610378469, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 527, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745616941326, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 598, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745616941327, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 603, "epoch_num": 2}} + +:::MLLOG {"namespace": "", "time_ms": 1745622185857, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.31207695716564665, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 679, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745622185858, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 680, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745622185858, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 527, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745628713800, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 598, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745628713800, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 603, "epoch_num": 3}} + +:::MLLOG {"namespace": "", "time_ms": 1745633828548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.32695300496649193, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 679, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745633828548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 680, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745633828549, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 527, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745640403678, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 598, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745640403679, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 603, "epoch_num": 4}} + +:::MLLOG {"namespace": "", "time_ms": 1745645485614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.34190927146960864, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 679, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745645485615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 680, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745645485615, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 686, "status": "success"}} diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_1.txt b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_1.txt new file mode 100644 index 0000000000..b600b209c7 --- /dev/null +++ b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_1.txt @@ -0,0 +1,46 @@ +:::MLLOG {"namespace": "", "time_ms": 1745708052929, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 380}} +:::MLLOG {"namespace": "", "time_ms": 1745708052970, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 381}} +:::MLLOG {"namespace": "", "time_ms": 1745708052970, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}} +:::MLLOG {"namespace": "", "time_ms": 1745708052970, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}} +:::MLLOG {"namespace": "", "time_ms": 1745708052970, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}} +:::MLLOG {"namespace": "", "time_ms": 1745708055312, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 388}} +:::MLLOG {"namespace": "", "time_ms": 1745708055312, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 389}} +:::MLLOG {"namespace": "", "time_ms": 1745709484510, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 656}} +:::MLLOG {"namespace": "", "time_ms": 1745709499880, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 392}} +:::MLLOG {"namespace": "", "time_ms": 1745709499922, "event_type": "POINT_IN_TIME", "key": "seed", "value": 3218, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 393}} +:::MLLOG {"namespace": "", "time_ms": 1745709506804, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 502}} +:::MLLOG {"namespace": "", "time_ms": 1745709506805, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 503}} +:::MLLOG {"namespace": "", "time_ms": 1745709506805, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}} +:::MLLOG {"namespace": "", "time_ms": 1745709506805, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}} +:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}} +:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}} +:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 509}} +:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}} +:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}} +:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}} +:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}} +:::MLLOG {"namespace": "", "time_ms": 1745709564057, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745716423332, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745716423333, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 1}} + +:::MLLOG {"namespace": "", "time_ms": 1745721892086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.2644758301871188, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745721892087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745721892087, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745728717917, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745728717918, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 2}} + +:::MLLOG {"namespace": "", "time_ms": 1745734129092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3183940553292647, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745734129092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745734129092, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745740758848, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745740758849, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 3}} + +:::MLLOG {"namespace": "", "time_ms": 1745746017219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.330829179299047, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745746017219, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745746017219, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745752685505, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745752685506, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 4}} + +:::MLLOG {"namespace": "", "time_ms": 1745757915230, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3430538198992862, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745757915231, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745757915231, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 684, "status": "success"}} diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_2.txt b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_2.txt new file mode 100644 index 0000000000..69c56fb045 --- /dev/null +++ b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_2.txt @@ -0,0 +1,46 @@ +:::MLLOG {"namespace": "", "time_ms": 1745757942370, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 380}} +:::MLLOG {"namespace": "", "time_ms": 1745757942411, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 381}} +:::MLLOG {"namespace": "", "time_ms": 1745757942411, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}} +:::MLLOG {"namespace": "", "time_ms": 1745757942411, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}} +:::MLLOG {"namespace": "", "time_ms": 1745757942411, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}} +:::MLLOG {"namespace": "", "time_ms": 1745757943058, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 388}} +:::MLLOG {"namespace": "", "time_ms": 1745757943059, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 389}} +:::MLLOG {"namespace": "", "time_ms": 1745759379793, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 656}} +:::MLLOG {"namespace": "", "time_ms": 1745759394363, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 392}} +:::MLLOG {"namespace": "", "time_ms": 1745759394404, "event_type": "POINT_IN_TIME", "key": "seed", "value": 7068, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 393}} +:::MLLOG {"namespace": "", "time_ms": 1745759401265, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 502}} +:::MLLOG {"namespace": "", "time_ms": 1745759401265, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 503}} +:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}} +:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}} +:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}} +:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}} +:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 509}} +:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}} +:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}} +:::MLLOG {"namespace": "", "time_ms": 1745759401267, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}} +:::MLLOG {"namespace": "", "time_ms": 1745759401267, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}} +:::MLLOG {"namespace": "", "time_ms": 1745759458864, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745766229351, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745766229352, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 1}} + +:::MLLOG {"namespace": "", "time_ms": 1745771664180, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.2618442233208197, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745771664180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745771664181, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745778271730, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745778271731, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 2}} + +:::MLLOG {"namespace": "", "time_ms": 1745783649281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3050222595524408, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745783649281, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745783649282, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745790269694, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745790269695, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 3}} + +:::MLLOG {"namespace": "", "time_ms": 1745795622203, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.33438554461867026, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745795622204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745795622204, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745802201592, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745802201593, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 4}} + +:::MLLOG {"namespace": "", "time_ms": 1745807409644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3413173788267323, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745807409644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745807409644, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 684, "status": "success"}} diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_3.txt b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_3.txt new file mode 100644 index 0000000000..65cf8c44a5 --- /dev/null +++ b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_3.txt @@ -0,0 +1,46 @@ +:::MLLOG {"namespace": "", "time_ms": 1745708740629, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 380}} +:::MLLOG {"namespace": "", "time_ms": 1745708740670, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 381}} +:::MLLOG {"namespace": "", "time_ms": 1745708740670, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}} +:::MLLOG {"namespace": "", "time_ms": 1745708740670, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}} +:::MLLOG {"namespace": "", "time_ms": 1745708740670, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}} +:::MLLOG {"namespace": "", "time_ms": 1745708753515, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 388}} +:::MLLOG {"namespace": "", "time_ms": 1745708753515, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 389}} +:::MLLOG {"namespace": "", "time_ms": 1745710196875, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 656}} +:::MLLOG {"namespace": "", "time_ms": 1745710211866, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 392}} +:::MLLOG {"namespace": "", "time_ms": 1745710211906, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1934, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 393}} +:::MLLOG {"namespace": "", "time_ms": 1745710219928, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 502}} +:::MLLOG {"namespace": "", "time_ms": 1745710219929, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 503}} +:::MLLOG {"namespace": "", "time_ms": 1745710219929, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}} +:::MLLOG {"namespace": "", "time_ms": 1745710219929, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}} +:::MLLOG {"namespace": "", "time_ms": 1745710219929, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}} +:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}} +:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 509}} +:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}} +:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}} +:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}} +:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}} +:::MLLOG {"namespace": "", "time_ms": 1745710276595, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745717038732, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745717038733, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 1}} + +:::MLLOG {"namespace": "", "time_ms": 1745722476155, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.24994336549495808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745722476156, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745722476156, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745729177485, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745729177486, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 2}} + +:::MLLOG {"namespace": "", "time_ms": 1745734589630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.30947442932060776, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745734589630, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745734589630, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745741107714, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745741107714, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 3}} + +:::MLLOG {"namespace": "", "time_ms": 1745746523920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3304143886715271, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745746523920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745746523920, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745753134001, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745753134002, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 4}} + +:::MLLOG {"namespace": "", "time_ms": 1745758428287, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3429861420134466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745758428288, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745758428288, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 684, "status": "success"}} diff --git a/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_4.txt b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_4.txt new file mode 100644 index 0000000000..8f538dfecc --- /dev/null +++ b/examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_4.txt @@ -0,0 +1,45 @@ +:::MLLOG {"namespace": "", "time_ms": 1745758455763, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 380}} +:::MLLOG {"namespace": "", "time_ms": 1745758455804, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 381}} +:::MLLOG {"namespace": "", "time_ms": 1745758455804, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}} +:::MLLOG {"namespace": "", "time_ms": 1745758455804, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}} +:::MLLOG {"namespace": "", "time_ms": 1745758455805, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}} +:::MLLOG {"namespace": "", "time_ms": 1745758457940, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 388}} +:::MLLOG {"namespace": "", "time_ms": 1745758457941, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 389}} +:::MLLOG {"namespace": "", "time_ms": 1745759900517, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 656}} +:::MLLOG {"namespace": "", "time_ms": 1745759915495, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 392}} +:::MLLOG {"namespace": "", "time_ms": 1745759915536, "event_type": "POINT_IN_TIME", "key": "seed", "value": 25159, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 393}} +:::MLLOG {"namespace": "", "time_ms": 1745759922365, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 502}} +:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 503}} +:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}} +:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}} +:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}} +:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}} +:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 509}} +:::MLLOG {"namespace": "", "time_ms": 1745759922367, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}} +:::MLLOG {"namespace": "", "time_ms": 1745759922367, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}} +:::MLLOG {"namespace": "", "time_ms": 1745759922367, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}} +:::MLLOG {"namespace": "", "time_ms": 1745759922367, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}} +:::MLLOG {"namespace": "", "time_ms": 1745759981024, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745766937876, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745766937877, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 1}} + +:::MLLOG {"namespace": "", "time_ms": 1745772433927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.25660616888772175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745772433927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 1}} +:::MLLOG {"namespace": "", "time_ms": 1745772433927, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745779249804, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745779249805, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 2}} + +:::MLLOG {"namespace": "", "time_ms": 1745784709047, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3114751446994825, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745784709048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 2}} +:::MLLOG {"namespace": "", "time_ms": 1745784709048, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745791366481, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745791366482, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 3}} + +:::MLLOG {"namespace": "", "time_ms": 1745796796512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.33395135022162803, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745796796512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 3}} +:::MLLOG {"namespace": "", "time_ms": 1745796796512, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745803562272, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745803562273, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 4}} + +:::MLLOG {"namespace": "", "time_ms": 1745808971898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3397162205764848, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 4}} +:::MLLOG {"namespace": "", "time_ms": 1745808971899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 4}}