feat: results

* gradient: add TRUNC backward

* test: move round quantization gradient to test_ops

.github/workflows/autogen.yml

@@ -33,12 +33,8 @@ jobs:
       uses: ./.github/actions/setup-tinygrad
       with:
         key: 'autogen'
-        opencl: 'true'
         amd: 'true'
-        cuda: 'true'
         llvm: 'true'
-        webgpu: 'true'
-        mesa: 'true'
         pydeps: 'pyyaml mako'
     - name: Install autogen support packages
       run: sudo apt-get install -y --no-install-recommends libclang-20-dev llvm-20-dev hip-dev libusb-1.0-0-dev libdrm-dev
@@ -48,7 +44,7 @@ jobs:
         python3 -c "from tinygrad.runtime.autogen import opencl"
         python3 -c "from tinygrad.runtime.autogen import cuda, nvrtc, nvjitlink, nv_570, nv_580, nv"
         python3 -c "from tinygrad.runtime.autogen import comgr_3, hsa, hip, amd_gpu, sqtt, rocprof, amdgpu_kd, amdgpu_drm"
-        python3 -c "from tinygrad.runtime.autogen.am import am, pm4_soc15, pm4_nv, sdma_4_0_0, sdma_5_0_0, sdma_6_0_0, smu_v13_0_0, smu_v13_0_6, smu_v13_0_12, smu_v14_0_2"
+        python3 -c "from tinygrad.runtime.autogen.am import *"
         python3 -c "from tinygrad.runtime.autogen import libc, kfd, io_uring, ib, pci, vfio"
         python3 -c "from tinygrad.runtime.autogen import llvm"
         python3 -c "from tinygrad.runtime.autogen import webgpu"

.github/workflows/benchmark.yml

@@ -51,34 +51,35 @@ jobs:
     - name: openpilot compile3 0.10.1 driving_vision
       run: FLOAT16=1 DEV=CL IMAGE=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_vision.onnx
 
-  testframeworkpytest:
-    name: framework pytest
-    env:
-      CI: ""
-      CAPTURE_PROCESS_REPLAY: "0"
-    runs-on: [self-hosted, framework]
-    timeout-minutes: 10
-    defaults:
-      run:
-        shell: bash -e -o pipefail {0}
-    if: github.repository_owner == 'tinygrad'
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v6
-    - name: setup python environment
-      run: |
-        rm -rf /tmp/tinygrad_pytest_ci
-        uv venv /tmp/tinygrad_pytest_ci
-        source /tmp/tinygrad_pytest_ci/bin/activate
-        uv pip install .[testing]
-    - name: setup staging db
-      run: |
-        echo "CACHEDB=/tmp/pytest-db-ci.db" >> $GITHUB_ENV
-        rm -f /tmp/pytest-db-ci*
-    - name: Run pytest -nauto
-      run: |
-        source /tmp/tinygrad_pytest_ci/bin/activate
-        pytest -nauto --durations=20
+  # TODO: reenable when not flaky
+  #testframeworkpytest:
+  #  name: framework pytest
+  #  env:
+  #    CI: ""
+  #    CAPTURE_PROCESS_REPLAY: "0"
+  #  runs-on: [self-hosted, framework]
+  #  timeout-minutes: 10
+  #  defaults:
+  #    run:
+  #      shell: bash -e -o pipefail {0}
+  #  if: github.repository_owner == 'tinygrad'
+  #  steps:
+  #  - name: Checkout Code
+  #    uses: actions/checkout@v6
+  #  - name: setup python environment
+  #    run: |
+  #      rm -rf /tmp/tinygrad_pytest_ci
+  #      uv venv /tmp/tinygrad_pytest_ci
+  #      source /tmp/tinygrad_pytest_ci/bin/activate
+  #      uv pip install .[testing]
+  #  - name: setup staging db
+  #    run: |
+  #      echo "CACHEDB=/tmp/pytest-db-ci.db" >> $GITHUB_ENV
+  #      rm -f /tmp/pytest-db-ci*
+  #  - name: Run pytest -nauto
+  #    run: |
+  #      source /tmp/tinygrad_pytest_ci/bin/activate
+  #      pytest -nauto --durations=20
 
   testmacbenchmark:
     name: Mac Benchmark
@@ -189,7 +190,7 @@ jobs:
         path: |
           onnx_inference_speed.csv
     - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3.11 process_replay.py
+      uses: ./.github/actions/process-replay
 
   testusbgpu:
     name: UsbGPU Benchmark
@@ -324,7 +325,7 @@ jobs:
         path: |
           onnx_inference_speed.csv
     - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
+      uses: ./.github/actions/process-replay
 
   testmorenvidiabenchmark:
     name: tinybox green Training Benchmark
@@ -386,7 +387,7 @@ jobs:
       # TODO: remove BERT_LAYERS once scheduler is fast
       run: BENCHMARK_LOG=bert_10steps_6gpu DEV=NV CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
     - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
+      uses: ./.github/actions/process-replay
 
   testamdbenchmark:
     name: tinybox red Benchmark
@@ -498,7 +499,7 @@ jobs:
     - name: Run GPT2 w HALF/BEAM
       run: BENCHMARK_LOG=gpt2_half_beam DEV=AMD HALF=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing
     - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
+      uses: ./.github/actions/process-replay
 
   testmoreamdbenchmark:
     name: tinybox red Training Benchmark
@@ -555,7 +556,7 @@ jobs:
     #- name: Test full tinyfs load
     #  run: TINYFS_ENDPOINT=10.0.52.11:6767 PYTHONPATH=. python extra/tinyfs/fetch_file.py --hash d734f5e3be9f1e9d863bfaa4fc6c1ef2 --len 175866113 --dest mapping.json --check
     - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
+      uses: ./.github/actions/process-replay
 
   testmlperfamdbenchmark:
     name: tinybox red MLPerf Benchmark
@@ -601,7 +602,7 @@ jobs:
       # TODO: remove BERT_LAYERS once scheduler is fast
       run: BENCHMARK_LOG=bert_10steps_6gpu DEV=AMD CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
     - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
+      uses: ./.github/actions/process-replay
 
   testqualcommbenchmark:
     name: comma Benchmark
@@ -628,7 +629,7 @@ jobs:
     - name: IR3 openpilot compile3 0.11.0 driving_vision
       run: BENCHMARK_LOG=ir3_openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx
     - name: openpilot compile3 0.11.0 driving_policy
-      run: BENCHMARK_LOG=openpilot_0_11_0_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=4 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx
+      run: BENCHMARK_LOG=openpilot_0_11_0_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx
     - name: openpilot compile3 0.11.0 dmonitoring
       run: BENCHMARK_LOG=openpilot_0_11_0_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx
     - name: DEBUG=2 openpilot compile3 0.10.1 driving_vision
@@ -636,7 +637,7 @@ jobs:
     - name: openpilot compile3 0.10.1 driving_vision
       run: BENCHMARK_LOG=openpilot_0_10_1_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_vision.onnx
     - name: openpilot compile3 0.10.1 driving_policy
-      run: BENCHMARK_LOG=openpilot_0_10_1_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=4 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_policy.onnx
+      run: BENCHMARK_LOG=openpilot_0_10_1_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_policy.onnx
     - name: openpilot compile3 0.10.1 dmonitoring
       run: BENCHMARK_LOG=openpilot_0_10_1_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/dmonitoring_model.onnx
     - name: benchmark MobileNetV2 on DSP
@@ -648,7 +649,7 @@ jobs:
         # benchmark on DSP with NOOPT=1, the devectorizer has issues
         PYTHONPATH=. CC=clang-19 DEV=DSP NOOPT=1 CNT=2 DEBUG=2 python3 examples/test_onnx_imagenet.py /tmp/model.quant.onnx
     - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
+      uses: ./.github/actions/process-replay
 
   testcommausbgpubenchmark:
     name: UsbGPU Benchmark (comma)
@@ -745,7 +746,7 @@ jobs:
         DEBUG=2 PYTHONPATH=. REMOTE=127.0.0.1:6482 AM_RESET=1 DEV=PCI+AMD AMD_AQL=1 python3 test/test_tiny.py
         pkill -f 'extra/remote/serve.py' || true
     - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
+      uses: ./.github/actions/process-replay
 
   testgreendriverbenchmark:
     name: NV Benchmark
@@ -808,4 +809,4 @@ jobs:
         DEBUG=2 PYTHONPATH=. REMOTE=127.0.0.1:6483 DEV=NV python3 test/test_tiny.py
         pkill -f 'extra/remote/serve.py' || true
     - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
+      uses: ./.github/actions/process-replay

.github/workflows/test.yml

@@ -333,7 +333,7 @@ jobs:
         deps: testing_unit
         python-version: '3.14'
     - name: Test SPEC=2
-      run: SPEC=2 pytest --maxfail=10 -n auto --durations=30 test/unit test/backend test/opt --ignore test/backend/test_custom_kernel.py --ignore test/unit/test_hashing.py --timeout 60 -k "not test_setitem_big" --splits 2 --group ${{ matrix.group }}
+      run: SPEC=2 pytest --maxfail=10 -n auto --durations=30 test/unit test/backend test/opt --ignore test/backend/test_custom_kernel.py --ignore test/unit/test_hashing.py --timeout 60 -k "not test_setitem_big" -k "not test_conv2d_ceildiv_edge_case" --splits 2 --group ${{ matrix.group }}
 
   fuzzing:
     name: Fuzzing
@@ -417,7 +417,7 @@ jobs:
           llvm: 'true'
       - name: Test openpilot model kernel count and gate usage
         run: |
-          ALLOWED_KERNEL_COUNT=123 ALLOWED_READ_IMAGE=1486 ALLOWED_GATED_READ_IMAGE=17 FLOAT16=1 DEV=CL IMAGE=1 python examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/cf6376aa9a090f0da26c280ef69eabf9bbdd51d1faac9ed392919c3db69be916
+          ALLOWED_KERNEL_COUNT=123 ALLOWED_READ_IMAGE=1486 ALLOWED_GATED_READ_IMAGE=18 FLOAT16=1 DEV=CL IMAGE=1 python examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/cf6376aa9a090f0da26c280ef69eabf9bbdd51d1faac9ed392919c3db69be916
       - name: Test openpilot CL compile fp16
         run: FLOAT16=1 DEV=CL IMAGE=1 python examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/cf6376aa9a090f0da26c280ef69eabf9bbdd51d1faac9ed392919c3db69be916
       - name: Test openpilot CL compile fp32 (test correctness)

docs/runtime.md

@@ -5,7 +5,7 @@ tinygrad supports various runtimes, enabling your code to scale across a wide ra
 | Runtime | Description | Compiler Options | Requirements |
 |---------|-------------|------------------|--------------|
 | [NV](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_nv.py) | Provides acceleration for NVIDIA GPUs | nvrtc (default)<br>PTX (`DEV=NV:PTX`) | Ampere/Ada/Blackwell series GPUs.<br>You can select an interface via [the `DEV` variable](env_vars.md#dev-variable). See [NV interfaces](#nv-interfaces) for details. |
-| [AMD](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_amd.py) | Provides acceleration for AMD GPUs | LLVM (`DEV=AMD:LLVM`)<br>HIP/COMGR (`DEV=AMD:HIP`) | RDNA2 or newer GPUs.<br>You can select an interface via [the `DEV` variable](env_vars.md#dev-variable). See [AMD interfaces](#amd-interfaces) for details. |
+| [AMD](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_amd.py) | Provides acceleration for AMD GPUs | LLVM (`DEV=AMD:LLVM`)<br>HIP/COMGR (`DEV=AMD:HIP`) | CDNA3, CDNA4, RDNA3 or RDNA4 GPUs.<br>You can select an interface via [the `DEV` variable](env_vars.md#dev-variable). See [AMD interfaces](#amd-interfaces) for details. |
 | [QCOM](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_qcom.py) | Provides acceleration for QCOM GPUs | - | 6xx series GPUs |
 | [METAL](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_metal.py) | Utilizes Metal for acceleration on Apple devices | - | M1+ Macs; Metal 3.0+ for `bfloat` support |
 | [CUDA](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cuda.py) | Utilizes CUDA for acceleration on NVIDIA GPUs | nvrtc (default)<br> PTX (`DEV=CUDA:PTX`) | NVIDIA GPU with CUDA support |

docs/tensor/elementwise.md

@@ -66,8 +66,8 @@ Elementwise ops operate on a per element basis. They don't change the shape of t
 ::: tinygrad.Tensor.sub
 ::: tinygrad.Tensor.mul
 ::: tinygrad.Tensor.div
-::: tinygrad.Tensor.idiv
 ::: tinygrad.Tensor.mod
+::: tinygrad.Tensor.fmod
 ::: tinygrad.Tensor.bitwise_xor
 ::: tinygrad.Tensor.bitwise_and
 ::: tinygrad.Tensor.bitwise_or

docs/tinygpu.md

@@ -4,7 +4,7 @@ TinyGPU app lets you use AMD and NVIDIA GPUs on macOS over USB4/Thunderbolt with
 
 ## Requirements
 
-- macOS (12.1+)
+- macOS (13.0+)
 - USB4/Thunderbolt port
 - A supported GPU (AMD RDNA3+ or NVIDIA Ampere+)
 

examples/llama3.py

@@ -123,7 +123,7 @@ def NF4Linear(block_size):
     def __call__(self, x: Tensor) -> Tensor:
       high_bits = self.weight
       low_bits = (self.weight * 2 ** 4).contiguous()
-      unpacked = Tensor.stack(high_bits, low_bits, dim=-1).idiv(2 ** 4)
+      unpacked = Tensor.stack(high_bits, low_bits, dim=-1).div(2 ** 4, rounding_mode="trunc")
       unscaled = CODE[unpacked].to(x.device).reshape(-1, block_size) * self.scale
       return x.linear(unscaled.reshape(self.out_features, self.in_features).T)
 

examples/mlperf/model_train.py

@@ -1419,7 +1419,10 @@ def train_llama3():
 
   for p in optim.params:
     grad_dtype = dtypes.bfloat16 if p.dtype == FP8_DTYPE else p.dtype
-    p.grad = Tensor.zeros(p.shape, dtype=grad_dtype, device=p.device).contiguous()
+    if isinstance(p.device, tuple) and p.uop.axis is not None:
+      p.grad = Tensor.zeros(p.shape, dtype=grad_dtype, device=p.device[0]).shard_(p.device, axis=p.uop.axis).contiguous()
+    else:
+      p.grad = Tensor.zeros(p.shape, dtype=grad_dtype, device=p.device).contiguous()
   grads = [p.grad for p in optim.params]
 
   scheduler = CosineAnnealingLRWithWarmup(optim, opt_base_learning_rate, opt_end_learning_rate, opt_learning_rate_warmup_steps, opt_learning_rate_decay_steps)
@@ -1446,6 +1449,10 @@ def train_llama3():
       idx = next(j for j, p in enumerate(optim.params) if p is w)
       optim.master_params[idx].assign((optim.master_params[idx] * w._inv_scale.reshape(-1, *([1]*(w.ndim-1)))).contiguous())
 
+  # realize everything here
+  if optim.master_params: Tensor.realize(*optim.master_params)
+  Tensor.realize(*optim.params, *fp8_inv_scales, *fp8_amax, *fp8_grad_amax)
+
   @TinyJit
   def minibatch(tokens:Tensor):
     if is_dp: tokens = tokens.to(None).shard(device, 0)

examples/mlperf/models/flat_llama.py

@@ -158,14 +158,14 @@ class FlatTransformer:
 
     xq, xk = apply_rotary_emb(xq, xk, freqs_cis)
     xq, xk, xv = xq.cast(dtypes.bfloat16), xk.cast(dtypes.bfloat16), xv.cast(dtypes.bfloat16)
-    xq, xk, xv = xq.transpose(1, 2), xk.transpose(1, 2), xv.transpose(1, 2)
     if getenv("HK_FLASH_ATTENTION"):
       from extra.thunder.amd.fa import flash_attention
       attn, *save = flash_attention(xq, xk, xv, is_causal=True)
       saves.extend(save)
     else:
-      attn = xq.scaled_dot_product_attention(xk, xv, is_causal=True, enable_gqa=True)
-    attn = attn.transpose(1, 2).reshape(bsz, seqlen, -1)
+      xq, xk, xv = xq.transpose(1, 2), xk.transpose(1, 2), xv.transpose(1, 2)
+      attn = xq.scaled_dot_product_attention(xk, xv, is_causal=True, enable_gqa=True).transpose(1, 2)
+    attn = attn.reshape(bsz, seqlen, -1)
 
     out, *ret = matmul(attn, wo, amax_x=amax_xo, w_inv_scale=s_o, grad_amax_state=grad_amax_xo)
     new_amaxs.extend(ret[:1])
@@ -257,18 +257,19 @@ def _get_pads(uop:UOp) -> list[UOp]:
 
 def apply_grad(grad_buf:Tensor, new_grad:UOp):
   pads = _get_pads(new_grad)
-  new_grad = new_grad.cast(grad_buf.dtype)
   if len(pads) <= 1:
+    new_grad = new_grad.cast(grad_buf.dtype)
     store = grad_buf.uop.store(grad_buf.uop + new_grad)
     grad_buf.uop = grad_buf.uop.after(store)
     return
   sorted_pads = sorted(pads, key=lambda p: p.marg[0][0] if p.op == Ops.PAD else 0)
-  inners = [Tensor(p.src[0] if p.op == Ops.PAD else p, device=grad_buf.device).cast(grad_buf.dtype) for p in sorted_pads]
+  inners_raw = [Tensor(p.src[0] if p.op == Ops.PAD else p, device=grad_buf.device) for p in sorted_pads]
   if getenv("FUSED_PAD_GRAD_ACCUM", 0):
     from extra.llama_kernels.fused_pad_grad_accum import fused_pad_grad_accum, can_fused_pad_grad_accum
-    if can_fused_pad_grad_accum(grad_buf, inners):
-      grad_buf.uop = fused_pad_grad_accum(grad_buf, inners).uop
+    if can_fused_pad_grad_accum(grad_buf, inners_raw):
+      grad_buf.uop = fused_pad_grad_accum(grad_buf, inners_raw).uop
       return
+  inners = [t.cast(grad_buf.dtype) for t in inners_raw]
   grad_buf.assign(grad_buf + inners[0].cat(*inners[1:], dim=0))
 
 if __name__ == "__main__":

examples/mlperf/optim.py

@@ -81,7 +81,7 @@ class GradAccClipAdamW(Optimizer):
     if STOCHASTIC_ROUND and t.dtype == dtypes.bfloat16: return stochastic_round_bf16(new_w)
     if t.dtype in dtypes.fp8s:
       from examples.mlperf.models.flat_llama import FP8_MAX
-      amax = new_w.float().abs().flatten(1).max(1).detach()  # per-layer amax for (n_layers, out, in)
+      amax = new_w.float().abs().max(axis=tuple(range(1, new_w.ndim))).detach()  # per-layer amax for (n_layers, out, in)
       scale = FP8_MAX / (amax + 1e-8)
       fp8_w = (new_w * scale.reshape(-1, *([1]*(new_w.ndim-1)))).clamp(-FP8_MAX, FP8_MAX).cast(t.dtype)
       if hasattr(t, '_inv_scale'):

examples/mlperf/training_submission_v6.0/.gitignore

@@ -0,0 +1 @@
+!*.txt

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_1xMI300X/dev_beam.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" GPUS=1 BS=128 EVAL_BS=128
-
-export CHECK_OOB=0
-
-export BEAM=3 BEAM_UOPS_MAX=4000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1
-# export BEAM_LOG_SURPASS_MAX=1
-# export BASEDIR="/raid/datasets/wiki"
-
-export RESET_STEP=1
-export BENCHMARK=10 BERT_LAYERS=2 DEBUG=2
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/README.md

@@ -1,69 +0,0 @@
-# 1. Problem
-
-This problem uses BERT for NLP.
-
-## Requirements
-
-Install tinygrad and mlperf-logging (uncomment mlperf from setup.py) from branch mlperf_training_v5.0.
-```
-git clone https://github.com/tinygrad/tinygrad.git
-python3 -m pip install -e ".[mlperf]"
-```
-Also install gdown (for dataset), numpy, tqdm and tensorflow.
-```
-pip install gdown numpy tqdm tensorflow
-```
-
-### tinybox_green
-Install the p2p driver per [README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md)
-This is the default on production tinybox green.
-
-# 2. Directions
-
-## Steps to download and verify data
-
-### 1. Download raw data
-
-```
-BASEDIR="/raid/datasets/wiki" WIKI_TRAIN=1 VERIFY_CHECKSUM=1 python3 extra/datasets/wikipedia_download.py
-```
-
-### 2. Preprocess train and validation data
-
-Note: The number of threads used for preprocessing is limited by available memory. With 128GB of RAM, a maximum of 16 threads is recommended. 
-
-#### Training:
-```
-BASEDIR="/raid/datasets/wiki" NUM_WORKERS=16 python3 extra/datasets/wikipedia.py pre-train all
-```
-
-Generating a specific topic (Between 0 and 499)
-```
-BASEDIR="/raid/datasets/wiki" python3 extra/datasets/wikipedia.py pre-train 42
-```
-
-#### Validation:
-```
-BASEDIR="/raid/datasets/wiki" python3 extra/datasets/wikipedia.py pre-eval
-```
-## Running
-
-### tinybox_green
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_green/run_and_time.sh
-```
-
-### tinybox_red
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_red/run_and_time.sh
-```
-### tinybox_8xMI300X
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/run_and_time.sh
-```

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/dev_beam.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
-export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=500000
-
-export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1 FREE_INTERMEDIATE=0
-export BASEDIR="/raid/datasets/wiki"
-
-export BENCHMARK=10 BERT_LAYERS=2
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/dev_run.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
-
-# similar to https://github.com/mlcommons/training_results_v3.1/blob/d06288b2bd675a9d88e0e6181f5bb5626b71ec19/Quanta_Cloud_Technology/results/D54U-3U/bert/result_1.txt#L54
-export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
-export TRAIN_STEPS=3900
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=500000
-
-export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1 FREE_INTERMEDIATE=0
-export BASEDIR="/raid/datasets/wiki"
-
-export WANDB=1 PARALLEL=0
-
-RUNMLPERF=1 python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/run_and_time.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-set -e  # Exit on any error
-set -o pipefail  # Make pipeline fail if any command fails
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="bert"
-export SUBMISSION_PLATFORM="tinybox_8xMI300X"
-export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
-
-# similar to https://github.com/mlcommons/training_results_v3.1/blob/d06288b2bd675a9d88e0e6181f5bb5626b71ec19/Quanta_Cloud_Technology/results/D54U-3U/bert/result_1.txt#L54
-export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
-export TRAIN_STEPS=3900
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=500000
-
-export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1 FREE_INTERMEDIATE=0
-export BASEDIR="/raid/datasets/wiki"
-
-# pip install -e ".[mlperf]"
-export LOGMLPERF=1
-
-export SEED=$RANDOM
-DATETIME=$(date "+%m%d%H%M")
-LOGFILE="bert_8xMI300x_${DATETIME}_${SEED}.log"
-
-BENCHMARK=10 INITMLPERF=1 BERT_LAYERS=2 python3 examples/mlperf/model_train.py | tee $LOGFILE
-
-# run
-PARALLEL=0 RUNMLPERF=1 python3 examples/mlperf/model_train.py | tee -a $LOGFILE

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI350X/dev_run_fa.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD DEBUG=0 JIT=1 FLASH_ATTENTION=1
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
-
-# similar to https://github.com/mlcommons/training_results_v3.1/blob/d06288b2bd675a9d88e0e6181f5bb5626b71ec19/Quanta_Cloud_Technology/results/D54U-3U/bert/result_1.txt#L54
-export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
-export TRAIN_STEPS=3900
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=5000000
-
-export BEAM=0 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1 FREE_INTERMEDIATE=0
-export BASEDIR="/raid/datasets/wiki"
-
-export WANDB=1 PARALLEL=0
-
-RUNMLPERF=1 python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI350X/dev_run_fp8.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
-
-# similar to https://github.com/mlcommons/training_results_v3.1/blob/d06288b2bd675a9d88e0e6181f5bb5626b71ec19/Quanta_Cloud_Technology/results/D54U-3U/bert/result_1.txt#L54
-export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
-export TRAIN_STEPS=3900
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=5000000
-
-export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1 FREE_INTERMEDIATE=0
-export BASEDIR="/raid/datasets/wiki"
-export BEAM_TIMEOUT_SEC=15
-export FP8_TRAIN=1
-# search
-IGNORE_BEAM_CACHE=1 BENCHMARK=10 BERT_LAYERS=2 RUNMLPERF=0 python3 examples/mlperf/model_train.py
-
-export WANDB=1 PARALLEL=0
-
-RUNMLPERF=1 python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI350X/run_and_time.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-set -e  # Exit on any error
-set -o pipefail  # Make pipeline fail if any command fails
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="bert"
-export SUBMISSION_PLATFORM="tinybox_8xMI350X"
-export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
-
-# similar to https://github.com/mlcommons/training_results_v3.1/blob/d06288b2bd675a9d88e0e6181f5bb5626b71ec19/Quanta_Cloud_Technology/results/D54U-3U/bert/result_1.txt#L54
-export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
-export TRAIN_STEPS=3900
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=5000000
-
-export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1 FREE_INTERMEDIATE=0
-export BASEDIR="/raid/datasets/wiki"
-
-# pip install -e ".[mlperf]"
-export LOGMLPERF=1
-
-export SEED=$RANDOM
-DATETIME=$(date "+%m%d%H%M")
-LOGFILE="bert_8xMI350x_${DATETIME}_${SEED}.log"
-
-BENCHMARK=10 INITMLPERF=1 BERT_LAYERS=2 python3 examples/mlperf/model_train.py | tee $LOGFILE
-
-# run
-PARALLEL=0 RUNMLPERF=1 python3 examples/mlperf/model_train.py | tee -a $LOGFILE

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_green/README.md

@@ -1,69 +0,0 @@
-# 1. Problem
-
-This problem uses BERT for NLP.
-
-## Requirements
-
-Install tinygrad and mlperf-logging (uncomment mlperf from setup.py) from branch mlperf_training_v5.0.
-```
-git clone https://github.com/tinygrad/tinygrad.git
-python3 -m pip install -e ".[mlperf]"
-```
-Also install gdown (for dataset), numpy, tqdm and tensorflow.
-```
-pip install gdown numpy tqdm tensorflow
-```
-
-### tinybox_green
-Install the p2p driver per [README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md)
-This is the default on production tinybox green.
-
-# 2. Directions
-
-## Steps to download and verify data
-
-### 1. Download raw data
-
-```
-BASEDIR="/raid/datasets/wiki" WIKI_TRAIN=1 VERIFY_CHECKSUM=1 python3 extra/datasets/wikipedia_download.py
-```
-
-### 2. Preprocess train and validation data
-
-Note: The number of threads used for preprocessing is limited by available memory. With 128GB of RAM, a maximum of 16 threads is recommended. 
-
-#### Training:
-```
-BASEDIR="/raid/datasets/wiki" NUM_WORKERS=16 python3 extra/datasets/wikipedia.py pre-train all
-```
-
-Generating a specific topic (Between 0 and 499)
-```
-BASEDIR="/raid/datasets/wiki" python3 extra/datasets/wikipedia.py pre-train 42
-```
-
-#### Validation:
-```
-BASEDIR="/raid/datasets/wiki" python3 extra/datasets/wikipedia.py pre-eval
-```
-## Running
-
-### tinybox_green
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_green/run_and_time.sh
-```
-
-### tinybox_red
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_red/run_and_time.sh
-```
-### tinybox_8xMI300X
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/run_and_time.sh
-```

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_green/dev_beam.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=NV
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=72 EVAL_BS=72
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=500000
-
-export BEAM=8 BEAM_UOPS_MAX=10000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1
-export BEAM_LOG_SURPASS_MAX=1
-export BASEDIR="/raid/datasets/wiki"
-
-export BENCHMARK=10 BERT_LAYERS=2 DEBUG=2
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_green/dev_run.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=NV
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=72 EVAL_BS=72
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=500000
-
-export BEAM=8 BEAM_UOPS_MAX=10000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1
-export BASEDIR="/raid/datasets/wiki"
-
-export WANDB=1 PARALLEL=0
-
-RUNMLPERF=1 python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_green/run_and_time.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-set -e  # Exit on any error
-set -o pipefail  # Make pipeline fail if any command fails
-
-export PYTHONPATH="." DEV=NV
-export MODEL="bert"
-export SUBMISSION_PLATFORM="tinybox_green"
-export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=72 EVAL_BS=72
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=500000
-
-export BEAM=8 BEAM_UOPS_MAX=10000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1
-export BASEDIR="/raid/datasets/wiki"
-
-# pip install -e ".[mlperf]"
-export LOGMLPERF=1
-
-export SEED=$RANDOM
-DATETIME=$(date "+%m%d%H%M")
-LOGFILE="bert_green_${DATETIME}_${SEED}.log"
-
-# init
-BENCHMARK=10 INITMLPERF=1 BERT_LAYERS=2 python3 examples/mlperf/model_train.py | tee $LOGFILE
-
-# run
-PARALLEL=0 RUNMLPERF=1 python3 examples/mlperf/model_train.py | tee -a $LOGFILE

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_red/README.md

@@ -1,69 +0,0 @@
-# 1. Problem
-
-This problem uses BERT for NLP.
-
-## Requirements
-
-Install tinygrad and mlperf-logging (uncomment mlperf from setup.py) from branch mlperf_training_v5.0.
-```
-git clone https://github.com/tinygrad/tinygrad.git
-python3 -m pip install -e ".[mlperf]"
-```
-Also install gdown (for dataset), numpy, tqdm and tensorflow.
-```
-pip install gdown numpy tqdm tensorflow
-```
-
-### tinybox_green
-Install the p2p driver per [README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md)
-This is the default on production tinybox green.
-
-# 2. Directions
-
-## Steps to download and verify data
-
-### 1. Download raw data
-
-```
-BASEDIR="/raid/datasets/wiki" WIKI_TRAIN=1 VERIFY_CHECKSUM=1 python3 extra/datasets/wikipedia_download.py
-```
-
-### 2. Preprocess train and validation data
-
-Note: The number of threads used for preprocessing is limited by available memory. With 128GB of RAM, a maximum of 16 threads is recommended. 
-
-#### Training:
-```
-BASEDIR="/raid/datasets/wiki" NUM_WORKERS=16 python3 extra/datasets/wikipedia.py pre-train all
-```
-
-Generating a specific topic (Between 0 and 499)
-```
-BASEDIR="/raid/datasets/wiki" python3 extra/datasets/wikipedia.py pre-train 42
-```
-
-#### Validation:
-```
-BASEDIR="/raid/datasets/wiki" python3 extra/datasets/wikipedia.py pre-eval
-```
-## Running
-
-### tinybox_green
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_green/run_and_time.sh
-```
-
-### tinybox_red
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_red/run_and_time.sh
-```
-### tinybox_8xMI300X
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/run_and_time.sh
-```

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_red/dev_beam.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=96 EVAL_BS=96
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=500000
-
-export BEAM=5 BEAM_UOPS_MAX=8000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1
-export BEAM_LOG_SURPASS_MAX=1
-export BASEDIR="/raid/datasets/wiki"
-
-export RESET_STEP=1
-export BENCHMARK=10 BERT_LAYERS=2 DEBUG=2
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_red/dev_run.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=96 EVAL_BS=96
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=500000
-
-export BEAM=5 BEAM_UOPS_MAX=8000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1
-export BASEDIR="/raid/datasets/wiki"
-
-export WANDB=1 PARALLEL=0
-
-RUNMLPERF=1 python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_red/run_and_time.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-set -e  # Exit on any error
-set -o pipefail  # Make pipeline fail if any command fails
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="bert"
-export SUBMISSION_PLATFORM="tinybox_red"
-export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=96 EVAL_BS=96
-
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=500000
-
-export BEAM=5 BEAM_UOPS_MAX=8000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1
-export BASEDIR="/raid/datasets/wiki"
-
-# pip install -e ".[mlperf]"
-export LOGMLPERF=${LOGMLPERF:-1}
-
-export SEED=$RANDOM
-DATETIME=$(date "+%m%d%H%M")
-LOGFILE="bert_red_${DATETIME}_${SEED}.log"
-
-export HCQDEV_WAIT_TIMEOUT_MS=100000  # prevents hang?
-
-# init
-sleep 5 && sudo rmmod amdgpu || true
-BENCHMARK=10 INITMLPERF=1 BERT_LAYERS=2 python3 examples/mlperf/model_train.py | tee $LOGFILE
-
-# run
-PARALLEL=0 RUNMLPERF=1 python3 examples/mlperf/model_train.py | tee -a $LOGFILE

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/README.md

@@ -0,0 +1,28 @@
+# 1. Problem
+
+small llm pretraining: llama 3.1 8b on c4.
+
+## Requirements
+
+Install tinygrad and mlperf-logging (uncomment mlperf from setup.py) from branch mlperf_training_v6.0.
+```
+git clone https://github.com/tinygrad/tinygrad.git
+python3 -m pip install -e ".[mlperf]"
+```
+
+# 2. Directions
+
+## Steps to download and verify data
+
+### 1. Download raw data
+
+follow mlperf steps to download the preprocessed c4 dataset.
+
+## Running
+
+### tinybox_8xMI350X
+
+#### Steps to run benchmark
+```
+examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh
+```

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh

@@ -9,13 +9,14 @@ export DEVICE_IN_FUNCTION_BUG=1
 export DEBUG=${DEBUG:-2}
 export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1}
 export ALL2ALL=${ALL2ALL:-1}
+export LATE_ALLREDUCE=${LATE_ALLREDUCE:-0}
 export USE_ATOMICS=${USE_ATOMICS:-1}
 export ASM_GEMM=${ASM_GEMM:-1}
 export WQKV=${WQKV:-1}
 export MASTER_WEIGHTS=${MASTER_WEIGHTS:-1}
 export FP8=${FP8:-1}
 export ALLREDUCE_CAST=${ALLREDUCE_CAST:-1}
-export FAST_CE=${FASE_CE:-1}
+export FAST_CE=${FAST_CE:-1}
 export FUSED_INPUT_QUANTIZE=${FUSED_INPUT_QUANTIZE:-1}
 export FUSED_ADD_NORM_MUL_QUANTIZE=${FUSED_ADD_NORM_MUL_QUANTIZE:-1}
 export FUSED_SILU_W13=${FUSED_SILU_W13:-1}

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh

@@ -9,13 +9,14 @@ export DEVICE_IN_FUNCTION_BUG=1
 export DEBUG=${DEBUG:-0}
 export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1}
 export ALL2ALL=${ALL2ALL:-1}
+export LATE_ALLREDUCE=${LATE_ALLREDUCE:-0}
 export USE_ATOMICS=${USE_ATOMICS:-1}
 export ASM_GEMM=${ASM_GEMM:-1}
 export WQKV=${WQKV:-1}
 export MASTER_WEIGHTS=${MASTER_WEIGHTS:-1}
 export FP8=${FP8:-1}
 export ALLREDUCE_CAST=${ALLREDUCE_CAST:-1}
-export FAST_CE=${FASE_CE:-1}
+export FAST_CE=${FAST_CE:-1}
 export FUSED_INPUT_QUANTIZE=${FUSED_INPUT_QUANTIZE:-1}
 export FUSED_ADD_NORM_MUL_QUANTIZE=${FUSED_ADD_NORM_MUL_QUANTIZE:-1}
 export FUSED_SILU_W13=${FUSED_SILU_W13:-1}

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/profile.sh

@@ -3,4 +3,4 @@ export BENCHMARK=5
 export EVAL_BS=0
 VIZ=${VIZ:--1} FULL_LAYERS=1 DEBUG=0 examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_beam.sh
 SRC="AMD"; [[ $DEV == NULL* ]] && SRC="NULL"
-python -m tinygrad.viz.cli -s "$SRC" --top 20
+python -m tinygrad.viz.cli -s "$SRC" -t

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh

@@ -10,6 +10,7 @@ export DEVICE_IN_FUNCTION_BUG=1
 
 export HK_FLASH_ATTENTION=1
 export ALL2ALL=1
+export LATE_ALLREDUCE=0
 export USE_ATOMICS=1
 export ASM_GEMM=1
 export WQKV=1

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama405b/implementations/tinybox_8xMI350X/dev_beam.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-export PYTHONPATH="."
-export DEV=${DEV:-AMD}
-export EMULATE="AMD_CDNA4"
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000
-export DEVICE_IN_FUNCTION_BUG=1
-
-export DEBUG=${DEBUG:-2}
-export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1}
-export ALL2ALL=${ALL2ALL:-1}
-export USE_ATOMICS=${USE_ATOMICS:-0}
-export ASM_GEMM=${ASM_GEMM:-1}
-export WQKV=${WQKV:-1}
-export OFFLOAD_OPTIM=${OFFLOAD_OPTIM:-1}
-
-export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16"
-export DP=${DP:-1} MP=${MP:-8}
-export BS=${BS:-1} EVAL_BS=${EVAL_BS:-1} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-2}
-
-export MODEL="llama3"
-export BASEDIR="/raid/datasets/c4/"
-export LLAMA3_SIZE=${LLAMA3_SIZE:-"405B"}
-export SEQLEN=${SEQLEN:-8192}
-
-export SEED=${SEED:-5760}
-export DATA_SEED=${DATA_SEED:-5760}
-
-export JITBEAM=${JITBEAM:-3}
-export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1
-
-export FAKEDATA=1 BENCHMARK=10
-if [ -z "$FULL_LAYERS" ]; then
-  export LLAMA_LAYERS=2
-fi
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama405b/implementations/tinybox_8xMI350X/dev_run.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-
-export PYTHONPATH="."
-export DEV=${DEV:-AMD}
-export EMULATE="AMD_CDNA4"
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000
-
-export DEBUG=${DEBUG:-0}
-export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1}
-export ALL2ALL=${ALL2ALL:-1}
-export USE_ATOMICS=${USE_ATOMICS:-0}
-export ASM_GEMM=${ASM_GEMM:-1}
-export WQKV=${WQKV:-1}
-export OFFLOAD_OPTIM=${OFFLOAD_OPTIM:-1}
-
-export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16"
-export DP=${DP:-1} MP=${MP:-8}
-export BS=${BS:-1} EVAL_BS=${EVAL_BS:-1} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-1152}
-
-export MODEL="llama3"
-export BASEDIR="/raid/datasets/c4/"
-export LLAMA3_SIZE=${LLAMA3_SIZE:-"405B"}
-export SEQLEN=${SEQLEN:-8192}
-
-export SEED=${SEED:-$RANDOM}
-export DATA_SEED=${DATA_SEED:-5760}
-
-export JITBEAM=${JITBEAM:-3}
-export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_beam_mp.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-export PYTHONPATH="."
-export DEV=${DEV:-AMD}
-export EMULATE="AMD_CDNA4"
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000
-export DEVICE_IN_FUNCTION_BUG=1
-
-export DEBUG=${DEBUG:-2}
-export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1}
-export ALL2ALL=${ALL2ALL:-1}
-export USE_ATOMICS=${USE_ATOMICS:-0}
-export ASM_GEMM=${ASM_GEMM:-1}
-export WQKV=${WQKV:-1}
-export OFFLOAD_OPTIM=${OFFLOAD_OPTIM:-1}
-
-export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16"
-export DP=${DP:-1} MP=${MP:-8} BS=${BS:-1} EVAL_BS=${EVAL_BS:-1} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-2}
-export GBS=$((BS * GRADIENT_ACC_STEPS))
-
-export MODEL="llama3"
-export BASEDIR="/raid/datasets/c4-8b/"
-export SMALL=1
-export LLAMA3_SIZE=${LLAMA3_SIZE:-"8B"}
-export EVAL_TARGET=3.3 EVAL_FREQ=12288
-export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=4096 MAX_STEPS=1200000
-export WARMUP_STEPS=$((WARMUP_SAMPLES / GBS))
-export SAMPLES=$((MAX_STEPS * GBS))
-export SEQLEN=${SEQLEN:-8192}
-
-export SEED=${SEED:-5760}
-export DATA_SEED=${DATA_SEED:-5760}
-
-export JITBEAM=${JITBEAM:-3}
-export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1
-
-export FAKEDATA=1 BENCHMARK=10
-if [ -z "$FULL_LAYERS" ]; then
-  export LLAMA_LAYERS=2
-fi
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_run_mp.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-export PYTHONPATH="."
-export DEV=${DEV:-AMD}
-export EMULATE="AMD_CDNA4"
-export CHECK_OOB=0
-export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000
-export DEVICE_IN_FUNCTION_BUG=1
-
-export DEBUG=${DEBUG:-0}
-export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1}
-export ALL2ALL=${ALL2ALL:-1}
-export USE_ATOMICS=${USE_ATOMICS:-0}
-export ASM_GEMM=${ASM_GEMM:-1}
-export WQKV=${WQKV:-1}
-export OFFLOAD_OPTIM=${OFFLOAD_OPTIM:-1}
-
-export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16"
-export DP=${DP:-1} MP=${MP:-8} BS=${BS:-1} EVAL_BS=${EVAL_BS:-1} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-32}
-export GBS=$((BS * GRADIENT_ACC_STEPS))
-
-export MODEL="llama3"
-export BASEDIR="/raid/datasets/c4-8b/"
-export SMALL=1
-export LLAMA3_SIZE=${LLAMA3_SIZE:-"8B"}
-export EVAL_TARGET=3.3 EVAL_FREQ=12288
-export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=4096 MAX_STEPS=1200000
-export WARMUP_STEPS=$((WARMUP_SAMPLES / GBS))
-export SAMPLES=$((MAX_STEPS * GBS))
-export SEQLEN=${SEQLEN:-8192}
-
-export SEED=${SEED:-$RANDOM}
-export DATA_SEED=${DATA_SEED:-5760}
-
-export JITBEAM=${JITBEAM:-3}
-export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/README.md

@@ -1,50 +0,0 @@
-# 1. Problem
-
-This problem uses the ResNet-50 CNN to do image classification.
-
-## Requirements
-
-Install tinygrad and mlperf-logging from master.
-```
-git clone https://github.com/tinygrad/tinygrad.git
-python3 -m pip install -e ".[mlperf]"
-```
-
-### tinybox_green
-Install the p2p driver per [README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md)
-This is the default on production tinybox green.
-
-### tinybox_red
-Disable cwsr
-This is the default on production tinybox red.
-```
-sudo vi /etc/modprobe.d/amdgpu.conf
-cat <<EOF > /etc/modprobe.d/amdgpu.conf
-options amdgpu cwsr_enable=0
-EOF
-sudo update-initramfs -u
-sudo reboot
-
-# validate
-sudo cat /sys/module/amdgpu/parameters/cwsr_enable #= 0
-```
-
-# 2. Directions
-
-## Steps to download and verify data
-
-```
-IMGNET_TRAIN=1 python3 extra/datasets/imagenet_download.py
-```
-
-## Steps for one time setup
-
-### tinybox_red
-```
-examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/setup.sh
-```
-
-## Steps to run benchmark
-```
-examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh
-```

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/dev_beam.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=NV
-export MODEL="resnet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=10 BEAM_PADTO=0
-
-export BENCHMARK=10 DEBUG=2
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/dev_run.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=NV
-export MODEL="resnet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=10 BEAM_PADTO=0
-
-export EVAL_START_EPOCH=3 EVAL_FREQ=4
-
-export WANDB=1 PARALLEL=0
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/run_and_time.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-set -e  # Exit on any error
-set -o pipefail  # Make pipeline fail if any command fails
-
-export PYTHONPATH="." DEV=NV
-export MODEL="resnet"
-export SUBMISSION_PLATFORM="tinybox_green"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=10 BEAM_PADTO=0
-
-# pip install -e ".[mlperf]"
-export LOGMLPERF=${LOGMLPERF:-1}
-
-export SEED=$RANDOM
-DATETIME=$(date "+%m%d%H%M")
-LOGFILE="resnet_green_${DATETIME}_${SEED}.log"
-
-# init
-BENCHMARK=10 INITMLPERF=1 python3 examples/mlperf/model_train.py | tee $LOGFILE
-
-# run
-PARALLEL=0 RUNMLPERF=1 EVAL_START_EPOCH=3 EVAL_FREQ=4 python3 examples/mlperf/model_train.py | tee -a $LOGFILE

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/README.md

@@ -1,50 +0,0 @@
-# 1. Problem
-
-This problem uses the ResNet-50 CNN to do image classification.
-
-## Requirements
-
-Install tinygrad and mlperf-logging from master.
-```
-git clone https://github.com/tinygrad/tinygrad.git
-python3 -m pip install -e ".[mlperf]"
-```
-
-### tinybox_green
-Install the p2p driver per [README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md)
-This is the default on production tinybox green.
-
-### tinybox_red
-Disable cwsr
-This is the default on production tinybox red.
-```
-sudo vi /etc/modprobe.d/amdgpu.conf
-cat <<EOF > /etc/modprobe.d/amdgpu.conf
-options amdgpu cwsr_enable=0
-EOF
-sudo update-initramfs -u
-sudo reboot
-
-# validate
-sudo cat /sys/module/amdgpu/parameters/cwsr_enable #= 0
-```
-
-# 2. Directions
-
-## Steps to download and verify data
-
-```
-IMGNET_TRAIN=1 python3 extra/datasets/imagenet_download.py
-```
-
-## Steps for one time setup
-
-### tinybox_red
-```
-examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/setup.sh
-```
-
-## Steps to run benchmark
-```
-examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh
-```

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/dev_beam.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="resnet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=2000 BEAM_UPCAST_MAX=96 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export BENCHMARK=10 DEBUG=${DEBUG:-2}
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/dev_run.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="resnet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=2000 BEAM_UPCAST_MAX=96 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export EVAL_START_EPOCH=3 EVAL_FREQ=4
-
-export WANDB=1 PARALLEL=0
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-set -e  # Exit on any error
-set -o pipefail  # Make pipeline fail if any command fails
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="resnet"
-export SUBMISSION_PLATFORM="tinybox_red"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=2000 BEAM_UPCAST_MAX=96 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-# pip install -e ".[mlperf]"
-export LOGMLPERF=${LOGMLPERF:-1}
-
-export SEED=$RANDOM
-DATETIME=$(date "+%m%d%H%M")
-LOGFILE="resnet_red_${DATETIME}_${SEED}.log"
-
-# init
-sleep 5 && sudo rmmod amdgpu || true
-BENCHMARK=10 INITMLPERF=1 python3 examples/mlperf/model_train.py | tee $LOGFILE
-
-# run
-PARALLEL=0 RUNMLPERF=1 EVAL_START_EPOCH=3 EVAL_FREQ=4 python3 examples/mlperf/model_train.py | tee -a $LOGFILE

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/setup.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-rocm-smi --setprofile compute
-rocm-smi --setmclk 3
-rocm-smi --setperflevel high
-
-# power cap to 350W
-echo "350000000" | sudo tee /sys/class/drm/card{1..6}/device/hwmon/hwmon*/power1_cap

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/README.md

@@ -1,38 +0,0 @@
-# 1. Problem
-
-This problem uses RetinaNet for SSD.
-
-## Requirements
-
-Install tinygrad and mlperf-logging (uncomment mlperf from setup.py) from branch mlperf_training_v5.0.
-```
-git clone https://github.com/tinygrad/tinygrad.git
-python3 -m pip install -e ".[mlperf]"
-```
-
-Also install the following dependencies:
-```
-pip install tqdm numpy pycocotools boto3 pandas torch torchvision
-```
-
-### tinybox_green
-Install the p2p driver per [README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md)
-This is the default on production tinybox green.
-
-# 2. Directions
-
-## Steps to download data
-
-Run the following:
-```
-BASEDIR=/raid/datasets/openimages python3 extra/datasets/openimages.py
-```
-
-## Running
-
-### tinybox_green
-
-#### Steps to run benchmark
-```
-examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/run_and_time.sh
-```

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/dev_beam.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=NV
-export MODEL="retinanet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=96 EVAL_BS=96
-export BASEDIR="/raid/datasets/openimages"
-
-# export RESET_STEP=0
-
-export TRAIN_BEAM=2 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export BENCHMARK=5 DEBUG=2
-
-python examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/dev_run.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=NV
-export MODEL="retinanet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=96 EVAL_BS=96
-export BASEDIR="/raid/datasets/openimages"
-
-# export RESET_STEP=0
-
-export TRAIN_BEAM=2 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export WANDB=1 PARALLEL=0
-export RUNMLPERF=1
-
-python examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/run_and_time.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-set -e  # Exit on any error
-set -o pipefail  # Make pipeline fail if any command fails
-
-export PYTHONPATH="." DEV=NV
-export MODEL="retinanet"
-export SUBMISSION_PLATFORM="tinybox_green"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=96 EVAL_BS=96
-
-export TRAIN_BEAM=2 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-export IGNORE_JIT_FIRST_BEAM=1
-export BASEDIR="/raid/datasets/openimages"
-
-# pip install -e ".[mlperf]"
-export LOGMLPERF=1
-
-export SEED=$RANDOM
-DATETIME=$(date "+%m%d%H%M")
-LOGFILE="retinanet_green_${DATETIME}_${SEED}.log"
-
-# init
-BENCHMARK=10 INITMLPERF=1 python3 examples/mlperf/model_train.py | tee $LOGFILE
-
-# run
-PARALLEL=0 RUNMLPERF=1 python3 examples/mlperf/model_train.py | tee -a $LOGFILE

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/retinanet/implementations/tinybox_red/dev_beam.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="retinanet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=96 EVAL_BS=96
-export BASEDIR="/raid/datasets/openimages"
-
-# export RESET_STEP=0
-
-export TRAIN_BEAM=2 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export BENCHMARK=5 DEBUG=2
-
-python examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/retinanet/implementations/tinybox_red/dev_run.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." DEV=AMD
-export MODEL="retinanet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=96 EVAL_BS=96
-export BASEDIR="/raid/datasets/openimages"
-
-# export RESET_STEP=0
-
-export TRAIN_BEAM=2 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export WANDB=1 PARALLEL=0
-export RUNMLPERF=1
-
-python examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_0.txt

@@ -0,0 +1,106 @@
+:::MLLOG {"namespace": "", "time_ms": 1778207373785, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778207373789, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778207373790, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778207373790, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778207373790, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778207373791, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778207373791, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778207734506, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747904, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "seed", "value": 25580, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747908, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747909, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747909, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747909, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747909, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747909, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747909, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747909, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778207747909, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778208080716, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778208080717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778208901302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778208901303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778208952059, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.705078125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778208952060, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778208952060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778209608282, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778209608282, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778209637796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.552001953125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778209637796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778209637797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778210294879, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778210294879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778210324584, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.1011962890625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778210324584, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778210324585, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778210980564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778210980565, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778211010225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.8807373046875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778211010225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778211010226, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778211667184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778211667185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778211696784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.7498779296875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778211696785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778211696786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778212356059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778212356060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778212385775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.65478515625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778212385776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778212385776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778213044774, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778213044775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778213074311, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5731201171875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778213074312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778213074313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778213732225, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778213732225, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778213761806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5137939453125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778213761806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778213761807, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778214419768, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778214419769, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778214449443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.46630859375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778214449444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778214449445, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778215112018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778215112019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778215141586, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.428955078125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778215141586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778215141587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778215794970, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778215794970, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778215824346, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.390869140625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778215824346, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778215824347, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778216475810, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778216475810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778216505269, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.361328125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778216505269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778216505270, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778217157389, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778217157390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778217186831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.346923828125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778217186832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778217186832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778217846265, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778217846266, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778217876013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3133544921875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778217876014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778217876014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778218532377, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778218532378, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778218561863, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.2989501953125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778218561863, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778218561864, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778218561864, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_1.txt

@@ -0,0 +1,111 @@
+:::MLLOG {"namespace": "", "time_ms": 1778218577779, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778218577783, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778218577784, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778218577784, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778218577784, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778218578371, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778218578371, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778218957180, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971058, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "seed", "value": 356, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971063, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971064, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971064, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971064, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971064, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971064, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778218971064, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778219289653, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778219289654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778220097041, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778220097042, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778220141757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.743896484375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778220141758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778220141758, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778220795772, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778220795773, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778220825439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.58349609375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778220825440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778220825440, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778221480609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778221480610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778221510284, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.1131591796875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778221510285, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778221510286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778222164664, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778222164665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778222194290, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.8935546875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778222194291, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778222194291, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778222848846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778222848847, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778222878557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.7567138671875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778222878558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778222878558, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778223532447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778223532447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778223562036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.658203125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778223562037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778223562037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778224215343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778224215344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778224244924, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5860595703125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778224244925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778224244925, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778224898378, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778224898379, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778224928021, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.51708984375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778224928021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778224928022, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778225581424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778225581425, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778225611002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.471923828125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778225611003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778225611003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778226265043, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778226265044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778226294659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.43701171875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778226294660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778226294661, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778226949577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778226949577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778226979238, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5406494140625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778226979239, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778226979239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778227635352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778227635352, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778227664978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3836669921875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778227664978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778227664979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778228323150, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778228323151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778228352865, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.355712890625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778228352865, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778228352866, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778229010307, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778229010307, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778229040142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3319091796875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778229040143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778229040143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778229696378, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778229696379, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778229726195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.30615234375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778229726195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778229726196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778230383239, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778230383240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778230412831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.29052734375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778230412832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778230412832, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778230412833, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_2.txt

@@ -0,0 +1,111 @@
+:::MLLOG {"namespace": "", "time_ms": 1778230427283, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778230427287, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778230427287, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778230427287, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778230427287, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778230427939, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778230427939, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778230779581, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792886, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792890, "event_type": "POINT_IN_TIME", "key": "seed", "value": 2774, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792891, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792892, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792892, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792892, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792892, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792892, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778230792892, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778231115792, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778231115793, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778232030906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778232030907, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778232075494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.812255859375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778232075494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778232075495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778232729579, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778232729580, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778232759140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.582275390625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778232759141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778232759142, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778233413630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778233413631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778233443219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.11767578125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778233443220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778233443220, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778234097427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778234097428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778234127034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.9005126953125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778234127034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778234127035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778234780955, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778234780956, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778234810558, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.7586669921875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778234810558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778234810559, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778235463904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778235463905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778235493473, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.657958984375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778235493474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778235493475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778236147005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778236147005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778236176551, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.585693359375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778236176552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778236176552, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778236830530, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778236830530, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778236860107, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.521484375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778236860108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778236860108, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778237514002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778237514003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778237543592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4742431640625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778237543592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778237543593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778238197935, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778238197936, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778238227501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.428955078125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778238227502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778238227503, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778238882036, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778238882037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778238911645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4019775390625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778238911645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778238911646, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778239565129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778239565130, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778239594721, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.37890625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778239594722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778239594722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778240248763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778240248764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778240278335, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3448486328125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778240278336, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778240278337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778240933651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778240933651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778240963429, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.325439453125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778240963430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778240963431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778241626264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778241626265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778241656303, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3072509765625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778241656304, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778241656304, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778242315322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778242315323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778242345178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.2781982421875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778242345178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778242345179, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778242345179, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_3.txt

@@ -0,0 +1,106 @@
+:::MLLOG {"namespace": "", "time_ms": 1778242359541, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778242359545, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778242359545, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778242359545, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778242359545, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778242360117, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778242360118, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778242702158, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715949, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715953, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1261, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715953, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715953, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715954, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715955, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715955, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715955, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778242715955, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778243033805, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778243033806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778243851371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778243851372, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778243896651, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.7802734375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778243896652, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778243896652, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778244555628, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778244555629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778244585531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.574951171875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778244585532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778244585533, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778245246511, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778245246512, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778245276502, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.1171875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778245276503, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778245276503, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778245937187, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778245937187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778245967058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.8995361328125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778245967059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778245967059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778246626117, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778246626117, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778246656019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.762451171875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778246656019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778246656020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778247315255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778247315256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778247345128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.6572265625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778247345128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778247345129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778248003582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778248003582, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778248033442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.58740234375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778248033443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778248033443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778248692764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778248692764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778248722726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5286865234375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778248722727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778248722727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778249383186, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778249383186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778249413099, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.475830078125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778249413099, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778249413100, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778250072852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778250072852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778250102740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4278564453125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778250102741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778250102741, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778250762230, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778250762230, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778250792198, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.400146484375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778250792199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778250792199, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778251455492, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778251455492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778251485544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3818359375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778251485545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778251485545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778252146772, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778252146772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778252176776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.345458984375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778252176776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778252176777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778252836585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778252836586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778252866442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.322265625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778252866443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778252866443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778253526422, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778253526422, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778253556343, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.299072265625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778253556343, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778253556344, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778253556344, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_4.txt

@@ -0,0 +1,111 @@
+:::MLLOG {"namespace": "", "time_ms": 1778253570454, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778253570459, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778253570459, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778253570459, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778253570459, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778253571045, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778253571045, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778253944036, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957691, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957695, "event_type": "POINT_IN_TIME", "key": "seed", "value": 14711, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957695, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957695, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957695, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957695, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957695, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957695, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957695, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957695, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778253957696, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778254276545, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778254276546, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778255100535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778255100536, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778255143977, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.77978515625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778255143977, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778255143978, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778255806844, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778255806845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778255836518, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.578857421875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778255836519, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778255836520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778256495933, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778256495933, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778256525443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.1239013671875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778256525443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778256525444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778257180826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778257180827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778257210282, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.906494140625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778257210283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778257210283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778257866434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778257866435, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778257895945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.75244140625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778257895945, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778257895946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778258550818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778258550819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778258580369, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.6553955078125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778258580369, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778258580370, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778259234200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778259234201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778259263770, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5762939453125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778259263771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778259263772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778259917494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778259917495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778259947011, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.52197265625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778259947012, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778259947013, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778260600453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778260600454, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778260629950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4765625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778260629951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778260629951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778261285126, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778261285127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778261314809, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4378662109375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778261314810, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778261314810, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778261971632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778261971632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778262001260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3968505859375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778262001261, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778262001261, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778262657393, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778262657394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778262686962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.365966796875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778262686962, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778262686963, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778263342665, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778263342666, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778263372176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3365478515625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778263372176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778263372177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778264027427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778264027428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778264056993, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3363037109375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778264056993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778264056994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778264710992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778264710993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778264740486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3016357421875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778264740486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778264740487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778265396989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778265396989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778265426521, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.2861328125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778265426522, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778265426522, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778265426522, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_5.txt

@@ -0,0 +1,111 @@
+:::MLLOG {"namespace": "", "time_ms": 1778265440911, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778265440915, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778265440915, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778265440916, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778265440916, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778265441493, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778265441493, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778265779467, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792765, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792769, "event_type": "POINT_IN_TIME", "key": "seed", "value": 27754, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792769, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792769, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792769, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792769, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792769, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792769, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792769, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778265792770, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778266108942, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778266108943, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778266913943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778266913944, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778266957471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.74072265625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778266957472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778266957472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778267616663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778267616663, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778267648052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.612060546875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778267648053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778267648053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778268306168, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778268306168, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778268335863, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.16552734375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778268335864, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778268335864, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778268998030, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778268998030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778269027991, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.915283203125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778269027992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778269027992, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778269689514, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778269689515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778269719312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.7637939453125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778269719313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778269719313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778270378319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778270378320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778270408037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.6695556640625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778270408038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778270408038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778271066429, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778271066430, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778271096134, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.583251953125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778271096135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778271096135, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778271754376, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778271754377, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778271784142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.525146484375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778271784142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778271784143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778272442458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778272442459, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778272472257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4774169921875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778272472257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778272472258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778273129575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778273129576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778273159231, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.443359375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778273159231, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778273159232, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778273816098, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778273816099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778273845769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4072265625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778273845770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778273845770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778274505683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778274505684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778274535540, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3677978515625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778274535541, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778274535541, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778275195662, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778275195662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778275225396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4146728515625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778275225397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778275225397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778275884245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778275884246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778275913924, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3697509765625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778275913925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778275913925, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778276570930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778276570931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778276600619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.321533203125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778276600620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778276600620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778277262406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778277262407, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778277292466, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.287353515625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778277292467, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778277292467, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778277292468, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_6.txt

@@ -0,0 +1,111 @@
+:::MLLOG {"namespace": "", "time_ms": 1778277306868, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778277306872, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778277306872, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778277306873, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778277306873, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778277307428, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778277307429, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778277671564, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685153, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685157, "event_type": "POINT_IN_TIME", "key": "seed", "value": 17816, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685157, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685157, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685158, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685159, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685159, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778277685159, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778278007248, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778278007260, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778278810368, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778278810369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778278855284, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.768798828125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778278855285, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778278855285, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778279519460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778279519461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778279549391, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.568603515625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778279549392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778279549392, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778280214562, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778280214563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778280244495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.151123046875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778280244496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778280244496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778280909906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778280909906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778280939913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.9197998046875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778280939913, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778280939914, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778281607749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778281607750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778281637814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.7734375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778281637815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778281637815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778282306223, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778282306224, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778282336322, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.673583984375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778282336323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778282336323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778283007699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778283007700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778283037808, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.6011962890625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778283037808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778283037809, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778283706598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778283706598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778283736748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.526123046875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778283736748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778283736749, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778284408590, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778284408590, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778284438316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.475341796875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778284438317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778284438317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778285098897, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778285098898, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778285128703, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.432861328125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778285128703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778285128704, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778285786660, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778285786660, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778285816222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4031982421875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778285816222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778285816223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778286473781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778286473782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778286503417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3638916015625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778286503418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778286503418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778287160556, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778287160556, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778287190213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.341796875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778287190214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778287190215, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778287846424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778287846424, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778287876044, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.32177734375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778287876045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778287876046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778288531947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778288531947, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778288561549, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5465087890625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778288561550, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778288561550, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778289220442, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778289220442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778289250127, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.2855224609375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778289250128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778289250128, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778289250129, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_7.txt

@@ -0,0 +1,106 @@
+:::MLLOG {"namespace": "", "time_ms": 1778289264340, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778289264344, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778289264344, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778289264344, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778289264344, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778289264911, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778289264912, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778289599730, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613197, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613200, "event_type": "POINT_IN_TIME", "key": "seed", "value": 16781, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613201, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613202, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613202, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613202, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613202, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613202, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778289613202, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778289929875, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778289929878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778290756967, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778290756968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778290801735, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.758544921875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778290801736, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778290801736, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778291460896, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778291460896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778291490685, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.683349609375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778291490685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778291490686, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778292152773, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778292152774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778292182518, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.1280517578125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778292182519, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778292182519, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778292842100, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778292842101, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778292871768, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.90185546875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778292871769, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778292871769, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778293529314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778293529315, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778293559042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.757080078125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778293559043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778293559043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778294218188, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778294218189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778294247880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.6575927734375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778294247880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778294247881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778294908017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778294908018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778294937688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.586181640625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778294937689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778294937690, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778295595710, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778295595710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778295625392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5230712890625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778295625393, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778295625394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778296283795, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778296283795, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778296313518, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.467529296875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778296313519, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778296313519, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778296973892, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778296973893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778297003579, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4351806640625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778297003580, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778297003580, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778297661577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778297661578, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778297691130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.406982421875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778297691130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778297691131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778298348217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778298348218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778298377837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3848876953125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778298377837, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778298377838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778299035939, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778299035940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778299065575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3480224609375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778299065576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778299065576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778299724382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778299724383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778299754023, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3209228515625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778299754023, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778299754024, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778300412415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778300412415, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778300442058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.2950439453125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778300442059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778300442060, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778300442060, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_8.txt

@@ -0,0 +1,111 @@
+:::MLLOG {"namespace": "", "time_ms": 1778300456451, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778300456455, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778300456455, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778300456455, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778300456455, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778300457011, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778300457012, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778300803665, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817390, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "seed", "value": 4729, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817395, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817396, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817396, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817396, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817396, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817396, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817396, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817396, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817396, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778300817396, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778301145773, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778301145774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778301985088, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778301985089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778302030319, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.865966796875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778302030319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778302030320, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778302687526, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778302687527, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778302717259, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.615966796875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778302717260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778302717261, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778303376036, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778303376037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778303406044, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.154296875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778303406045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778303406045, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778304071224, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778304071225, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778304101168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.9095458984375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778304101169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778304101170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778304762172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778304762173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778304792161, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.775634765625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778304792162, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778304792162, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778305452836, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778305452836, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778305482708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.676513671875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778305482708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778305482709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778306140246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778306140246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778306169947, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5947265625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778306169947, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778306169948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778306828284, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778306828285, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778306858077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5255126953125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778306858077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778306858078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778307519609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778307519610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778307549531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4757080078125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778307549532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778307549532, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778308208151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778308208152, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778308237856, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4312744140625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778308237857, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778308237857, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778308896397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778308896398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778308926271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.402099609375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778308926271, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778308926272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778309586346, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778309586347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778309616134, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.37060546875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778309616134, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778309616135, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778310273337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778310273338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778310303090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3968505859375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778310303091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778310303092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778310958883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778310958883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778310988541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3284912109375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778310988542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778310988542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778311645004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778311645004, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778311674742, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.302001953125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778311674743, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778311674744, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778312331845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778312331846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778312361570, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.2777099609375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778312361571, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778312361571, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 196608}}
+:::MLLOG {"namespace": "", "time_ms": 1778312361572, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/results/tinybox_8xMI350X/llama31_8b/result_9.txt

@@ -0,0 +1,106 @@
+:::MLLOG {"namespace": "", "time_ms": 1778312377935, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1328}}
+:::MLLOG {"namespace": "", "time_ms": 1778312377940, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1329}}
+:::MLLOG {"namespace": "", "time_ms": 1778312377940, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1330}}
+:::MLLOG {"namespace": "", "time_ms": 1778312377940, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1331}}
+:::MLLOG {"namespace": "", "time_ms": 1778312377940, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "llama31_8b", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1333}}
+:::MLLOG {"namespace": "", "time_ms": 1778312378485, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1336}}
+:::MLLOG {"namespace": "", "time_ms": 1778312378485, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1337}}
+:::MLLOG {"namespace": "", "time_ms": 1778312726494, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1629}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740045, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1340}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740049, "event_type": "POINT_IN_TIME", "key": "seed", "value": 12228, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1341}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740049, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1343}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740049, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 8192, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1344}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "max_steps", "value": 1200000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1345}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 2, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1346}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 1024, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1347}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 38400000, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1348}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adamw", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1350}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1351}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "opt_end_learning_rate", "value": 0.0001, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1352}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_1", "value": 0.9, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1353}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "opt_adamw_beta_2", "value": 0.95, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1354}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "opt_adamw_epsilon", "value": 1e-05, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1355}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "opt_adamw_weight_decay", "value": 0.1, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1356}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1357}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740050, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 128, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1358}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740051, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_steps", "value": 1199872, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1359}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740051, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_decay_schedule", "value": "cosine with linear warmup", "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1360}}
+:::MLLOG {"namespace": "", "time_ms": 1778312740051, "event_type": "POINT_IN_TIME", "key": "opt_gradient_clip_norm", "value": 1.0, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1361}}
+:::MLLOG {"namespace": "", "time_ms": 1778313057094, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1529, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778313057095, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1530, "samples_count": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1778313872567, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778313872567, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778313917470, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 5.736083984375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778313917471, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778313917472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 12288}}
+:::MLLOG {"namespace": "", "time_ms": 1778314572849, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778314572850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778314602523, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.584716796875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778314602524, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778314602525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 24576}}
+:::MLLOG {"namespace": "", "time_ms": 1778315258897, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778315258898, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778315288494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 4.114501953125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778315288495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778315288496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 36864}}
+:::MLLOG {"namespace": "", "time_ms": 1778315946776, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778315946777, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778315976384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.906005859375, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778315976385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778315976386, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 49152}}
+:::MLLOG {"namespace": "", "time_ms": 1778316632177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778316632178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778316661800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.76513671875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778316661800, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778316661801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 61440}}
+:::MLLOG {"namespace": "", "time_ms": 1778317318705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778317318706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778317348421, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.6568603515625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778317348421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778317348422, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 73728}}
+:::MLLOG {"namespace": "", "time_ms": 1778318007246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778318007246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778318036837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.5897216796875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778318036838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778318036839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 86016}}
+:::MLLOG {"namespace": "", "time_ms": 1778318691769, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778318691770, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778318721376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.52587890625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778318721377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778318721377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 98304}}
+:::MLLOG {"namespace": "", "time_ms": 1778319374807, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778319374808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778319404256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.473388671875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778319404257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778319404258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 110592}}
+:::MLLOG {"namespace": "", "time_ms": 1778320058613, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778320058613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778320087986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.4307861328125, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778320087987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778320087988, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 122880}}
+:::MLLOG {"namespace": "", "time_ms": 1778320742022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778320742022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778320771659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3931884765625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778320771660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778320771660, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 135168}}
+:::MLLOG {"namespace": "", "time_ms": 1778321426019, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778321426019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778321455724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3629150390625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778321455725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778321455726, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 147456}}
+:::MLLOG {"namespace": "", "time_ms": 1778322114634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778322114634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778322144126, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3377685546875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778322144127, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778322144127, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 159744}}
+:::MLLOG {"namespace": "", "time_ms": 1778322801727, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778322801728, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778322831371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.3150634765625, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778322831372, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778322831372, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1654, "samples_count": 172032}}
+:::MLLOG {"namespace": "", "time_ms": 1778323487126, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1616, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778323487126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1617, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778323516691, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 3.2889404296875, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1637, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778323516691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1638, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778323516692, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1646, "samples_count": 184320}}
+:::MLLOG {"namespace": "", "time_ms": 1778323516692, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad3/examples/mlperf/model_train.py", "lineno": 1647, "status": "success"}}

examples/mlperf/training_submission_v6.0/tinycorp/systems/tinybox_8xMI300X.json

@@ -1,38 +0,0 @@
-{
-    "submitter": "tinycorp",
-    "division": "closed",
-    "status": "Available on-premise",
-    "system_name": "tinybox 8xMI300X",
-    "number_of_nodes": "1",
-    "host_processors_per_node": "2",
-    "host_processor_model_name": "AMD EPYC 9354",
-    "host_processor_core_count": "32",
-    "host_processor_vcpu_count": "64",
-    "host_processor_frequency": "",
-    "host_processor_caches": "",
-    "host_processor_interconnect": "",
-    "host_memory_capacity": "2304GB",
-    "host_storage_type": "NVMe SSD",
-    "host_storage_capacity": "3x 4TB raid array",
-    "host_networking": "",
-    "host_networking_topology": "",
-    "host_memory_configuration": "24x 96GB DDR5",
-    "accelerators_per_node": "8",
-    "accelerator_model_name": "AMD Instinct MI300X 192GB HBM3",
-    "accelerator_host_interconnect": "PCIe 5.0 x16",
-    "accelerator_frequency": "",
-    "accelerator_on-chip_memories": "",
-    "accelerator_memory_configuration": "HBM3",
-    "accelerator_memory_capacity": "192GB",
-    "accelerator_interconnect": "",
-    "accelerator_interconnect_topology": "",
-    "cooling": "air",
-    "hw_notes": "",
-    "framework": "tinygrad, branch mlperf_training_v5.0",
-    "other_software_stack": {
-        "python": "3.10.16",
-        "ROCm": "3.0.0+94441cb"
-    },
-    "operating_system": "Ubuntu 24.04.1 LTS",
-    "sw_notes": ""
-  }

examples/mlperf/training_submission_v6.0/tinycorp/systems/tinybox_8xMI350X.json

@@ -34,5 +34,5 @@
         "ROCm": "7.1.1"
     },
     "operating_system": "Ubuntu 24.04.3 LTS",
-    "sw_notes": ""
+    "sw_notes": "tinygrad @ 026688f03f84a75ec3fef034bcba916bf8f8bdc6"
   }

examples/mlperf/training_submission_v6.0/tinycorp/systems/tinybox_green.json

@@ -1,38 +0,0 @@
-{
-  "submitter": "tinycorp",
-  "division": "closed",
-  "status": "Available on-premise",
-  "system_name": "tinybox green",
-  "number_of_nodes": "1",
-  "host_processors_per_node": "1",
-  "host_processor_model_name": "AMD EPYC 7532",
-  "host_processor_core_count": "32",
-  "host_processor_vcpu_count": "64",
-  "host_processor_frequency": "",
-  "host_processor_caches": "",
-  "host_processor_interconnect": "",
-  "host_memory_capacity": "128GB",
-  "host_storage_type": "NVMe SSD",
-  "host_storage_capacity": "4 TB raid array + 1 TB boot",
-  "host_networking": "",
-  "host_networking_topology": "",
-  "host_memory_configuration": "8x 16GB DDR4",
-  "accelerators_per_node": "6",
-  "accelerator_model_name": "NVIDIA GeForce RTX 4090",
-  "accelerator_host_interconnect": "PCIe 4.0 x16",
-  "accelerator_frequency": "",
-  "accelerator_on-chip_memories": "",
-  "accelerator_memory_configuration": "GDDR6X",
-  "accelerator_memory_capacity": "24GB",
-  "accelerator_interconnect": "",
-  "accelerator_interconnect_topology": "",
-  "cooling": "air",
-  "hw_notes": "",
-  "framework": "tinygrad, branch mlperf_training_v5.0",
-  "other_software_stack": {
-    "python": "3.10.12",
-    "CUDA": "12.4"
-  },
-  "operating_system": "Ubuntu 22.04.4",
-  "sw_notes": ""
-}

examples/mlperf/training_submission_v6.0/tinycorp/systems/tinybox_red.json

@@ -1,37 +0,0 @@
-{
-  "submitter": "tinycorp",
-  "division": "closed",
-  "status": "Available on-premise",
-  "system_name": "tinybox red",
-  "number_of_nodes": "1",
-  "host_processors_per_node": "1",
-  "host_processor_model_name": "AMD EPYC 7532",
-  "host_processor_core_count": "32",
-  "host_processor_vcpu_count": "64",
-  "host_processor_frequency": "",
-  "host_processor_caches": "",
-  "host_processor_interconnect": "",
-  "host_memory_capacity": "128GB",
-  "host_storage_type": "NVMe SSD",
-  "host_storage_capacity": "4 TB raid array + 1 TB boot",
-  "host_networking": "",
-  "host_networking_topology": "",
-  "host_memory_configuration": "8x 16GB DDR4",
-  "accelerators_per_node": "6",
-  "accelerator_model_name": "AMD Radeon RX 7900 XTX",
-  "accelerator_host_interconnect": "PCIe 4.0 x16",
-  "accelerator_frequency": "",
-  "accelerator_on-chip_memories": "",
-  "accelerator_memory_configuration": "GDDR6",
-  "accelerator_memory_capacity": "24GB",
-  "accelerator_interconnect": "",
-  "accelerator_interconnect_topology": "",
-  "cooling": "air",
-  "hw_notes": "",
-  "framework": "tinygrad, branch mlperf_training_v5.0",
-  "other_software_stack": {
-    "python": "3.10.12"
-  },
-  "operating_system": "Ubuntu 22.04.4",
-  "sw_notes": ""
-}

extra/amdpci/am_smi.py

@@ -64,7 +64,7 @@ def get_bar0_size(pcibus):
 
 class AMSMI(AMDev):
   def __init__(self, pcibus, vram_bar:MMIOInterface, doorbell_bar:MMIOInterface, mmio_bar:MMIOInterface):
-    self.pcibus = pcibus
+    self.pcibus, self.devfmt = pcibus, pcibus
     self.vram, self.doorbell64, self.mmio = vram_bar, doorbell_bar, mmio_bar
     self.pci_state = self.read_pci_state()
     if self.pci_state == "D0": self._init_from_d0()
@@ -91,6 +91,7 @@ class SMICtx:
     self.prev_lines_cnt = 0
     self.prev_terminal_width = 0
     self.prev_terminal_height = 0
+    self.prev_metrics = {}
 
     remove_parts = ["Advanced Micro Devices, Inc. [AMD/ATI]", "VGA compatible controller:", "Processing accelerators:"]
     lspci = subprocess.check_output(["lspci"]).decode("utf-8").splitlines()
@@ -235,6 +236,29 @@ class SMICtx:
       case (13,0,12): return self._smuq10_round(metrics.SocketPower), self._smuq10_round(metrics.SocketPowerLimit)
       case _: return metrics.SmuMetrics.AverageSocketPower, metrics.SmuMetrics.dGPU_W_MAX
 
+  def get_throttle_info(self, dev, metrics):
+    match dev.ip_ver[am.MP1_HWIP]:
+      case (13,0,6)|(13,0,12):
+        throttle_fields = [('ProchotResidencyAcc', 'Prochot'), ('PptResidencyAcc', 'PPT'),
+                           ('SocketThmResidencyAcc', 'Socket Thm'), ('VrThmResidencyAcc', 'VR Thm'), ('HbmThmResidencyAcc', 'HBM Thm')]
+        prev = self.prev_metrics.get(dev.pcibus)
+        active = []
+        if prev is not None:
+          acc_delta = metrics.AccumulationCounter - prev.AccumulationCounter
+          if acc_delta > 0:
+            for field, name in throttle_fields:
+              delta = getattr(metrics, field) - getattr(prev, field)
+              if delta > 0 and (pct := min(100, (delta * 100 + acc_delta // 2) // acc_delta)) > 0: active.append((name, pct))
+        return active
+      case _:
+        smu_mod = dev.smu.smu_mod
+        throttler_names = {getattr(smu_mod, a): a[len('THROTTLER_'):-len('_BIT')]
+                           for a in dir(smu_mod) if a.startswith('THROTTLER_') and a.endswith('_BIT')}
+        active = []
+        for i, pct in enumerate(metrics.SmuMetrics.ThrottlingPercentage):
+          if pct > 0: active.append((throttler_names.get(i, f"UNK_{i}"), int(pct)))
+        return active
+
   def get_mem_usage(self, dev):
     usage = 0
     pt_stack = [dev.mm.root_page_table]
@@ -281,6 +305,13 @@ class SMICtx:
                     + [f"MEM Activity {draw_bar(self.get_mem_activity(dev, metrics) / 100, activity_line_width)}"] \
                     + [f"MEM Usage    {draw_bar(mem_used / mem_total, activity_line_width, opt_text=mem_fmt)}"] \
 
+      throttle_info = self.get_throttle_info(dev, metrics)
+      if throttle_info:
+        throttle_text = colored(', '.join(f"{name} {pct}%" for name, pct in throttle_info), "red")
+      else:
+        throttle_text = colored("None", "green")
+      activity_line += [f"Throttle     {throttle_text}" + " " * (activity_line_width + 2)]
+
       temps_data, temps_data_compact = self.get_temps(dev, metrics), self.get_temps(dev, metrics, compact=True)
       temps_table = ["=== Temps (°C) ==="] + [f"{name:<16}: {color_temp(val)}" for name, val in temps_data.items()]
       temps_table_compact = ["Temps (°C):" + '/'.join([f"{color_temp(val)} {name}" for name, val in temps_data_compact.items()])]
@@ -324,6 +355,8 @@ class SMICtx:
 
       dev_content.append(device_line + activity_line + same_line([temps_table, power_table, frequency_table]))
 
+    self.prev_metrics = {dev.pcibus: m for dev, m in dev_metrics.items() if m is not None}
+
     raw_text = 'AM Monitor'.center(terminal_width) + "\n" + "=" * terminal_width + "\n\n"
     for i in range(0, len(dev_content), 2):
       if i + 1 < len(dev_content): raw_text += '\n'.join(same_line([dev_content[i], dev_content[i+1]], split=padding))

extra/gemm/cdna_asm_gemm.py

@@ -2628,21 +2628,24 @@ def custom_asm_gemm(C:UOp, A:UOp, B:UOp, dname:str) -> UOp:
 # ** FP8 GEMM custom kernel
 
 @functools.cache
-def custom_hk_fp8_gemm(C:UOp, A:UOp, B:UOp, X_s:UOp, W_s:UOp, *extra:UOp, dname:str) -> UOp:
-  # A is (batch, M, K), B is (N, K) transposed, X_s is x_scale, W_s is w_scale — kernel multiplies by both.
-  # extra is unused fwd inputs (e.g. grad_amax_state) plumbed through so the bwd can read them via kernel.src.
+def custom_hk_fp8_gemm(C:UOp, A:UOp, B:UOp, *args:UOp, dname:str, scale_mode:int=3) -> UOp:
+  # scale_mode: 0=no scale, 1=x only, 2=w only, 3=both
+  n_scales = (1 if scale_mode & 1 else 0) + (1 if scale_mode & 2 else 0)
+  scales, extra = args[:n_scales], args[n_scales:]
   M, K = A.shape[0]*A.shape[1], A.shape[2]
   N, K2 = B.shape[(1 if B.ndim == 3 else 0):]
   assert K == K2, f"{A.shape} {B.shape}"
   block_size = 256
   threads = UOp.special(64 * 8, "lidx0")
   workgroups = UOp.special((M // block_size) * (N // block_size), "gidx0")
-  sink = UOp.sink(C.base, A.base, B.base, X_s.base, W_s.base, threads, workgroups,
+  sink_inputs = (C.base, A.base, B.base) + tuple(s.base for s in scales) + (threads, workgroups)
+  sink = UOp.sink(*sink_inputs,
                   arg=KernelInfo(f"hk_fp8_gemm_{M}_{N}_{K}", estimates=Estimates(ops=2*M*N*K, mem=(M*K+N*K)*A.dtype.itemsize+M*N*C.dtype.itemsize)))
   kittens_path = pathlib.Path(__file__).parent.parent/"thunder"/"amd"
   src = (kittens_path/"gemm_fp8.cpp").read_text()
   lib = HIPCCCompiler("gfx950", [f"-I{(kittens_path/'include').as_posix()}", "-std=c++20", "-DKITTENS_CDNA4", "-ffast-math",
-                                 "-DHIP_ENABLE_WARP_SYNC_BUILTINS", f"-DGEMM_M={M}", f"-DGEMM_N={N}", f"-DGEMM_K={K}"]).compile_cached(src)
+                                 "-DHIP_ENABLE_WARP_SYNC_BUILTINS", f"-DGEMM_M={M}", f"-DGEMM_N={N}", f"-DGEMM_K={K}",
+                                 f"-DSCALE_MODE={scale_mode}"]).compile_cached(src)
   return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=dname), UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=src),
                                UOp(Ops.BINARY, arg=lib)))
 
@@ -2699,8 +2702,7 @@ def custom_uop_gemm(C:UOp, A:UOp, B:UOp) -> UOp:
 
 def custom_gemm_bw(gradient:UOp, kernel:UOp):
   inputs = kernel.src[1:]
-  # fp8 scaled gemm has 5 inputs (out, a, b, x_scale, w_scale) optionally plus grad_amax_state (6 total); plain gemm has 3
-  if len(inputs) >= 5:
+  if inputs[1].dtype == FP8_DTYPE:
     grad_amax_state = inputs[5] if len(inputs) == 6 else None
     out, a, b, s_x, s_w = inputs[:5]
     a_t, b_t, g_t = Tensor(a, device=a.device), Tensor(b, device=a.device), Tensor(gradient, device=a.device)
@@ -2720,8 +2722,13 @@ def custom_gemm_bw(gradient:UOp, kernel:UOp):
     # dgrad: uses g_scale * x_scale * w_scale
     grad_a = asm_gemm(g_fp8, b_t, x_scale=g_scale * s_x_t, w_scale=s_w_t)
     # wgrad: no w_scale
-    _one = Tensor(1.0, dtype=dtypes.float, device=a.device)
-    grad_b = asm_gemm(g_fp8.permute(2, 0, 1).reshape(g_t.shape[-1], -1), a_t.reshape(-1, a_t.shape[-1]), x_scale=g_scale * s_x_t, w_scale=_one)
+    g_fp8_2d = g_fp8.reshape(-1, g_fp8.shape[-1])
+    if getenv("FAST_FP8_TRANSPOSE", 0) and g_fp8_2d.shape[0] % 64 == 0 and g_fp8_2d.shape[1] % 64 == 0:
+      from extra.llama_kernels.fp8_transpose import fast_fp8_transpose
+      g_fp8_T = fast_fp8_transpose(g_fp8_2d)
+    else:
+      g_fp8_T = g_fp8.permute(2, 0, 1).reshape(g_t.shape[-1], -1)
+    grad_b = asm_gemm(g_fp8_T, a_t.reshape(-1, a_t.shape[-1]), x_scale=g_scale * s_x_t)
     # Attach the delayed-amax store effect (if any) to grad_a so realizing grads commits the amax update.
     ret = (None, grad_a.uop.after(store_effect), grad_b.uop, None, None)
     if len(inputs) == 6: ret = ret + (None,)
@@ -2774,11 +2781,11 @@ def asm_gemm(a:Tensor, b:Tensor, x_scale:Tensor|None=None, w_scale:Tensor|None=N
   if arch.startswith("gfx950") and getenv("USE_ASM", 1):
     # fp8 gemm computes [email protected], kernel multiplies output by x_scale * w_scale before bf16 store
     if a.dtype == FP8_DTYPE:
-      _one = lambda: Tensor(1.0, dtype=dtypes.float, device=a.device)
-      xs = x_scale if x_scale is not None else _one()
-      ws = w_scale if w_scale is not None else _one()
+      scales = tuple(s for s in (x_scale, w_scale) if s is not None)
+      scale_mode = (1 if x_scale is not None else 0) | (2 if w_scale is not None else 0)
       extra = [grad_amax_state] if grad_amax_state is not None else []
-      out = Tensor.custom_kernel(out, a, b.T, xs, ws, *extra, fxn=functools.partial(custom_hk_fp8_gemm, dname=dname), grad_fxn=custom_gemm_bw)[0]
+      fxn = functools.partial(custom_hk_fp8_gemm, dname=dname, scale_mode=scale_mode)
+      out = Tensor.custom_kernel(out, a, b.T, *scales, *extra, fxn=fxn, grad_fxn=custom_gemm_bw)[0]
     else:
       out = Tensor.custom_kernel(out, a, b, fxn=functools.partial(custom_asm_gemm, dname=dname), grad_fxn=custom_gemm_bw)[0]
   else:

extra/llama_kernels/__init__.py

@@ -34,13 +34,12 @@ def dname_of(device) -> str:
   return device.split(":")[0] if isinstance(device, str) else device
 
 def alloc_like(shape, dtype, device, axis=None) -> Tensor:
-  if isinstance(device, tuple):
-    if axis is None: return Tensor(Tensor.invalids(*shape, dtype=dtype, device=device).uop.multi(0), device=device)
+  if isinstance(device, tuple) and axis is not None:
     return Tensor(Tensor.invalids(*shard_shape(shape, axis, len(device)), dtype=dtype, device=device).uop.multi(axis), device=device)
   return Tensor.invalids(*shape, dtype=dtype, device=device)
 
-def alloc_local(shape, dtype, device) -> Tensor:
-  if isinstance(device, tuple):
+def alloc_local(shape, dtype, device, axis=None) -> Tensor:
+  if isinstance(device, tuple) and axis is not None:
     return Tensor(Tensor.invalids(*shape, dtype=dtype, device=device).uop.multi(0), device=device)
   return Tensor.invalids(*shape, dtype=dtype, device=device)
 

extra/llama_kernels/cast_amax/__init__.py

@@ -41,10 +41,9 @@ def _fused_quantize_bwd_w13(gradient:UOp, kernel:UOp):
   _, _, xw13, amax_state, grad_amax_state = kernel.src[1:]
   device = xw13.device
   axis = xw13.axis if isinstance(device, tuple) else None
-  if isinstance(device, tuple): assert axis in (0, 1), f"unsupported sharding axis={axis}"
   grad_xw13     = alloc_like(xw13.shape, dtypes.bfloat16, device, axis)
   grad_xw13_fp8 = alloc_like(xw13.shape, dtypes.fp8e4m3,  device, axis)
-  grad_amax_buf = alloc_local((NUM_WG,), dtypes.float32,  device)
+  grad_amax_buf = alloc_local((NUM_WG,), dtypes.float32,  device, axis)
   grad_amax_state_t = Tensor(grad_amax_state, device=device)
   fxn = functools.partial(_custom_fused_bwd_w13, dname=dname_of(device))
   grad_xw13, grad_xw13_fp8, grad_amax_buf, *_ = Tensor.custom_kernel(
@@ -66,9 +65,8 @@ def fused_quantize_fp8_w13(xw13:Tensor, amax_state:Tensor, fp8_dtype, grad_amax_
   assert H2 % 2 == 0, f"w13 last-axis must be even, got {H2}"
   HIDDEN = H2 // 2
   axis = xw13.uop.axis if isinstance(xw13.device, tuple) else None
-  if isinstance(xw13.device, tuple): assert axis in (0, 1), f"unsupported sharding axis={axis}"
   fp8_out  = alloc_like((MBS, SEQ, HIDDEN), fp8_dtype,      xw13.device, axis)
-  amax_buf = alloc_local((NUM_WG,),         dtypes.float32, xw13.device)
+  amax_buf = alloc_local((NUM_WG,),         dtypes.float32, xw13.device, axis)
   fxn = functools.partial(_custom_fused_cast_amax_w13, dname=dname_of(xw13.device))
   fp8_out, amax_buf, *_ = Tensor.custom_kernel(fp8_out, amax_buf, xw13, amax_state, grad_amax_state,
                                                 fxn=fxn, grad_fxn=_fused_quantize_bwd_w13)

extra/llama_kernels/fp8_transpose/__init__.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+import functools, pathlib
+from tinygrad import Tensor, dtypes
+from tinygrad.uop.ops import UOp, Ops, KernelInfo
+from tinygrad.renderer import Estimates
+from extra.llama_kernels import THREADS_PER_WG, alloc_like, dname_of, compile_hip
+
+TILE = 64
+
+@functools.cache
+def _custom_fp8_transpose(out:UOp, inp:UOp, dname:str) -> UOp:
+  M, N = inp.shape
+  num_wg = (M // TILE) * (N // TILE)
+  threads, workgroups = UOp.special(THREADS_PER_WG, "lidx0"), UOp.special(num_wg, "gidx0")
+  mem = M * N * 2  # one byte read + one byte write per element
+  sink = UOp.sink(out.base, inp.base, threads, workgroups,
+                  arg=KernelInfo(f"fp8_transpose_{M}_{N}",
+                                 estimates=Estimates(ops=M*N, mem=mem)))
+  src = (pathlib.Path(__file__).parent/"fp8_transpose.cpp").read_text()
+  defines = [f"-DM_DIM={M}", f"-DN_DIM={N}", f"-DTHREADS_PER_WG={THREADS_PER_WG}"]
+  return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=dname), UOp(Ops.LINEAR, src=(*sink.src, sink)),
+                               UOp(Ops.SOURCE, arg=src), UOp(Ops.BINARY, arg=compile_hip(src, defines))))
+
+def fast_fp8_transpose(t:Tensor) -> Tensor:
+  assert t.ndim == 2, f"fast_fp8_transpose needs 2D input, got shape {t.shape}"
+  assert t.dtype in dtypes.fp8s, f"fast_fp8_transpose needs fp8 dtype, got {t.dtype}"
+  M, N = t.shape
+  assert M % TILE == 0 and N % TILE == 0, f"M={M}, N={N} must be multiples of {TILE}"
+
+  device = t.device
+  axis = t.uop.axis if isinstance(device, tuple) else None
+  out_axis = None
+  if axis == 0: out_axis = 1
+  elif axis == 1: out_axis = 0
+  elif axis is not None:
+    raise ValueError(f"fast_fp8_transpose: unsupported axis {axis}")
+
+  out = alloc_like((N, M), t.dtype, device, out_axis)
+  fxn = functools.partial(_custom_fp8_transpose, dname=dname_of(device))
+  out, _ = Tensor.custom_kernel(out, t, fxn=fxn)
+  return out

extra/llama_kernels/fp8_transpose/fp8_transpose.cpp

@@ -0,0 +1,74 @@
+#include <hip/hip_runtime.h>
+
+// LDS-staged 64x64 fp8 transpose.
+//   in : (M_DIM, N_DIM) fp8 contiguous
+//   out: (N_DIM, M_DIM) fp8 contiguous, out[c][r] = in[r][c]
+//
+// One WG processes one 64x64 output tile.  Each thread reads one uint4 (16 fp8) coalesced
+// from input rows, stages into LDS, then writes one uint4 coalesced to the output (whose
+// 16 fp8 come from 16 different input rows via in-LDS gather).
+//
+// LDS layout: lds[64][LDS_STRIDE] with LDS_STRIDE=65 (1 byte pad) to mitigate bank conflicts
+// during the column-direction read of the write phase.
+
+#ifndef M_DIM
+#define M_DIM 16384
+#endif
+#ifndef N_DIM
+#define N_DIM 28672
+#endif
+#ifndef THREADS_PER_WG
+#define THREADS_PER_WG 256
+#endif
+
+constexpr int TILE = 64;
+constexpr int VEC = 16;                    // fp8 per uint4 (128-bit) load/store
+constexpr int LDS_PAD = 1;
+constexpr int LDS_STRIDE = TILE + LDS_PAD; // 65 fp8 per row
+
+static_assert(THREADS_PER_WG * VEC == TILE * TILE, "256 threads * 16 fp8 = 64*64");
+static_assert(M_DIM % TILE == 0, "M_DIM must be a multiple of 64");
+static_assert(N_DIM % TILE == 0, "N_DIM must be a multiple of 64");
+
+constexpr int N_TILES_N = N_DIM / TILE;
+
+struct alignas(16) fp8x16 { uint8_t v[16]; };
+
+extern "C" __global__ __launch_bounds__(THREADS_PER_WG) void
+fp8_transpose(uint8_t* __restrict__ out,         // (N_DIM, M_DIM)
+              const uint8_t* __restrict__ in)    // (M_DIM, N_DIM)
+{
+  __shared__ uint8_t lds[TILE * LDS_STRIDE];
+
+  const int tid     = threadIdx.x;
+  const int wg_id   = blockIdx.x;
+  const int tile_r  = wg_id / N_TILES_N;       // tile index along M dim of input
+  const int tile_c  = wg_id % N_TILES_N;       // tile index along N dim of input
+
+  const int a = tid / (TILE / VEC);            // 0..63 (row within tile during read; col within tile during write)
+  const int b = tid % (TILE / VEC);            // 0..3
+  const int b16 = b * VEC;                     // 0,16,32,48
+
+  // ---- Read phase: input rows -> LDS rows
+  {
+    const long long src = (long long)(tile_r * TILE + a) * (long long)N_DIM
+                        + (long long)(tile_c * TILE + b16);
+    fp8x16 v = *reinterpret_cast<const fp8x16*>(&in[src]);
+    *reinterpret_cast<fp8x16*>(&lds[a * LDS_STRIDE + b16]) = v;
+  }
+  __syncthreads();
+
+  // ---- Write phase: LDS columns (gathered) -> output rows
+  // out[(tile_c*TILE + a)][(tile_r*TILE + b16 + i)] = in[(tile_r*TILE + b16 + i)][(tile_c*TILE + a)]
+  //                                                  = lds[b16 + i][a]
+  {
+    fp8x16 v;
+    #pragma unroll
+    for (int i = 0; i < VEC; ++i) {
+      v.v[i] = lds[(b16 + i) * LDS_STRIDE + a];
+    }
+    const long long dst = (long long)(tile_c * TILE + a) * (long long)M_DIM
+                        + (long long)(tile_r * TILE + b16);
+    *reinterpret_cast<fp8x16*>(&out[dst]) = v;
+  }
+}

extra/llama_kernels/fused_rmsnorm_mul_quantize_fp8/__init__.py

@@ -63,7 +63,7 @@ def _bwd_common(fp8_grad_u, h_grad_u, x_u, x_normed_u, rrms_u, weight_u, amax_st
   MBS, SEQ, HIDDEN = x_normed_u.shape
   axis = x_normed_u.axis if isinstance(device, tuple) else None
   grad_x = alloc_like((MBS, SEQ, HIDDEN), dtypes.bfloat16, device, axis)
-  grad_weight_partial = alloc_local((NUM_WG, HIDDEN), dtypes.float32, device)
+  grad_weight_partial = alloc_local((NUM_WG, HIDDEN), dtypes.float32, device, axis)
   grad_h_from_fp8 = None
   grad_weight_uop = None
   if fp8_grad_u is not None:
@@ -119,11 +119,11 @@ def fused_rmsnorm_mul_quantize_fp8(x:Tensor, weight:Tensor, amax_state:Tensor, e
   assert x.shape[-1] == weight.shape[-1], f"HIDDEN mismatch: x={x.shape}, weight={weight.shape}"
   MBS, SEQ, HIDDEN = x.shape
   axis = x.uop.axis if isinstance(x.device, tuple) else None
-  if isinstance(x.device, tuple): assert axis in (0, 1), f"unsupported sharding axis={axis}"
+  if isinstance(x.device, tuple): assert axis in (None, 0, 1), f"unsupported sharding axis={axis}"
   fp8_out      = alloc_like((MBS, SEQ, HIDDEN), fp8_dtype,       x.device, axis)
   x_normed_out = alloc_like((MBS, SEQ, HIDDEN), dtypes.bfloat16, x.device, axis)
   rrms_out     = alloc_like((MBS, SEQ),         dtypes.float32,  x.device, axis)
-  amax_buf     = alloc_local((NUM_WG,),         dtypes.float32,  x.device)
+  amax_buf     = alloc_local((NUM_WG,),         dtypes.float32,  x.device, axis)
   fxn = functools.partial(_custom_fwd, dname=dname_of(x.device), eps_val=eps)
   fp8_out, x_normed_out, rrms_out, amax_buf, *_ = Tensor.custom_kernel(
     fp8_out, x_normed_out, rrms_out, amax_buf, x, weight, amax_state, fxn=fxn, grad_fxn=_fused_bwd)
@@ -139,12 +139,12 @@ def fused_add_rmsnorm_mul_quantize_fp8(x:Tensor, residual:Tensor, weight:Tensor,
   assert x.shape == residual.shape
   MBS, SEQ, HIDDEN = x.shape
   axis = x.uop.axis if isinstance(x.device, tuple) else None
-  if isinstance(x.device, tuple): assert axis in (0, 1), f"unsupported sharding axis={axis}"
+  if isinstance(x.device, tuple): assert axis in (None, 0, 1), f"unsupported sharding axis={axis}"
   fp8_out      = alloc_like((MBS, SEQ, HIDDEN), fp8_dtype,       x.device, axis)
   h_out        = alloc_like((MBS, SEQ, HIDDEN), dtypes.bfloat16, x.device, axis)
   x_normed_out = alloc_like((MBS, SEQ, HIDDEN), dtypes.bfloat16, x.device, axis)
   rrms_out     = alloc_like((MBS, SEQ),         dtypes.float32,  x.device, axis)
-  amax_buf     = alloc_local((NUM_WG,),         dtypes.float32,  x.device)
+  amax_buf     = alloc_local((NUM_WG,),         dtypes.float32,  x.device, axis)
   fxn = functools.partial(_custom_fwd_add, dname=dname_of(x.device), eps_val=eps)
   fp8_out, h_out, x_normed_out, rrms_out, amax_buf, *_ = Tensor.custom_kernel(
     fp8_out, h_out, x_normed_out, rrms_out, amax_buf, x, residual, weight, amax_state,

extra/llama_kernels/quantize_fp8_delayed/__init__.py

@@ -49,7 +49,7 @@ def quantize_fp8_delayed(x:Tensor, amax_state:Tensor, fp8_dtype=dtypes.fp8e4m3)
   assert x.dtype == dtypes.bfloat16, f"expected bf16, got {x.dtype}"
   axis = x.uop.axis if isinstance(x.device, tuple) else None
   fp8_out      = alloc_like(x.shape,  fp8_dtype,      x.device, axis)
-  amax_partial = alloc_local((NUM_WG,), dtypes.float32, x.device)
+  amax_partial = alloc_local((NUM_WG,), dtypes.float32, x.device, axis)
   fxn = functools.partial(_custom_quantize_fp8_with_amax, dname=dname_of(x.device))
   fp8_out, amax_partial, *_ = Tensor.custom_kernel(fp8_out, amax_partial, x, amax_state,
                                                     fxn=fxn, grad_fxn=_quantize_fp8_delayed_bwd)

extra/nv_gpu_driver/fw.h

@@ -0,0 +1,25 @@
+/* adapted from linux/drivers/gpu/drm/nouveau/include/nvfw/fw.h */
+/* SPDX-License-Identifier: MIT */
+#ifndef __NVFW_FW_H__
+#define __NVFW_FW_H__
+typedef unsigned int u32;
+
+struct nvfw_bin_hdr {
+	u32 bin_magic;
+	u32 bin_ver;
+	u32 bin_size;
+	u32 header_offset;
+	u32 data_offset;
+	u32 data_size;
+};
+
+struct nvfw_bl_desc {
+	u32 start_tag;
+	u32 dmem_load_off;
+	u32 code_off;
+	u32 code_size;
+	u32 data_off;
+	u32 data_size;
+};
+
+#endif

extra/nv_gpu_driver/hs.h

@@ -0,0 +1,52 @@
+/* adapted from linux/drivers/gpu/drm/nouveau/include/nvfw/hs.h */
+/* SPDX-License-Identifier: MIT */
+#ifndef __NVFW_HS_H__
+#define __NVFW_HS_H__
+typedef unsigned int u32;
+
+struct nvfw_hs_header {
+	u32 sig_dbg_offset;
+	u32 sig_dbg_size;
+	u32 sig_prod_offset;
+	u32 sig_prod_size;
+	u32 patch_loc;
+	u32 patch_sig;
+	u32 hdr_offset;
+	u32 hdr_size;
+};
+
+struct nvfw_hs_header_v2 {
+	u32 sig_prod_offset;
+	u32 sig_prod_size;
+	u32 patch_loc;
+	u32 patch_sig;
+	u32 meta_data_offset;
+	u32 meta_data_size;
+	u32 num_sig;
+	u32 header_offset;
+	u32 header_size;
+};
+
+struct nvfw_hs_load_header {
+	u32 non_sec_code_off;
+	u32 non_sec_code_size;
+	u32 data_dma_base;
+	u32 data_size;
+	u32 num_apps;
+	u32 apps[];
+};
+
+struct nvfw_hs_load_header_v2 {
+	u32 os_code_offset;
+	u32 os_code_size;
+	u32 os_data_offset;
+	u32 os_data_size;
+	u32 num_apps;
+	struct {
+		u32 offset;
+		u32 size;
+		u32 data_offset;
+		u32 data_size;
+	} app[];
+};
+#endif

extra/thunder/amd/fa.py

@@ -55,8 +55,6 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
   assert attn_mask is None, "attn_mask not supported"
   assert is_causal, "only causal attention supported"
 
-  xq, xk, xv = xq.transpose(1, 2), xk.transpose(1, 2), xv.transpose(1, 2)
-
   B, N, H, D = xq.shape
   H_KV = xk.shape[2]
   assert D == 128, "only D=128 supported"
@@ -81,7 +79,7 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
 
   attn, l_vec = Tensor.custom_kernel(attn, l_vec, xq, xk, xv, fxn=functools.partial(custom_fa_forward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D), grad_fxn=grad)[:2]
 
-  return attn.transpose(1, 2), attn, l_vec
+  return attn, attn, l_vec
 
 @functools.cache
 def custom_fa_forward(o:UOp, l_vec:UOp, q:UOp, k:UOp, v:UOp, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int):

extra/thunder/amd/gemm_fp8.cpp

@@ -93,7 +93,20 @@ constexpr int NUM_WARPS = 8;
 
 using G = kittens::group<NUM_WARPS>;
 
-__global__ __launch_bounds__(512, 2) void hk_fp8_gemm(bf16 *C_ptr, fp8e4m3 *A_ptr, fp8e4m3 *B_ptr, float *x_scale_ptr, float *w_scale_ptr) {
+// scale_mode: 0=no scale, 1=x only, 2=w only, 3=both
+#ifndef SCALE_MODE
+#define SCALE_MODE 3
+#endif
+
+__global__ __launch_bounds__(512, 2) void hk_fp8_gemm(bf16 *C_ptr, fp8e4m3 *A_ptr, fp8e4m3 *B_ptr
+#if SCALE_MODE == 1
+    , float *x_scale_ptr
+#elif SCALE_MODE == 2
+    , float *w_scale_ptr
+#elif SCALE_MODE == 3
+    , float *x_scale_ptr, float *w_scale_ptr
+#endif
+) {
     constexpr int M = GEMM_M, N = GEMM_N, K = GEMM_K;
 
     kittens::gl<fp8e4m3, 1, 1, M, K> A{A_ptr, nullptr, nullptr, nullptr, nullptr};
@@ -333,11 +346,25 @@ __global__ __launch_bounds__(512, 2) void hk_fp8_gemm(bf16 *C_ptr, fp8e4m3 *A_pt
     }
 
     // apply x_scale * w_scale before bf16 store to prevent overflow
+#if SCALE_MODE == 1
+    float scale = *x_scale_ptr;
+    mul(cA, cA, scale);
+    mul(cB, cB, scale);
+    mul(cC, cC, scale);
+    mul(cD, cD, scale);
+#elif SCALE_MODE == 2
+    float scale = *w_scale_ptr;
+    mul(cA, cA, scale);
+    mul(cB, cB, scale);
+    mul(cC, cC, scale);
+    mul(cD, cD, scale);
+#elif SCALE_MODE == 3
     float scale = *x_scale_ptr * *w_scale_ptr;
     mul(cA, cA, scale);
     mul(cB, cB, scale);
     mul(cC, cC, scale);
     mul(cD, cD, scale);
+#endif
 
     store(C, cA, {0, 0, block_row * WARPS_ROW * 2 + warp_m, block_col * WARPS_COL * 2 + warp_n});
     store(C, cB, {0, 0, block_row * WARPS_ROW * 2 + warp_m, block_col * WARPS_COL * 2 + WARPS_COL + warp_n});

extra/usbgpu/tbgpu/installer/TinyGPUDriverExtension.xcodeproj/project.pbxproj

@@ -359,7 +359,7 @@
 					"$(inherited)",
 					"@executable_path/../Frameworks",
 				);
-				MACOSX_DEPLOYMENT_TARGET = 12.1;
+				MACOSX_DEPLOYMENT_TARGET = 13.0;
 				MARKETING_VERSION = 1.0.0;
 				PRODUCT_BUNDLE_IDENTIFIER = org.tinygrad.tinygpu.installer;
 				PRODUCT_NAME = TinyGPU;
@@ -397,7 +397,7 @@
 					"$(inherited)",
 					"@executable_path/../Frameworks",
 				);
-				MACOSX_DEPLOYMENT_TARGET = 12.1;
+				MACOSX_DEPLOYMENT_TARGET = 13.0;
 				MARKETING_VERSION = 1.0.0;
 				PRODUCT_BUNDLE_IDENTIFIER = org.tinygrad.tinygpu.installer;
 				PRODUCT_NAME = TinyGPU;
@@ -446,7 +446,7 @@
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
 				COPY_PHASE_STRIP = NO;
 				DEBUG_INFORMATION_FORMAT = dwarf;
-				DRIVERKIT_DEPLOYMENT_TARGET = 21.0;
+				DRIVERKIT_DEPLOYMENT_TARGET = 22.0;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
 				ENABLE_TESTABILITY = YES;
 				GCC_C_LANGUAGE_STANDARD = gnu11;
@@ -506,7 +506,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				COPY_PHASE_STRIP = NO;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				DRIVERKIT_DEPLOYMENT_TARGET = 21.0;
+				DRIVERKIT_DEPLOYMENT_TARGET = 22.0;
 				ENABLE_NS_ASSERTIONS = NO;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
 				GCC_C_LANGUAGE_STANDARD = gnu11;
@@ -533,7 +533,7 @@
 				CODE_SIGN_STYLE = Automatic;
 				CURRENT_PROJECT_VERSION = 3;
 				DEVELOPMENT_TEAM = 9YG3G8543N;
-				DRIVERKIT_DEPLOYMENT_TARGET = 21.0;
+				DRIVERKIT_DEPLOYMENT_TARGET = 22.0;
 				ENABLE_USER_SCRIPT_SANDBOXING = YES;
 				EXCLUDED_ARCHS = "";
 				FRAMEWORK_SEARCH_PATHS = (
@@ -566,7 +566,7 @@
 				CURRENT_PROJECT_VERSION = 3;
 				DEVELOPMENT_TEAM = "";
 				"DEVELOPMENT_TEAM[sdk=driverkit*]" = 9YG3G8543N;
-				DRIVERKIT_DEPLOYMENT_TARGET = 21.0;
+				DRIVERKIT_DEPLOYMENT_TARGET = 22.0;
 				ENABLE_USER_SCRIPT_SANDBOXING = YES;
 				EXCLUDED_ARCHS = "";
 				FRAMEWORK_SEARCH_PATHS = (

extra/usbgpu/tbgpu/installer/TinyGPUDriverExtension/TinyGPUDriver.cpp

@@ -188,8 +188,8 @@ kern_return_t TinyGPUDriver::CfgWrite(uint32_t off, uint32_t size, uint32_t val)
 kern_return_t TinyGPUDriver::ResetDevice()
 {
 	if (!ivars->pci) return kIOReturnNotReady;
-	ivars->pci->Reset(kIOPCIDeviceResetTypeFunctionReset);
-	return 0;
+	kern_return_t ret = ivars->pci->Reset(kIOPCIDeviceResetTypeFunctionReset);
+	return ret == kIOReturnSuccess ? ret : ivars->pci->Reset(kIOPCIDeviceResetTypeHotReset);
 }
 
 IOPCIDevice* TinyGPUDriver::GetPCI()

pyproject.toml

@@ -251,8 +251,7 @@ select = [
   "F541",
   "F841",
 ]
-"tinygrad/runtime/autogen/**/*.py" = ["E501", "F401", "E722", "E731", "F821", "A006", "A002", "F811"]
-"tinygrad/runtime/autogen/amd/**/*.py" = ["E501"]
+"tinygrad/runtime/autogen/**/*.py" = ["E501", "F401", "E731", "F821", "A006", "A002", "F811", "F822"]
 "test/amd/**/*.py" = ["F403", "F405"]
 
 [tool.ruff.format]

spec/README.md

@@ -0,0 +1 @@
+Run `./render.sh` whenever you update tinyspec.tex to regenerate tinyspec.pdf.

spec/render.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+if ! command -v tectonic &>/dev/null; then
+  echo "tectonic not found, installing..."
+  sudo pacman -S --noconfirm tectonic
+fi
+
+tectonic tinyspec.tex
+echo "done: tinyspec.pdf"

spec/tinyspec.tex

@@ -0,0 +1,450 @@
+\documentclass[10pt,letterpaper]{article}
+
+\usepackage[margin=0.75in]{geometry}
+\usepackage{amsmath,amssymb}
+\usepackage{booktabs}
+\usepackage{array}
+\usepackage[dvipsnames]{xcolor}
+\usepackage{enumitem}
+\usepackage{listings}
+\lstset{language=Python, basicstyle=\ttfamily\small, columns=fullflexible, keepspaces=true}
+
+\newcommand{\op}[1]{\textsc{#1}}
+
+\definecolor{movgreen}{HTML}{2E7D32}
+\definecolor{reducered}{HTML}{C62828}
+\definecolor{elwyellow}{HTML}{F9A825}
+\definecolor{callblue}{HTML}{1565C0}
+\definecolor{assignbrown}{HTML}{795548}
+\definecolor{multipurple}{HTML}{7B1FA2}
+\definecolor{markerorange}{HTML}{E65100}
+% AxisType colors (from tinygrad)
+\definecolor{axblue}{HTML}{1565C0}       % GLOBAL
+\definecolor{axcyan}{HTML}{00838F}       % LOCAL
+\definecolor{axbrcyan}{HTML}{00ACC1}     % WARP
+\definecolor{axbrblue}{HTML}{42A5F5}     % THREAD
+\definecolor{axwhite}{HTML}{616161}      % LOOP (gray on white paper)
+\definecolor{axred}{HTML}{C62828}        % REDUCE
+\definecolor{axbrred}{HTML}{E53935}      % GROUP_REDUCE
+\definecolor{axyellow}{HTML}{F9A825}     % UPCAST
+\definecolor{axmagenta}{HTML}{7B1FA2}    % UNROLL
+
+\title{tinygrad: a single dialect from Tensor programs to Command Buffers}
+\author{tinygrad, Corp. \\ \texttt{[email protected]}}
+\date{}
+
+\begin{document}
+\maketitle
+\thispagestyle{empty}
+
+\section*{UOps}
+
+All nodes in the tinygrad graph are \textbf{UOps}. A UOp is a tuple $(\mathrm{op},\;\mathrm{src},\;\mathrm{arg},\;\mathrm{tag})$ where $\mathrm{op}$ is from the set below, $\mathrm{src}$ is a tuple of input UOps, $\mathrm{arg}$ is op-dependent, and $\mathrm{tag}$ is for temporary processing. The full program is a DAG of UOps. Each UOp has five derived properties --- \textbf{dtype}, \textbf{shape}, \textbf{device}, \textbf{min\_max}, and \textbf{axis} --- determined by the rules at the end of this document.
+
+%% ============================================================
+\subsection*{Source Ops \normalfont\small--- leaf nodes}
+
+\begin{tabular}{@{}l p{3.2cm} p{3.0cm} p{6.2cm}@{}}
+\toprule
+\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\
+\midrule
+\op{Buffer} & () & size, dtype, device, addrspace &
+  Shape $(n \cdot \textit{size},)$ if device is $n$-tuple, else $(\textit{size},)$. \\
+\op{BufferView} & (buf,) & size, dtype, offset &
+  Typed access into a buffer. Zero-copy $(\textit{size},)$ slice at offset; inherits addrspace. \\
+\op{Param} & $(\mathbf{s})$ or $(\mathbf{s}, \text{min}, \text{max})$ & slot, dtype, device? &
+  Placeholder with shape $\mathbf{s}$. Substituted in \op{Function}. \\[4pt]
+\op{Const} & () & value, dtype &
+  A scalar constant with shape $(\ )$. \\
+\op{Vconst} & () & values, dtype &
+  A vector constant with shape $(n,)$. \\
+\bottomrule
+\end{tabular}
+
+\smallskip
+A \op{Buffer}'s \textbf{addrspace} is \texttt{GLOBAL}, \texttt{LOCAL}, or \texttt{REG}.
+
+%% ============================================================
+\subsection*{{\color{movgreen}Movement Ops} \normalfont\small--- no arithmetic, shapes are $(k,)$-shaped UOps with dtype \texttt{index} in src}
+
+\begin{tabular}{@{}l l l l@{}}
+\toprule
+\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\
+\midrule
+\op{Permute} & $(T,)$ & axis order $\pi$ & Reorder axes. $\pi = (1,0)$ is transpose. \\
+\op{Flip} & $(T,)$ & bools $\mathbf{f}$ & Reverse along flagged axes. \\
+\op{Reshape} & $(T, \mathbf{s'})$ & --- & Reinterpret in row-major order. $\prod s_k = \prod s'_k$. \\
+\op{Expand} & $(T, \mathbf{s'})$ & --- & Broadcast size-1 axes. $s_k \in \{1, s'_k\}$. \\
+\op{Pad} & $(T, \mathbf{b}, \mathbf{e})$ & --- & Pad with $0$s: $b_k$ before, $e_k$ after each axis. \\
+\op{Shrink} & $(T, \mathbf{b}, \mathbf{e})$ & --- & Keep $[b_k, e_k)$ per axis. Inverse of \op{Pad}. \\
+\op{Index} & $(T, i_0, i_1, \ldots)$ & --- & Index from left. $()$-shaped $i$ removes dim; $(k,)$-shaped makes it $k$. \\
+\op{Stack} & $(T_0, T_1, \ldots)$ & --- & Join along a newly created leading axis. All shapes must match. \\
+\op{Replicated} & $(T,)$ & axes & Mark $T$ as replicated along axes. Collapse axes to $1$. \\
+\bottomrule
+\end{tabular}
+
+%% ============================================================
+\subsection*{{\color{reducered}Reduce Ops} \normalfont\small--- collapse axes to size $1$}
+
+\begin{tabular}{@{}l l l l@{}}
+\toprule
+\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\
+\midrule
+\op{Reduce} & $(T,)$ & op, axes & Reduce $T$ along axes. Op is \op{Add}, \op{Max}, or \op{Mul}. \\
+\bottomrule
+\end{tabular}
+
+%% ============================================================
+\subsection*{{\color{callblue}Call Ops} \normalfont\small--- function abstraction, like the lambda calculus}
+
+\begin{tabular}{@{}l l l l@{}}
+\toprule
+\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\
+\midrule
+\op{Function} & (body, $a_0$, $a_1$, \ldots) & --- & Substitute each \op{Param} $k$ in \op{Tuple} body with $a_k$. Gradient-able. \\
+\op{Call} & (body, $a_0$, $a_1$, \ldots) & --- & Opaque invocation of a compiled kernel or custom function. \\
+\op{Tuple} & $(v_0, v_1, \ldots)$ & --- & Pack values; required as \op{Function} body to return a value. \\
+\op{GetTuple} & $(T,)$ & idx & Extract element at idx from a \op{Tuple}. \\
+\bottomrule
+\end{tabular}
+
+%% ============================================================
+\subsection*{{\color{multipurple}Store Ops} \normalfont\small--- side effects}
+
+\begin{tabular}{@{}l l l l@{}}
+\toprule
+\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\
+\midrule
+\op{Store} & (buf, val, gate?) & --- & Write val into buf. buf.shape $=$ val.shape. \\
+  & & & If gate is present, write only when gate is true. Output is void. \\
+\bottomrule
+\end{tabular}
+
+%% ============================================================
+\subsection*{{\color{assignbrown}Ordering Ops} \normalfont\small--- execution order}
+
+\begin{tabular}{@{}l l l p{6.0cm}@{}}
+\toprule
+\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\
+\midrule
+\op{Range} & $(\text{bound},)$ & type & Iterator from $0$ to bound. \\
+\op{End} & (body, range) & --- & Close a \op{Range} loop. \\
+\op{After} & (buf, deps\ldots) & --- & Passthrough of buf; guarantees deps execute first. \\
+\op{Group} & $(u_0, u_1, \ldots)$ & --- & Void no-op that merges multiple \op{Store}s into one node, unordered. \\
+\op{Sink} & $(s_0, s_1, \ldots)$ & --- & Collect side effects into a single root node. \\
+\op{Linear} & (uops\ldots) & --- & Linearized (toposorted) instruction sequence. \\
+\bottomrule
+\end{tabular}
+
+\smallskip
+Assign is \op{Store} followed by \op{After}: write the value, then return the buffer with an ordering dependency.
+
+%% ============================================================
+\subsection*{{\color{elwyellow}Elementwise Ops} \normalfont\small--- all inputs same shape, output same shape, applied per-element}
+
+\begin{tabular}{@{}l l l l@{}}
+\toprule
+\textbf{Arity} & \textbf{src} & \textbf{Op} & \textbf{Semantics} \\
+\midrule
+Unary & $(T,)$
+  & \op{Recip}
+  & $1/x$ \\
+& & \op{Trunc}
+  & $\mathrm{trunc}(x)$: round toward zero. \\
+& & \op{Cast}
+  & Convert to target dtype (specified in arg). \\
+& & \op{Bitcast}
+  & Reinterpret bits as target dtype. Must be same size. \\[4pt]
+Binary & $(A, B)$
+  & \op{Add}, \op{Mul}, \op{Max}, \op{Mod}, \op{Idiv}
+  & $a+b$, $a \cdot b$, $\max(a,b)$, $a \bmod b$, $\lfloor a/b \rfloor$ \\
+& & \op{CmpLt}, \op{CmpNe}
+  & $[a < b]$, $[a \ne b]$ \\
+& & \op{Xor}, \op{Or}, \op{And}, \op{Shr}, \op{Shl}
+  & $a \oplus b$, $a \mid b$, $a \mathbin{\&} b$, $a \gg b$, $a \ll b$ \\[4pt]
+Ternary & $(P, A, B)$
+  & \op{Where}
+  & $A[\mathbf{i}]$ if $P[\mathbf{i}] \ne 0$, else $B[\mathbf{i}]$ \\
+
+\bottomrule
+\end{tabular}
+
+\medskip
+\textbf{Decomposed elementwise ops} --- defined in terms of the primitives above.
+
+\smallskip
+\begin{tabular}{@{}l l l@{}}
+\toprule
+\textbf{Op} & \textbf{Decomposition} & \textbf{Semantics} \\
+\midrule
+\op{Neg} & \op{Mul}($A$, $-1$) & $-x$ \\
+\op{Sub} & \op{Add}($A$, \op{Neg}($B$)) & $a - b$ \\
+\op{Div} & \op{Mul}($A$, \op{Recip}($B$)) & $a / b$ \\
+\op{CmpGt} & \op{CmpLt}($B$, $A$) & $[a > b]$ \\
+\op{CmpGe} & \op{CmpNe}(\op{CmpLt}($A$, $B$),\, $1$) & $[a \ge b]$ \\
+\op{CmpLe} & \op{CmpNe}(\op{CmpLt}($B$, $A$),\, $1$) & $[a \le b]$ \\
+\op{CmpEq} & \op{CmpNe}(\op{CmpNe}($A$, $B$),\, $1$) & $[a = b]$ \\
+\op{Not} & \op{CmpNe}($A$, $1$) & $\lnot a$ \\[4pt]
+\op{Exp2} & polynomial approx + \op{Mul}, \op{Add} & $2^x$ \\
+\op{Log2} & exponent extract + polynomial approx & $\log_2 x$ \\
+\op{Sin} & argument reduction + polynomial approx & $\sin x$ \\
+\op{Sqrt} & \op{Exp2}($0.5 \cdot$ \op{Log2}($A$)) & $\sqrt{x}$ \\
+\op{Pow} & \op{Exp2}(\op{Log2}($A$) $\cdot\, B$) & $a^b$ \\
+\op{Mulacc} & \op{Add}(\op{Mul}($A$, $B$),\, $C$) & $a \cdot b + c$ \\
+\op{Threefry} & 5 rounds of add-rotate-xor (ARX) & Threefry 2x32 PRNG \\
+\bottomrule
+\end{tabular}
+
+%% ============================================================
+\subsection*{{\color{markerorange}Marker Ops} \normalfont\small--- identity on data}
+
+\begin{tabular}{@{}l l l l@{}}
+\toprule
+\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\
+\midrule
+\op{Contiguous} & $(T,)$ & --- & Force contiguous memory layout. \\
+\op{ContiguousBackward} & $(T,)$ & --- & Force contiguous in backward pass. \\
+\op{Detach} & $(T,)$ & --- & Stops gradient propagation. \\
+\op{Copy} & $(T,)$ & device & Copy to target device. \\
+\bottomrule
+\end{tabular}
+
+%% ============================================================
+\subsection*{Codegen Ops \normalfont\small--- generated code primitives, these do not appear in the main graph}
+
+\begin{tabular}{@{}l l l l@{}}
+\toprule
+\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\
+\midrule
+\op{Load} & (idx,alt?,gate?) & --- & Dereference: read element at index from buffer. \\
+ & & & All loads will be replaced by \op{Store}. \\
+\op{Barrier} & (deps\ldots) & --- & Synchronize threads within a workgroup. \\
+\op{Ins} & \ldots & \ldots & A single machine instruction (e.g.\ AMD ISA). \\
+\op{Special} & (bound,) & name & GPU thread/workgroup index (e.g.\ \texttt{gidx0}, \texttt{lidx1}). \\
+\op{If} & (gate,) & --- & Begin conditional execution block. \\
+\op{Endif} & (if,) & --- & End conditional execution block. \\
+\op{Wmma} & (A, B, acc) & config & Warp matrix multiply-accumulate (tensor cores). \\
+\op{Custom} & (args\ldots) & fmt & Inject custom code string into generated source. \\
+\op{AtomicAdd} & (idx, val) & --- & Atomic read-modify-write: \texttt{buf[idx] += val}. \\[4pt]
+\op{CustomFunction} & (meta\ldots) & name & Opaque device function (e.g.\ HW decode). Via \op{Call}. \\
+\op{Program} & (linear, source, binary) & --- & Compiled kernel: instructions, source, and machine code. \\
+\op{Source} & () & str & Human-readable rendered source code. \\
+\op{Binary} & () & bytes & Compiled machine code. \\
+\bottomrule
+\end{tabular}
+
+\smallskip
+These ops are not part of the core specification and are subject to change.
+
+%% ============================================================
+\subsection*{Derived Properties}
+
+Every UOp has a \textbf{dtype}, \textbf{shape}, \textbf{device}, \textbf{min\_max}, and \textbf{axis}, derived from its op, src, and arg:
+
+\medskip
+\begin{tabular}{@{}l l l l l@{}}
+\toprule
+\textbf{Op} & \textbf{dtype} & \textbf{shape} & \textbf{device} & \textbf{min\_max} \\
+\midrule
+\op{Buffer} & from arg & $(\text{size},)$ from arg & from arg & dtype range \\
+\op{Const} & from arg & $()$ & \textsc{null} & $[v, v]$ \\
+\op{Param} & from arg & from $\mathrm{src}[0]$ & from arg & from src or dtype range \\[3pt]
+Movement ops & $\mathrm{src}[0].\mathrm{dtype}$ & (see op) & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\
+\op{Reduce} & $\mathrm{src}[0].\mathrm{dtype}$ & collapse axes to $1$ & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\[3pt]
+\op{Cast} & from arg & $\mathrm{src}[0].\mathrm{shape}$ & $\mathrm{src}[0].\mathrm{device}$ & clamped to dtype \\
+\op{Bitcast} & from arg & $\mathrm{src}[0].\mathrm{shape}$ & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\
+\op{Copy} & $\mathrm{src}[0].\mathrm{dtype}$ & $\mathrm{src}[0].\mathrm{shape}$ & from arg & $\mathrm{src}[0]$ \\
+ALU unary & $\mathrm{src}[0].\mathrm{dtype}$ & $\mathrm{src}[0].\mathrm{shape}$ & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\
+\op{Add} & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[a+b,\, A+B]$ \\
+\op{Mul} & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[\min,\max]$ of products \\
+\op{Max} & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[\max(a,b),\, \max(A,B)]$ \\
+Other binary & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\
+\op{CmpLt}, \op{CmpNe} & bool & broadcast & $\mathrm{src}[0].\mathrm{device}$ & from intervals \\
+\op{Where} & $\mathrm{src}[1].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[\min(b,c),\, \max(B,C)]$ \\[3pt]
+\op{Function}, \op{Call} & $\mathrm{src}[0].\mathrm{dtype}$ & substitute \op{Param} shapes & $\mathrm{src}[1].\mathrm{device}$ & dtype range \\
+\op{Range} & index & $()$ & \textsc{null} & $[0,\, n{-}1]$ \\
+\op{Index} & $\mathrm{src}[0].\mathrm{dtype}$ & remaining dims & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\
+\op{Store} & void & $()$ & $\mathrm{src}[0].\mathrm{device}$ & --- \\
+\op{After} & $\mathrm{src}[0].\mathrm{dtype}$ & $\mathrm{src}[0].\mathrm{shape}$ & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\
+\bottomrule
+\end{tabular}
+
+\smallskip
+$\mathrm{broadcast}$: right-align shapes, element-wise max; each axis must be equal or $1$.
+$[a,A]$, $[b,B]$, $[c,C]$ denote min\_max of $\mathrm{src}[0]$, $\mathrm{src}[1]$, $\mathrm{src}[2]$.
+Default \emph{dtype range}: $[\mathrm{dtype\_min},\, \mathrm{dtype\_max}]$.
+
+\medskip
+\textbf{axis} tracks the multi-device sharding dimension. \op{Buffer} with $n$-tuple device: axis $= 0$ (device dim).
+\op{Reshape} remaps axis to preserve the shard boundary. \op{Permute} follows the permutation.
+\op{Reduce} on the shard axis $\to$ \textsc{null}. \op{Replicated} on the shard axis $\to$ \textsc{null}. \op{Copy} $\to$ \textsc{null}. ALU ops inherit from sources. Default: \textsc{null}.
+
+%% ============================================================
+\subsection*{Kernel Optimizations (OptOps) \normalfont\small--- schedule-level transforms on kernel ranges}
+
+Each kernel's iteration space is a set of \op{Range} axes. Every range has an \textbf{AxisType}:
+
+\medskip
+\begin{tabular}{@{}l l l l l@{}}
+\toprule
+\textbf{AxisType} & \textbf{Letter} & \textbf{Split from} & \textbf{Direction} & \textbf{Semantics} \\
+\midrule
+{\color{axblue}\texttt{GLOBAL}} & \texttt{g} & --- & --- & GPU global workgroup dimension. \\
+{\color{axcyan}\texttt{LOCAL}} & \texttt{l} & g, L & inner & Workgroup local dimension (shared memory). \\
+{\color{axbrcyan}\texttt{WARP}} & \texttt{w} & \multicolumn{2}{l}{(created by \op{TC})} & Warp-level lanes for tensor cores. \\
+{\color{axbrblue}\texttt{THREAD}} & \texttt{t} & g & outer & CPU thread parallelism. \\
+{\color{axwhite}\texttt{LOOP}} & \texttt{L} & --- & --- & Generic sequential loop (initial state). \\
+{\color{axred}\texttt{REDUCE}} & \texttt{R} & --- & --- & Reduction axis. \\
+{\color{axbrred}\texttt{GROUP\_REDUCE}} & \texttt{G} & R & inner/outer & Shared-memory group reduction. \\
+{\color{axyellow}\texttt{UPCAST}} & \texttt{u} & g, l, L & inner & Register-level vectorization. \\
+{\color{axmagenta}\texttt{UNROLL}} & \texttt{r} & R, G & inner & Fully unrolled loop. \\
+\bottomrule
+\end{tabular}
+
+\medskip
+An optimization is a triple $(\mathrm{op},\;\mathrm{axis},\;\mathrm{arg})$:
+
+\smallskip
+\begin{tabular}{@{}l l l p{6.5cm}@{}}
+\toprule
+\textbf{OptOp} & \textbf{axis} & \textbf{arg} & \textbf{Semantics} \\
+\midrule
+\op{Split} & any & (factor $k$, target, top?) &
+  Split axis $n$ by $k$ into $(n/k, k)$ or $(k, n/k)$ if top. New sub-axis gets target AxisType (see table above). \\
+\op{Padto} & any & multiple $m$ &
+  Pad axis to next multiple of $m$ with validity masks. \\[4pt]
+\op{Swap} & axis$_i$ & axis$_j$ &
+  Swap two axes $i \leftrightarrow j$. \\
+\op{Nolocals} & --- & --- &
+  Disable local memory; no workgroup dims emitted. \\
+\op{TC} & reduce idx & (tc, opt, mode) &
+  Apply tensor core \op{Wmma}: split reduce/output axes into \texttt{WARP}, \texttt{UPCAST}, and \texttt{UNROLL} dims. \\
+\bottomrule
+\end{tabular}
+
+\smallskip
+Optimizations compose left-to-right. \op{TC} must be first. The search space is explored by BEAM search or hand-coded heuristics.
+
+%% ============================================================
+\subsection*{Common Ops as Compositions}
+
+All high-level tensor operations decompose into the primitives above.
+
+\begin{lstlisting}
+# gemm: C[M,N] = A[M,K] @ B[K,N]
+def gemm(A, B):
+  M,K = A.shape; _,N = B.shape
+  return (A.reshape(M,K,1) * B.reshape(1,K,N)).sum(1)
+
+# prefix_sum: cumulative sum via repeat+reshape sliding window trick
+def prefix_sum(T):
+  n = T.shape[0]
+  x = T.pad((n-1, 0))                            # (2n-1,)
+  x = x.reshape(1,2*n-1).expand(n+1,2*n-1)       # tile
+  x = x.reshape((n+1)*(2*n-1)).shrink_to(2*n*n)  # trim
+  x = x.reshape(n,2*n).shrink_to(n,n)            # windows
+  return x.sum(-1)                               # reduce
+
+# arange: prefix_sum of all 1s gives [1,2,...,n], subtract 1 for [0,1,...,n-1]
+def arange(n):
+  return prefix_sum(Tensor(1).reshape(1).expand(n)) - 1
+
+# gather: out[i] = T[idx[i]]. one-hot mask along gather axis, then reduce
+def gather(T, idx):
+  K = T.shape[0]
+  pos = arange(K).reshape(K, 1)                     # (K, 1)
+  mask = (pos == idx.reshape(1, -1)).cast(T.dtype)  # (K, D)
+  return (T.reshape(K, 1) * mask).sum(0)            # (D,)
+
+# scatter_add: T[idx[i]] += val[i]
+def scatter_add(T, idx, val):
+  K, D = T.shape[0], idx.shape[0]
+  pos = arange(K).reshape(K, 1)                     # (K, 1)
+  mask = (pos == idx.reshape(1, D)).cast(T.dtype)   # (K, D)
+  return T + (mask * val.reshape(1, D)).sum(1)      # (K,)
+\end{lstlisting}
+
+%% ============================================================
+\subsection*{{\color{multipurple}Multi-Device Collectives} \normalfont\small--- derived from primitives}
+
+Let $D = (d_0, \ldots, d_{n-1})$ be an $n$-tuple device.
+\op{Copy} to an $n$-tuple device reshards with axis $= 0$. \op{Copy} never changes shape.
+
+\begin{lstlisting}
+# T has shape (s,) on a single device.
+
+# broadcast: replicate T to all n devices
+def broadcast(T):
+  return T.reshape(1, s).expand(n, s).copy(D).replicated(0)        # (s,) on D, axis=null
+
+# scatter: split T into n chunks, one per device
+def scatter(T):
+  return T.copy(D)                                                  # (s,) on D, axis=0
+
+# T has shape (n*s,) on D with axis=0, so each device holds (s,) elements.
+
+# gather: collect all shards onto one device
+def gather(T):
+  return T.copy(D[0])                                               # (n*s,) on D[0], axis=null
+
+# reduce: gather + sum
+def reduce(T):
+  return gather(T).reshape(n, s).sum(0)                             # (s,) on D[0], axis=null
+
+# allgather: collect all shards, replicate to all devices
+def allgather(T):
+  return T.reshape(1, n*s).expand(n, n*s).copy(D).replicated(0)    # (n*s,) on D, axis=null
+
+# reduce_scatter: reduce across devices, scatter result
+def reduce_scatter(T):
+  return T.reshape(n, n, s//n).permute(1, 0, 2).copy(D).sum(1).reshape(s)  # (s,) on D, axis=0
+
+# allreduce: reduce_scatter + allgather
+def allreduce(T):
+  return allgather(reduce_scatter(T))                                       # (s,) on D, axis=null
+\end{lstlisting}
+
+%% ============================================================
+\subsection*{{\color{callblue}The \texttt{@function} Decorator} \normalfont\small--- graph capture via tracing}
+
+The \texttt{@function} decorator transforms a Python function on Tensors into a single \op{Function} node.
+
+\begin{lstlisting}
+@function
+def f(a: Tensor, b: Tensor) -> Tensor:
+  return a + b
+\end{lstlisting}
+
+When \texttt{f(x, y)} is called, the decorator:
+
+\begin{enumerate}[leftmargin=1.5em, itemsep=2pt]
+  \item \textbf{Extracts inputs}: walks all arguments to find every Tensor, deduplicates by identity.
+  \item \textbf{Runs the function} lazily (no device execution), building a UOp graph from the result.
+  \item \textbf{Parameterizes}: replaces each input UOp with a \op{Param}$(k)$ placeholder.
+  \item \textbf{Wraps the body} in a \op{Tuple} (even for single returns) and creates\\
+        \op{Function}(\op{Tuple}(body), $x$, $y$).
+  \item \textbf{Returns} the result via \op{GetTuple}$(0)$, or one \op{GetTuple} per element for tuple returns.
+\end{enumerate}
+
+The result is a reusable graph fragment: the body contains only \op{Param} references, not concrete buffers. At schedule time, the \op{Function} is resolved by substituting each \op{Param}$(k)$ back with its corresponding argument $a_k$, or lowered into an opaque \op{Call} if it is to be compiled as a reusable kernel.
+
+%% ============================================================
+\subsection*{Lowering Pipeline \normalfont\small--- from Tensor graph to machine code}
+
+\begin{tabular}{@{}l p{9.7cm}@{}}
+\toprule
+\textbf{Stage} & \textbf{Semantics} \\
+\midrule
+\textbf{Callify} & Transform the Tensor graph into a single stateless function. \\
+\textbf{Rangeify} & Determine the kernel split of the function. Break everything down to shape () \\
+\textbf{Optimize} & Insert local buffers. Swap and split ranges, and determine which axes are parallel and which are serial. \\
+\textbf{Expand} & Expand the parallel ranges into shape. \\
+\textbf{Instruction Selection} & Select target instructions, including WMMA and devectorization. \\
+\textbf{Linearize} & Topologically sort the graph and determine execution order. \\
+\textbf{Register/Memory Plan} & Allocate and reuse \texttt{GLOBAL}, \texttt{LOCAL}, and \texttt{REG} storage for values with non-overlapping lifetimes. \\
+\textbf{Render} & Output the machine code. \\
+\bottomrule
+\end{tabular}
+
+
+\end{document}

test/amd/test_custom_kernel.py

@@ -3,7 +3,7 @@ import functools
 import numpy as np
 from tinygrad import Tensor, Device, dtypes
 from tinygrad.uop.ops import UOp, Ops, KernelInfo
-from tinygrad.engine.realize import run_linear, estimate_uop
+from tinygrad.engine.realize import run_linear, estimate_uop, compile_linear
 from tinygrad.renderer import Estimates
 from tinygrad.dtype import AddrSpace
 from tinygrad.helpers import getenv
@@ -169,7 +169,7 @@ class TestCustomKernel(unittest.TestCase):
     if self.arch != "rdna3": self.skipTest("only rdna3")
     a = Tensor.full((16, 16), 1.).contiguous().realize()
     a = Tensor.custom_kernel(a, fxn=custom_add_one)[0]
-    linear = a.schedule_linear()
+    linear = compile_linear(a.schedule_linear())
     est = estimate_uop(linear.src[-1])
     self.assertEqual(est.ops, a.numel())
     self.assertEqual(est.mem, a.nbytes()*2)

test/amd/test_sqttmap.py

@@ -1,5 +1,5 @@
 # test to compare every packet with the rocprof decoder
-import unittest, pickle, functools
+import unittest, pickle, functools, json
 from typing import Iterator
 from pathlib import Path
 from tinygrad.helpers import DEBUG, getenv, temp, ansistrip, Context
@@ -156,7 +156,7 @@ class TestSQTTMapRDNA4(TestSQTTMapBase):
     row_counts:dict[str, int] = {}
     for e in sqtt_timeline(events[1].blob, lib, target):
       if type(e).__name__ != "ProfileRangeEvent": continue
-      info = e.name.ret or ""
+      info = json.loads(e.name.ret) if e.name.ret else {}
       if e.device.startswith("WAVE"):
         idx = row_counts.get(e.device, 0)
         dispatch_st[f"{e.device}-{idx}"] = int(e.st)

test/backend/test_arange.py

@@ -2,14 +2,14 @@ import unittest
 import numpy as np
 from tinygrad import Tensor, GlobalCounters, dtypes, nn, Device, Variable
 from tinygrad.helpers import Context, getenv, DEV
-from tinygrad.engine.realize import run_linear, estimate_uop
+from tinygrad.engine.realize import run_linear, estimate_uop, compile_linear
 from tinygrad.renderer.ptx import PTXRenderer
 from test.helpers import needs_second_gpu
 
 class TestArange(unittest.TestCase):
   def _get_flops(self, tensor, desired):
     GlobalCounters.reset()
-    linear = tensor.schedule_linear()
+    linear = compile_linear(tensor.schedule_linear())
     self.assertEqual(len(linear.src), 1)
     run_linear(linear)
     np.testing.assert_equal(tensor.numpy(), desired)
@@ -36,7 +36,7 @@ class TestArange(unittest.TestCase):
   def test_tri_complexity(self):
     with Context(NOOPT=1):
       t = Tensor.ones(256, 256).contiguous().realize()
-      linear = t.triu().schedule_linear()
+      linear = compile_linear(t.triu().schedule_linear())
       self.assertLessEqual(estimate_uop(linear.src[-1]).ops, 4 * 256 * 256)
 
 DSET, DDIM = 2048, 32
@@ -229,7 +229,7 @@ class TestIndexing(unittest.TestCase):
     xq = xq.reshape(bs, seqlen, n_heads, head_dim)
     xq_rope, _ = apply_rotary_emb(xq, xq, freqs_cis)
     xq_rope.sum().backward()
-    linear = wq.grad.schedule_linear()
+    linear = compile_linear(wq.grad.schedule_linear())
     assert len(linear.src) == 1, f"expected one kernel for backward, got: {len(linear.src)}"
     bwd_ops = estimate_uop(linear.src[0]).ops
     # bfloat16 on non CDNA4 has ~10x ops overhead because of the software emulation

test/backend/test_custom_kernel.py

@@ -1,5 +1,5 @@
 import unittest
-from tinygrad import Tensor, UOp
+from tinygrad import Tensor, UOp, GlobalCounters
 from tinygrad.dtype import AddrSpace, dtypes
 from tinygrad.uop.ops import KernelInfo, AxisType
 
@@ -308,6 +308,36 @@ class TestCustomKernel(unittest.TestCase):
     expected = (3+2)*2+2
     assert all(x == expected for x in result), f"expected all {expected}, got {result}"
 
+  def test_custom_kernel_sched(self, use_custom=False):
+    x = Tensor.arange(32).reshape(8, 4).realize()
+    y = Tensor.empty_like(x)
+    y = Tensor.custom_kernel(y, x, fxn=custom_add_one_kernel)[0]
+    if use_custom:
+      z = Tensor.empty_like(x)
+      z = Tensor.custom_kernel(y, y.T.T, fxn=custom_add_one_kernel)[0]
+    else: z = y.T.T+1
+    GlobalCounters.reset()
+    z.realize()
+    self.assertEqual(GlobalCounters.kernel_count, 2)
+    self.assertEqual(z.tolist(), x.add(2).tolist())
+
+  @unittest.expectedFailure
+  def test_custom_kernel_sched_copy(self): self.test_custom_kernel_sched(use_custom=True)
+
+  @unittest.expectedFailure
+  def test_sliced_buffer_function(self):
+    x = Tensor.arange(32).reshape(8, 4).realize()
+    from tinygrad import function
+    @function(precompile=True)
+    def run(x:Tensor) -> Tensor:
+      y = Tensor.invalids(*x.shape, dtype=x.dtype)
+      return Tensor.custom_kernel(y, x, fxn=custom_add_one_kernel)[0]
+    GlobalCounters.reset()
+    y = run(x[0]).realize()
+    # it's copying the input and the output
+    self.assertEqual(GlobalCounters.kernel_count, 1)
+    self.assertEqual(y.tolist(), [1, 2, 3, 4])
+
 class TestUOpReduce(unittest.TestCase):
   def test_uop_sum(self):
     a = Tensor([1.0, 2, 3, 4, 5])

test/backend/test_jit.py

@@ -4,6 +4,7 @@ import numpy as np
 
 from hypothesis import given, settings, strategies as strat
 from test.helpers import assert_jit_cache_len, call_is_graph, not_support_multi_device, needs_second_gpu
+from tinygrad import Variable
 from tinygrad.tensor import Tensor
 from tinygrad.engine.jit import TinyJit, JitError, graph_class
 from tinygrad.device import Device
@@ -39,6 +40,19 @@ class TestJit(unittest.TestCase):
     def add(a, b): return (a+b).realize()
     _simple_test(add)
 
+  @unittest.skipUnless(Device.DEFAULT == "CPU", "core_id is a CPU runtimevar")
+  def test_hcq_core_id_runtimevar_merge(self):
+    N = 262144
+    @TinyJit
+    def f(x, st):
+      y = (x + 1).contiguous().realize()
+      z = x.shrink(((st, st + N),)).contiguous().realize()
+      return y, z
+    x = Tensor.arange(2*N).contiguous().realize()
+    for _ in range(3): y, z = f(x, Variable("a", 0, N).bind(0))
+    self.assertEqual(y.shape, (2*N,))
+    self.assertEqual(z.shape, (N,))
+
   def test_jitbeam_triggers_beam(self):
     from unittest.mock import patch
     from tinygrad.helpers import getenv as _getenv

test/backend/test_linearizer.py

@@ -14,7 +14,7 @@ from tinygrad.renderer.cstyle import CUDARenderer
 from test.helpers import replace_opts
 MOCKGPU = DEV.interface.startswith("MOCK")
 
-from tinygrad.uop.ops import print_uops # noqa: F401 # pylint: disable=unused-import
+from tinygrad.uop.render import print_uops # noqa: F401 # pylint: disable=unused-import
 
 class TestLinearizer(unittest.TestCase):
   def test_arg_dedup(self):

test/backend/test_ops.py

@@ -431,6 +431,9 @@ class TestOps(unittest.TestCase):
     helper_test_op(None, lambda x: x.round(), vals=[[1.499, 1.5, 1.501, 1.0, 2.1, 0.0, -5.0, -2.499, -2.5, -2.501]], forward_only=True)
     helper_test_op(None, lambda x: x.round(), vals=[[2.5, -1.5]], forward_only=True)
 
+  def test_round_quantization_gradient(self):
+    helper_test_op(None, lambda x: x + 0.125 * (x.round() - x), vals=[[-1.2, -0.7, -0.2, 0.2, 0.7, 1.2]])
+
   def test_isinf(self):
     val = [float('-inf'), 0., float('inf'), float('nan'), 1.1]
     helper_test_op(None, torch.isinf, Tensor.isinf, vals=[val], forward_only=True)
@@ -606,10 +609,11 @@ class TestOps(unittest.TestCase):
     helper_test_op(None, lambda x,y: x//y, forward_only=True, vals=[[5, 6, 7],[1, 2, 3]])
     helper_test_op(None, lambda x: x/2, forward_only=True, vals=[[3, 4, 5]])
     helper_test_op(None, lambda x: x//2, forward_only=True, vals=[[3, 4, 5]])
-    helper_test_op(None, functools.partial(torch.div, rounding_mode="trunc"), Tensor.idiv, forward_only=True,
+    helper_test_op(None, functools.partial(torch.div, rounding_mode="trunc"),
+                   functools.partial(Tensor.div, rounding_mode="trunc"), forward_only=True,
                    vals=[[-4, 7, 5, 4, -7, 8], [2, -3, 8, -2, 3, 5]])
     if not COMPILE_ONLY:
-      x = Tensor(2**64 - 1, dtype=dtypes.uint64).idiv(1)
+      x = Tensor(2**64 - 1, dtype=dtypes.uint64).div(1, rounding_mode="trunc")
       np.testing.assert_equal(x.numpy(), 2**64 - 1)
 
   def test_scalar_div(self):
@@ -636,6 +640,17 @@ class TestOps(unittest.TestCase):
         helper_test_op(None, lambda x: 100%x, forward_only=True, vals=[va])
         helper_test_op(None, lambda x: 100.5%x, forward_only=True, vals=[va])
 
+  def test_fmod(self):
+    a = [-4, 7, 5, 4, -7, 8, -9]
+    b = [2, -3, 8, -2, 3, 5, -5]
+    for float_a in [True, False]:
+      for float_b in [True, False]:
+        va = [float(ai) for ai in a] if float_a else a
+        vb = [float(bi) for bi in b] if float_b else b
+        helper_test_op(None, lambda x,y: x.fmod(y), forward_only=True, vals=[va, vb])
+        helper_test_op(None, lambda x: x.fmod(2), forward_only=True, vals=[va])
+        helper_test_op(None, lambda x: x.fmod(3.5), forward_only=True, vals=[va])
+
   def test_mul_naninf(self):
     helper_test_op([(45,65)], lambda x: x*math.inf)
     helper_test_op([(45,65)], lambda x: x*-math.inf)
@@ -867,10 +882,10 @@ class TestOps(unittest.TestCase):
     helper_test_op([], lambda: tor >> 31, lambda: ten >> 31, forward_only=True)
 
   def test_idiv_shift_rewrite_negative(self):
-    a = Tensor(-5).idiv(2).item()
-    b = Tensor(-5).contiguous().idiv(2).item()
+    a = Tensor(-5).div(2, rounding_mode="trunc").item()
+    b = Tensor(-5).contiguous().div(2, rounding_mode="trunc").item()
     self.assertEqual(a, b)
-    self.assertEqual(Tensor(-1).contiguous().idiv(4).item(), 0)  # NOTE this is trunc-div behaviour
+    self.assertEqual(Tensor(-1).contiguous().div(4, rounding_mode="trunc").item(), 0)  # NOTE this is trunc-div behaviour
 
   @unittest.skipIf(DEV.renderer == "NAK", "MUFU.SIN is not accurate enough")
   def test_sin(self):

test/backend/test_profiler.py

@@ -51,11 +51,11 @@ class TestProfiler(unittest.TestCase):
     TestProfiler.runtime = get_runtime(TestProfiler.d0.device, TestProfiler.prg)
     TestProfiler.b.uop.buffer.allocate()
 
-  def test_profile_kernel_run(self):
+  def test_profile_kernel_run(self, wait=False):
     runner_name = TestProfiler.runtime.name
     with helper_collect_profile(TestProfiler.d0) as profile:
       gs, ls = TestProfiler.prg.arg.launch_dims({})
-      TestProfiler.runtime(TestProfiler.b.uop.buffer._buf, TestProfiler.a.uop.buffer._buf, global_size=gs, local_size=ls)
+      TestProfiler.runtime(TestProfiler.b.uop.buffer._buf, TestProfiler.a.uop.buffer._buf, global_size=gs, local_size=ls, wait=wait)
 
     profile, _ = helper_profile_filter_device(profile, TestProfiler.d0.device)
     kernel_runs = [x for x in profile if isinstance(x, ProfileRangeEvent)]
@@ -63,6 +63,9 @@ class TestProfiler(unittest.TestCase):
     assert kernel_runs[0].name == runner_name, "kernel name is not correct"
     assert _dev_base(kernel_runs[0].device) == kernel_runs[0].device, "kernel should not be on a sub-device"
 
+  def test_profile_kernel_run_wait(self):
+    self.test_profile_kernel_run(wait=True)
+
   def test_profile_copyin(self):
     buf1 = Buffer(Device.DEFAULT, 2, dtypes.float, options=BufferSpec(nolru=True)).ensure_allocated()